diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS
index 90aca704cbe..7613dc59da5 100644
--- a/.github/CODEOWNERS
+++ b/.github/CODEOWNERS
@@ -1,57 +1,12 @@
-megatron/core/ @NVIDIA/core-adlr @NVIDIA/core-nemo
-
-megatron/core/models/gpt/ @NVIDIA/core-adlr @NVIDIA/core-nemo @NVIDIA/gpt
-
-megatron/core/models/multimodal/ @NVIDIA/core-adlr @NVIDIA/core-nemo @NVIDIA/multi-modal
-
-megatron/core/models/mamba/ @NVIDIA/core-adlr @NVIDIA/core-nemo @NVIDIA/hybrid-mamba
-megatron/core/ssm/ @NVIDIA/core-adlr @NVIDIA/core-nemo @NVIDIA/hybrid-mamba
-
-megatron/core/datasets/ @NVIDIA/core-adlr @NVIDIA/core-nemo @NVIDIA/datasets
-
-megatron/core/distributed/fsdp/ @NVIDIA/core-adlr @NVIDIA/core-nemo @NVIDIA/megatron-fsdp
-
-megatron/core/transformer/fsdp_dtensor_checkpoint.py @NVIDIA/core-adlr @NVIDIA/core-nemo @NVIDIA/megatron-fsdp
-
-megatron/core/dist_checkpointing/ @NVIDIA/core-adlr @NVIDIA/core-nemo @NVIDIA/dist-checkpointing
-
-megatron/core/optimizer/distrib_optimizer/ @NVIDIA/core-adlr @NVIDIA/core-nemo @NVIDIA/dist-optimizer
-
-megatron/core/inference/modelopt_support @NVIDIA/core-adlr @NVIDIA/core-nemo @NVIDIA/quantization-and-inference
-
-megatron/core/datasets/ @NVIDIA/core-adlr @NVIDIA/core-nemo @NVIDIA/datasets
-
-megatron/core/pipeline_parallel/ @NVIDIA/core-adlr @NVIDIA/core-nemo @NVIDIA/pipeline-parallelism
-
-megatron/core/transformer/ @NVIDIA/core-adlr @NVIDIA/core-nemo
-
-megatron/core/transformer/moe/ @NVIDIA/core-adlr @NVIDIA/core-nemo @NVIDIA/mixture-of-experts-adlr @NVIDIA/mixture-of-experts-devtech
-
-megatron/core/inference/ @NVIDIA/core-adlr @NVIDIA/core-nemo @NVIDIA/inference
-
-megatron/core/parallel_state.py @NVIDIA/core-adlr @NVIDIA/core-nemo
-
-megatron/core/post_training/ @NVIDIA/core-adlr @NVIDIA/core-nemo @NVIDIA/post-training
-
-megatron/post_training/ @NVIDIA/post-training
+* @NVIDIA/core-nemo @NVIDIA/core-devtech
.gitlab/ @NVIDIA/ci
.github/ @NVIDIA/ci
.gitlab-ci.yml @NVIDIA/ci
docker/ @NVIDIA/ci
+tests/unit_tests/run_ci_test.sh @NVIDIA/ci
+tests/test_utils/python_scripts/
tests/functional_tests/python_test_utils/ @NVIDIA/ci
tests/functional_tests/shell_test_utils/ @NVIDIA/ci
-tests/test_utils/recipes/ @NVIDIA/ci
-tests/unit_tests/run_ci_test.sh @NVIDIA/ci
-
-# API Backwards Compatibility Check
-scripts/check_api_backwards_compatibility.py @NVIDIA/ci @pablo-garay
-scripts/README_API_COMPAT.md @NVIDIA/ci @pablo-garay
-.github/workflows/check_api_backwards_compatibility_workflow.yml @NVIDIA/ci @pablo-garay
-docs/api-backwards-compatibility-check.md @NVIDIA/ci @pablo-garay
-tests/unit_tests/test_api_backwards_compat_setup.py @NVIDIA/ci @pablo-garay
-
-megatron/rl/ @NVIDIA/reinforcement-learning
-examples/rl/ @NVIDIA/reinforcement-learning
-test/unit_tests/test_rl_utils.py @NVIDIA/reinforcement-learning
-train_rl.py @NVIDIA/reinforcement-learning
+pyproject.toml @NVIDIA/ci
+uv.lock @NVIDIA/ci
diff --git a/.github/actions/action.yml b/.github/actions/action.yml
index 5c35385b036..a17b4a9a8c1 100644
--- a/.github/actions/action.yml
+++ b/.github/actions/action.yml
@@ -77,6 +77,7 @@ runs:
export PYTHONPATH=$(pwd)
export NEMORUN_HOME=$(pwd)
+ export NCCL_DEBUG=INFO
pip install --no-cache-dir uv
uv sync --only-group test
uv run python tests/test_utils/python_scripts/launch_nemo_run_workload.py \
diff --git a/.github/workflows/build-test-publish-wheel.yml b/.github/workflows/build-test-publish-wheel.yml
index 2b2ea3dfc85..bca859d0e61 100644
--- a/.github/workflows/build-test-publish-wheel.yml
+++ b/.github/workflows/build-test-publish-wheel.yml
@@ -17,6 +17,7 @@ name: Build, test, and publish a PyPi wheel (to testpypi).
on:
push:
branches:
+ - dev
- main
- 'pull-request/[0-9]+'
- 'deploy-release/*'
diff --git a/.github/workflows/check_api_backwards_compatibility_workflow.yml b/.github/workflows/check_api_backwards_compatibility_workflow.yml
index f4fcd4c3713..4ba0ed2780c 100644
--- a/.github/workflows/check_api_backwards_compatibility_workflow.yml
+++ b/.github/workflows/check_api_backwards_compatibility_workflow.yml
@@ -28,7 +28,7 @@ jobs:
uses: actions/checkout@v4
with:
fetch-depth: 0
-
+
- name: Check if relevant files changed
id: check_files
run: |
@@ -46,11 +46,11 @@ jobs:
else
# For push events, use merge-base to find common ancestor
# This ensures we only detect changes actually made in this PR branch,
- # not changes that happened in main after the branch was created
- BASE_SHA=$(git merge-base origin/main HEAD 2>/dev/null || echo "")
+ # not changes that happened in dev after the branch was created
+ BASE_SHA=$(git merge-base origin/dev HEAD 2>/dev/null || echo "")
if [ -z "$BASE_SHA" ]; then
- # Fallback for pull-request/* branches targeting dev
- BASE_SHA=$(git merge-base origin/dev HEAD 2>/dev/null || echo "")
+ # Fallback for branches targeting main
+ BASE_SHA=$(git merge-base origin/main HEAD 2>/dev/null || echo "")
fi
echo "Push event - comparing against merge-base: $BASE_SHA"
fi
@@ -83,7 +83,7 @@ jobs:
if: needs.pre-flight.outputs.should_skip != 'true'
name: "OPTIONAL: Check API Backward Compatibility"
runs-on: ubuntu-latest
-
+
# ============================================================================
# Configuration Parameters (modify here)
# ============================================================================
@@ -91,24 +91,24 @@ jobs:
# Default baseline for automatic PR checks
# Can be: branch name (e.g., 'main'), commit hash, or tag
# Will be resolved to commit hash during execution
- DEFAULT_BASELINE: '5ab481cb45efc72add12f8ba0378e849b3d2bc50'
+ DEFAULT_BASELINE: 'ed804b49860201e7103ce0f9c1129a330a384a65'
# Tag pattern for auto-detection (e.g., 'core_r*', 'core_v*')
TAG_PATTERN: 'core_v*'
# Tag regex filter (e.g., '^core_v[0-9]+\.[0-9]+\.[0-9]+$' for stable versions only)
TAG_REGEX_FILTER: '^core_v[0-9]+\.[0-9]+\.[0-9]+$'
# ============================================================================
-
+
steps:
- name: Checkout code
uses: actions/checkout@v4
with:
- fetch-depth: 0 # Need full history to access baseline ref
-
+ fetch-depth: 0 # Need full history to access baseline ref
+
- name: Set up Python
uses: actions/setup-python@v5
with:
python-version: '3.12'
-
+
- name: Install griffe
run: |
python -m pip install --upgrade pip
@@ -116,7 +116,7 @@ jobs:
python -c "import griffe; print('Griffe installed successfully')"
python -c "from griffe import Object; print('Object import successful')" || echo "Object import from griffe failed"
python -c "from griffe.dataclasses import Object; print('Object import from dataclasses successful')" || echo "Object import from dataclasses failed"
-
+
- name: Determine baseline reference
id: baseline
run: |
@@ -134,13 +134,13 @@ jobs:
# BASELINE_REF="${{ env.DEFAULT_BASELINE }}"
# fi
fi
-
+
# Resolve baseline to commit hash (works for branches, tags, or commit hashes)
BASELINE_HASH=$(git rev-parse "$BASELINE_REF")
-
+
echo "baseline=$BASELINE_HASH" >> $GITHUB_OUTPUT
echo "Using baseline: $BASELINE_REF (resolved to commit: $BASELINE_HASH)"
-
+
- name: Run compatibility check
id: compat_check
run: |
@@ -148,13 +148,13 @@ jobs:
python scripts/check_api_backwards_compatibility.py \
--baseline ${{ steps.baseline.outputs.baseline }} \
--verbose 2>&1 | tee compat_check_output.txt
-
+
# Capture exit code
EXIT_CODE=${PIPESTATUS[0]}
echo "exit_code=$EXIT_CODE" >> $GITHUB_OUTPUT
exit $EXIT_CODE
continue-on-error: true
-
+
- name: Fail job if breaking changes detected
if: steps.compat_check.outcome == 'failure'
run: |
@@ -233,10 +233,10 @@ jobs:
echo "🔧 Checker script: scripts/check_api_backwards_compatibility.py"
echo "❓ Questions? Check the docs or ask in #megatron-core"
echo ""
-
+
echo "::error::Breaking API changes detected. Please review the output above and choose a resolution strategy."
exit 1
-
+
- name: Success message
if: steps.compat_check.outcome == 'success'
run: |
@@ -271,4 +271,3 @@ jobs:
gh run view $GITHUB_RUN_ID --json jobs --jq '.jobs[] | select(.status == "completed" and .conclusion != "success" and .name != "OPTIONAL: API Backward Compatibility Check Summary") | .name'
exit 1
fi
-
diff --git a/.github/workflows/cherry-pick-release-commit.yml b/.github/workflows/cherry-pick-release-commit.yml
index 882b3f5b268..58b447939a7 100644
--- a/.github/workflows/cherry-pick-release-commit.yml
+++ b/.github/workflows/cherry-pick-release-commit.yml
@@ -17,6 +17,7 @@ on:
push:
branches:
- main
+ - dev
jobs:
cherry-pick:
diff --git a/.github/workflows/cicd-approve-test-queue.yml b/.github/workflows/cicd-approve-test-queue.yml
index f34657eb509..1c35031cb35 100644
--- a/.github/workflows/cicd-approve-test-queue.yml
+++ b/.github/workflows/cicd-approve-test-queue.yml
@@ -155,6 +155,8 @@ jobs:
workflow_id = workflow["id"]
workflow_name = workflow["display_title"]
+ pr_info = workflow.get("pull_requests", [{}])[0]
+ pr_number = pr_info.get("number", "unknown")
print(f"Approving workflow {workflow_name} with Run Id: {workflow_id}")
deployment_url = f"actions/runs/{workflow_id}/pending_deployments"
diff --git a/.github/workflows/cicd-main.yml b/.github/workflows/cicd-main.yml
index 0a102101b10..b8ca3d29047 100644
--- a/.github/workflows/cicd-main.yml
+++ b/.github/workflows/cicd-main.yml
@@ -40,6 +40,7 @@ env:
jobs:
is-not-external-contributor:
runs-on: ubuntu-latest
+ environment: nemo-ci
if: github.repository == 'NVIDIA/Megatron-LM'
outputs:
is_external_contributor: ${{ github.event.pull_request.user.type == 'User' }}
@@ -51,6 +52,7 @@ jobs:
env:
GITHUB_TOKEN: ${{ secrets.PAT }}
REPO: ${{ github.repository }}
+ DISABLE_EXTERNAL_CONTRIBUTOR: ${{ vars.DISABLE_EXTERNAL_CONTRIBUTOR }}
steps:
- name: Checkout repository
uses: actions/checkout@v4
@@ -73,17 +75,55 @@ jobs:
id: check-membership
env:
IS_MAIN_BRANCH: ${{ github.ref == 'refs/heads/main' }}
+ IS_DEV_BRANCH: ${{ github.ref == 'refs/heads/dev' }}
IS_MERGE_GROUP: ${{ github.event_name == 'merge_group' }}
SCHEDULED_JOB: ${{ github.event_name == 'schedule' }}
run: |
- # Skip SSO check for scheduled jobs, main branch, or merge groups
- if [ "${{ env.SCHEDULED_JOB }}" == "true" ] || [ "${IS_MAIN_BRANCH}" == "true" ] || [ "${IS_MERGE_GROUP}" == "true" ]; then
+ # Skip SSO check for scheduled jobs, main branch, dev branch, or merge groups
+ if [ "${{ env.SCHEDULED_JOB }}" == "true" ] || [ "${IS_MAIN_BRANCH}" == "true" ] || [ "${IS_DEV_BRANCH}" == "true" ] || [ "${IS_MERGE_GROUP}" == "true" ]; then
echo "is_maintainer=true" | tee -a $GITHUB_OUTPUT
exit 0
fi
# Use SSO membership check result
IS_MEMBER="${{ steps.check-sso.outputs.is_member }}"
+
+ # If external contributor is disabled, check if user is a repo collaborator or an org collaborator to NVIDIA or NVIDIA-NeMo
+ if [ "${{ env.DISABLE_EXTERNAL_CONTRIBUTOR }}" == "true" ] && [ "${{ steps.check-sso.outputs.is_member }}" != "true" ]; then
+ PR_AUTHOR=${{ fromJSON(steps.get-pr-info.outputs.pr-info || '{}').user.login }}
+
+ echo "Checking if $PR_AUTHOR is a repo collaborator..."
+ API_URL="https://api.github.com/repos/$REPO/collaborators/$PR_AUTHOR"
+ REPO_MEMBERSHIP_RESPONSE=$(curl -s -o /dev/null -w "%{http_code}" -L \
+ -H "Accept: application/vnd.github+json" \
+ -H "Authorization: Bearer $GITHUB_TOKEN" \
+ -H "X-GitHub-Api-Version: 2022-11-28" \
+ $API_URL)
+
+ echo "Checking if $PR_AUTHOR is an org collaborator to NVIDIA-NeMo..."
+ API_URL="https://api.github.com/orgs/NVIDIA-NeMo/members/$PR_AUTHOR"
+ ORG_NVIDIA_NEMO_MEMBERSHIP_RESPONSE=$(curl -s -o /dev/null -w "%{http_code}" -L \
+ -H "Accept: application/vnd.github+json" \
+ -H "Authorization: Bearer $GITHUB_TOKEN" \
+ -H "X-GitHub-Api-Version: 2022-11-28" \
+ $API_URL)
+
+ echo "Checking if $PR_AUTHOR is an org collaborator to NVIDIA..."
+ API_URL="https://api.github.com/orgs/NVIDIA/members/$PR_AUTHOR"
+ ORG_NVIDIA_MEMBERSHIP_RESPONSE=$(curl -s -o /dev/null -w "%{http_code}" -L \
+ -H "Accept: application/vnd.github+json" \
+ -H "Authorization: Bearer $GITHUB_TOKEN" \
+ -H "X-GitHub-Api-Version: 2022-11-28" \
+ $API_URL)
+
+ if [ "$REPO_MEMBERSHIP_RESPONSE" -eq 204 ] || [ "$ORG_NVIDIA_NEMO_MEMBERSHIP_RESPONSE" -eq 204 ] || [ "$ORG_NVIDIA_MEMBERSHIP_RESPONSE" -eq 204 ]; then
+ IS_MEMBER="true"
+ else
+ exit 1
+ fi
+ fi
+
+ # Use SSO membership check result
if [ "$IS_MEMBER" == "true" ]; then
echo "is_maintainer=true" | tee -a $GITHUB_OUTPUT
else
@@ -376,6 +416,7 @@ jobs:
- cicd-wait-in-queue
- cicd-container-build
- cicd-unit-tests-latest
+ environment: nemo-ci
if: |
(
success()
@@ -550,6 +591,7 @@ jobs:
&& needs.pre-flight.outputs.is_ci_workload == 'false'
&& !cancelled()
&& github.repository == 'NVIDIA/Megatron-LM'
+ environment: nemo-ci
steps:
- name: Generate fake coverage report
uses: actions/github-script@v6
diff --git a/.github/workflows/copyright-check.yml b/.github/workflows/copyright-check.yml
index 9bbb7a1f201..ac0d49daf9a 100644
--- a/.github/workflows/copyright-check.yml
+++ b/.github/workflows/copyright-check.yml
@@ -48,13 +48,8 @@ jobs:
&& github.repository == 'NVIDIA/Megatron-LM'
runs-on: ubuntu-latest
steps:
- - name: Checkout repository
- uses: actions/checkout@v4
-
- name: Result
env:
- GH_TOKEN: ${{ github.token }}
- GITHUB_RUN_ID: ${{ github.run_id }}
SKIPPING_IS_ALLOWED: ${{ needs.pre-flight.outputs.docs_only == 'true' || needs.pre-flight.outputs.is_deployment_workflow == 'true' || needs.pre-flight.outputs.is_merge_group == 'true' || needs.pre-flight.outputs.is_ci_workload == 'true' }}
run: |
FAILED_JOBS=$(gh run view $GITHUB_RUN_ID --json jobs --jq '[.jobs[] | select(.status == "completed" and .conclusion != "success")] | length') || echo 0
diff --git a/.github/workflows/mirror-to-main.yml b/.github/workflows/mirror-to-main.yml
new file mode 100644
index 00000000000..cb77851942b
--- /dev/null
+++ b/.github/workflows/mirror-to-main.yml
@@ -0,0 +1,129 @@
+# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+name: Mirror Dev to Main
+
+on:
+ push:
+ branches:
+ - "pull-request/[0-9]+"
+
+jobs:
+ cherry-pick-to-main:
+ runs-on: ubuntu-latest
+ permissions:
+ contents: write
+ pull-requests: write
+
+ steps:
+ - name: Checkout repository
+ uses: actions/checkout@v4
+ with:
+ fetch-depth: 0
+ token: ${{ secrets.PAT }}
+
+ - name: Get PR info
+ id: get-pr-info
+ uses: nv-gha-runners/get-pr-info@main
+
+ - name: Configure Git
+ run: |
+ git config --global user.email "github-actions[bot]@users.noreply.github.com"
+ git config --global user.name "GitHub Actions Bot"
+
+ - name: Cherry-pick to main
+ env:
+ GH_TOKEN: ${{ secrets.PAT }}
+ run: |
+ set -x
+
+ PR_NUMBER=${{ fromJSON(steps.get-pr-info.outputs.pr-info || '{}').number }}
+ BASE_REF="${{ fromJSON(steps.get-pr-info.outputs.pr-info).base.ref }}"
+ HAS_MIRROR_MAIN_LABEL=$(gh pr view $PR_NUMBER --json labels | jq '[.labels[].name] | any(. == "mirror-to-main")' || echo "false")
+ TARGET_BRANCH="cherry-pick-$PR_NUMBER-into-main"
+
+ # Skip if not labeled with mirror-to-main
+ if [ "$HAS_MIRROR_MAIN_LABEL" != "true" ]; then
+ echo "PR is not labeled with mirror-to-main, will not mirror to main."
+ exit 0
+ fi
+
+ # Skip if not targeting dev
+ if [ "$BASE_REF" != "dev" ]; then
+ echo "PR is not targeting dev, will not mirror to main."
+ exit 0
+ fi
+
+ # Check if target branch already exists
+ if git ls-remote --heads origin "refs/heads/$TARGET_BRANCH" | grep -q .; then
+ echo "Target branch already exists, will not cherry-pick again."
+ exit 0
+ fi
+
+ # Get PR details
+ PR_AUTHOR="${{ fromJSON(steps.get-pr-info.outputs.pr-info).user.login }}"
+ PR_TITLE="${{ fromJSON(steps.get-pr-info.outputs.pr-info).title }}"
+ SOURCE_BRANCH="${{ fromJSON(steps.get-pr-info.outputs.pr-info).head.ref }}"
+ SOURCE_REPO="${{ fromJSON(steps.get-pr-info.outputs.pr-info).head.repo.full_name }}"
+
+ # Fetch all branches
+ git fetch origin dev
+
+ # Handle forks vs same repo
+ if [ "$SOURCE_REPO" = "${{ github.repository }}" ]; then
+ git fetch origin "$SOURCE_BRANCH"
+ git checkout "$SOURCE_BRANCH"
+ else
+ git fetch "https://github.com/$SOURCE_REPO.git" "$SOURCE_BRANCH"
+ git checkout FETCH_HEAD
+ fi
+
+ # Find commit range to cherry-pick
+ START_COMMIT=$(git merge-base origin/dev HEAD)
+ END_COMMIT=$(git rev-parse HEAD)
+
+ # Create cherry-pick branch from main
+ git fetch origin main
+ git checkout main
+ git checkout -b "$TARGET_BRANCH"
+
+ # Cherry-pick commits
+ if ! git cherry-pick "$START_COMMIT..$END_COMMIT"; then
+ # Comment on the original PR about the failure
+ COMMENT_BODY=$(cat <<'EOF'
+ ❌ **Cherry-pick to main failed**
+
+ The cherry-pick encountered conflicts and could not be completed automatically.
+
+ **Next steps:**
+ 1. Manually create a PR with these changes to main
+ 2. Resolve any conflicts
+ EOF
+ )
+
+ gh pr comment $PR_NUMBER --body "$COMMENT_BODY"
+ exit 1
+ fi
+
+ # Push branch
+ git push -u origin "$TARGET_BRANCH"
+
+ # Create PR to main
+ gh pr create \
+ --base main \
+ --head "$TARGET_BRANCH" \
+ --title "cp: \`$PR_TITLE ($PR_NUMBER)\` into \`main\`" \
+ --body "[🤖]: Hi @$PR_AUTHOR 👋
We've cherry-picked \`$PR_TITLE (#$PR_NUMBER)\` into \`main\` for you! 🚀
Please review and approve this cherry-pick at your convenience!" \
+ --label "cherry-pick" \
+ --reviewer "$PR_AUTHOR"
+
diff --git a/.github/workflows/multi-approval-bot.yml b/.github/workflows/multi-approval-bot.yml
deleted file mode 100644
index e8507605aa7..00000000000
--- a/.github/workflows/multi-approval-bot.yml
+++ /dev/null
@@ -1,75 +0,0 @@
-name: "Codeowners Approval Workflow"
-
-on:
- push:
- branches:
- - "pull-request/[0-9]+"
- merge_group:
- types: [checks_requested]
-
-jobs:
- pre-flight:
- uses: NVIDIA-NeMo/FW-CI-templates/.github/workflows/_cicd_preflight.yml@v0.65.5
- if: github.repository == 'NVIDIA/Megatron-LM'
-
- codeowners-approval:
- needs: [pre-flight]
- runs-on: ubuntu-latest
- environment: nemo-ci
- if: |
- !(needs.pre-flight.outputs.docs_only == 'true'
- || needs.pre-flight.outputs.is_merge_group == 'true'
- || needs.pre-flight.outputs.is_deployment_workflow == 'true')
- steps:
- - name: Get PR info
- id: get-pr-info
- if: startsWith(github.ref, 'refs/heads/pull-request/')
- uses: nv-gha-runners/get-pr-info@main
-
- - name: Checkout action
- uses: actions/checkout@v3
- with:
- repository: noamelf/codeowner-multi-approval-action
- ref: v0.1
- path: codeowner-multi-approval-action
-
- - name: Check Codeowners Approval
- uses: ./codeowner-multi-approval-action
- with:
- pr-number: ${{ fromJSON(steps.get-pr-info.outputs.pr-info || '{}').number }}
- repo-name: ${{ github.repository }}
- github-token: ${{ secrets.PAT }}
-
- multi-approval-bot-summary:
- needs: [pre-flight, codeowners-approval]
- if: |
- (
- needs.pre-flight.outputs.docs_only == 'true'
- || needs.pre-flight.outputs.is_merge_group == 'true'
- || needs.pre-flight.outputs.is_deployment_workflow == 'true'
- || always()
- )
- && github.repository == 'NVIDIA/Megatron-LM'
- && !cancelled()
- runs-on: ubuntu-latest
- steps:
- - name: Checkout repository
- uses: actions/checkout@v4
-
- - name: Result
- env:
- GH_TOKEN: ${{ github.token }}
- GITHUB_RUN_ID: ${{ github.run_id }}
- SKIPPING_IS_ALLOWED: ${{ needs.pre-flight.outputs.docs_only == 'true' || needs.pre-flight.outputs.is_deployment_workflow == 'true' || needs.pre-flight.outputs.is_merge_group == 'true' || needs.pre-flight.outputs.is_ci_workload == 'true' }}
- run: |
- FAILED_JOBS=$(gh run view $GITHUB_RUN_ID --json jobs --jq '[.jobs[] | select(.status == "completed" and .conclusion != "success")] | length') || echo 0
-
- if [ "${FAILED_JOBS:-0}" -eq 0 ] || [ "$SKIPPING_IS_ALLOWED" == "true" ]; then
- echo "✅ All previous jobs completed successfully"
- exit 0
- else
- echo "❌ Found $FAILED_JOBS failed job(s)"
- # Show which jobs failed
- gh run view $GITHUB_RUN_ID --json jobs --jq '.jobs[] | select(.status == "completed" and .conclusion != "success") | .name'
- exit 1
- fi
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 53574fdea22..a238f2c9999 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -1,16 +1,16 @@
.merge_train_rule: &merge_train_rule
- UNIT_TEST: "yes"
+ UNIT_TEST: 'yes'
UNIT_TEST_REPEAT: 1
UNIT_TEST_TIMEOUT: 30
- INTEGRATION_TEST: "no"
+ INTEGRATION_TEST: 'no'
INTEGRATION_TEST_SCOPE: mr
- FUNCTIONAL_TEST: "yes"
+ FUNCTIONAL_TEST: 'yes'
FUNCTIONAL_TEST_SCOPE: mr-slim
FUNCTIONAL_TEST_REPEAT: 1
FUNCTIONAL_TEST_TIME_LIMIT: 2700
- CLUSTER_A100: ""
- CLUSTER_H100: ""
- PUBLISH: "no"
+ CLUSTER_A100: ''
+ CLUSTER_H100: ''
+ PUBLISH: 'no'
workflow:
rules:
@@ -35,30 +35,30 @@ workflow:
# For push to main
- if: $CI_PIPELINE_SOURCE == 'push' && ($CI_COMMIT_BRANCH == "main" || $CI_COMMIT_BRANCH == "dev" || $CI_COMMIT_BRANCH =~ /^core_/)
variables:
- UNIT_TEST: "no"
- INTEGRATION_TEST: "no"
- FUNCTIONAL_TEST: "yes"
+ UNIT_TEST: 'no'
+ INTEGRATION_TEST: 'no'
+ FUNCTIONAL_TEST: 'yes'
FUNCTIONAL_TEST_SCOPE: mr
FUNCTIONAL_TEST_REPEAT: 5
- FUNCTIONAL_TEST_RECORD_CHECKPOINTS: "no"
+ FUNCTIONAL_TEST_RECORD_CHECKPOINTS: 'no'
FUNCTIONAL_TEST_TIME_LIMIT: 3600
- CLUSTER_A100: ""
- CLUSTER_H100: ""
- PUBLISH: "no"
+ CLUSTER_A100: ''
+ CLUSTER_H100: ''
+ PUBLISH: 'no'
auto_cancel:
on_new_commit: interruptible
# For merge-trains that need to be fast-tracked
- if: $CI_MERGE_REQUEST_EVENT_TYPE == 'merge_train' && $CI_MERGE_REQUEST_LABELS =~ /fast-track/
variables:
- UNIT_TEST: "yes"
+ UNIT_TEST: 'yes'
UNIT_TEST_REPEAT: 1
UNIT_TEST_TIMEOUT: 30
- INTEGRATION_TEST: "no"
- FUNCTIONAL_TEST: "no"
- CLUSTER_A100: ""
- CLUSTER_H100: ""
- PUBLISH: "no"
+ INTEGRATION_TEST: 'no'
+ FUNCTIONAL_TEST: 'no'
+ CLUSTER_A100: ''
+ CLUSTER_H100: ''
+ PUBLISH: 'no'
# For normal merge-trains
- if: $CI_MERGE_REQUEST_EVENT_TYPE == 'merge_train'
@@ -67,75 +67,75 @@ workflow:
# For MRs with integration suite
- if: $CI_MERGE_REQUEST_EVENT_TYPE == 'merged_result' && $CI_MERGE_REQUEST_LABELS =~ /Run tests/
variables:
- UNIT_TEST: "yes"
+ UNIT_TEST: 'yes'
UNIT_TEST_REPEAT: 1
UNIT_TEST_TIMEOUT: 30
- INTEGRATION_TEST: "yes"
+ INTEGRATION_TEST: 'yes'
INTEGRATION_TEST_SCOPE: mr
- FUNCTIONAL_TEST: "no"
+ FUNCTIONAL_TEST: 'no'
FUNCTIONAL_TEST_SCOPE: mr-slim
FUNCTIONAL_TEST_REPEAT: 1
FUNCTIONAL_TEST_TIME_LIMIT: 2700
- CLUSTER_A100: ""
- CLUSTER_H100: ""
- PUBLISH: "no"
+ CLUSTER_A100: ''
+ CLUSTER_H100: ''
+ PUBLISH: 'no'
# For MRs with nightly
- if: $CI_MERGE_REQUEST_EVENT_TYPE == 'merged_result' && $CI_MERGE_REQUEST_LABELS =~ /Run nightly/
variables:
- UNIT_TEST: "yes"
+ UNIT_TEST: 'yes'
UNIT_TEST_REPEAT: 1
UNIT_TEST_TIMEOUT: 30
- INTEGRATION_TEST: "no"
- FUNCTIONAL_TEST: "yes"
+ INTEGRATION_TEST: 'no'
+ FUNCTIONAL_TEST: 'yes'
FUNCTIONAL_TEST_SCOPE: nightly
FUNCTIONAL_TEST_REPEAT: 5
- FUNCTIONAL_TEST_RECORD_CHECKPOINTS: "no"
+ FUNCTIONAL_TEST_RECORD_CHECKPOINTS: 'no'
FUNCTIONAL_TEST_TIME_LIMIT: 2700
- CLUSTER_A100: ""
- CLUSTER_H100: ""
- PUBLISH: "no"
+ CLUSTER_A100: ''
+ CLUSTER_H100: ''
+ PUBLISH: 'no'
# For MRs with weekly
- if: $CI_MERGE_REQUEST_EVENT_TYPE == 'merged_result' && $CI_MERGE_REQUEST_LABELS =~ /Run weekly/
variables:
- UNIT_TEST: "yes"
+ UNIT_TEST: 'yes'
UNIT_TEST_REPEAT: 1
UNIT_TEST_TIMEOUT: 30
- INTEGRATION_TEST: "no"
- FUNCTIONAL_TEST: "yes"
+ INTEGRATION_TEST: 'no'
+ FUNCTIONAL_TEST: 'yes'
FUNCTIONAL_TEST_SCOPE: weekly
FUNCTIONAL_TEST_REPEAT: 1
- FUNCTIONAL_TEST_RECORD_CHECKPOINTS: "no"
+ FUNCTIONAL_TEST_RECORD_CHECKPOINTS: 'no'
FUNCTIONAL_TEST_TIME_LIMIT: 9000
- CLUSTER_A100: ""
- CLUSTER_H100: ""
- PUBLISH: "no"
+ CLUSTER_A100: ''
+ CLUSTER_H100: ''
+ PUBLISH: 'no'
# For MRs with heavy suite
- if: $CI_MERGE_REQUEST_EVENT_TYPE == 'merged_result' && $CI_MERGE_REQUEST_LABELS =~ /Run functional tests/
variables:
- UNIT_TEST: "yes"
+ UNIT_TEST: 'yes'
UNIT_TEST_REPEAT: 1
UNIT_TEST_TIMEOUT: 30
- INTEGRATION_TEST: "no"
- FUNCTIONAL_TEST: "yes"
+ INTEGRATION_TEST: 'no'
+ FUNCTIONAL_TEST: 'yes'
FUNCTIONAL_TEST_SCOPE: mr
FUNCTIONAL_TEST_REPEAT: 1
FUNCTIONAL_TEST_TIME_LIMIT: 2700
- CLUSTER_A100: ""
- CLUSTER_H100: ""
- PUBLISH: "no"
+ CLUSTER_A100: ''
+ CLUSTER_H100: ''
+ PUBLISH: 'no'
# Default MRs
- if: $CI_MERGE_REQUEST_EVENT_TYPE == 'merged_result'
variables:
- UNIT_TEST: "yes"
+ UNIT_TEST: 'yes'
UNIT_TEST_REPEAT: 1
UNIT_TEST_TIMEOUT: 30
- INTEGRATION_TEST: "no"
- FUNCTIONAL_TEST: "no"
- PUBLISH: "no"
+ INTEGRATION_TEST: 'no'
+ FUNCTIONAL_TEST: 'no'
+ PUBLISH: 'no'
- when: never
@@ -157,104 +157,109 @@ default:
variables:
BUILD:
- value: "yes"
+ value: 'yes'
UNIT_TEST:
- value: "yes"
+ value: 'yes'
options:
- - "yes"
- - "no"
+ - 'yes'
+ - 'no'
description: To run the funtional test suite
UNIT_TEST_REPEAT:
- value: "1"
- description: "Number of repetitions"
+ value: '1'
+ description: 'Number of repetitions'
UNIT_TEST_TIMEOUT:
- value: "30"
+ value: '30'
description: Timeout (minutes) for Unit tests (all repeats)
INTEGRATION_TEST:
- value: "yes"
+ value: 'yes'
options:
- - "yes"
- - "no"
+ - 'yes'
+ - 'no'
description: To run the integration test suite
INTEGRATION_TEST_SCOPE:
- value: "mr"
+ value: 'mr'
options:
- - "mr"
- - "nightly"
- - "weekly"
- - "pre-release"
- - "release"
- description: "Testsuite to run (only for INTEGRATION_TEST=yes)"
+ - 'mr'
+ - 'nightly'
+ - 'weekly'
+ - 'pre-release'
+ - 'release'
+ description: 'Testsuite to run (only for INTEGRATION_TEST=yes)'
INTEGRATION_TEST_TIME_LIMIT:
- value: "900"
- description: "Timeout in seconds per test"
+ value: '900'
+ description: 'Timeout in seconds per test'
INTEGRATION_TEST_CASES:
- value: "all"
+ value: 'all'
description: "Comma-separated list of test_cases to run. Use 'all' to run the full suite."
FUNCTIONAL_TEST:
- value: "yes"
+ value: 'yes'
options:
- - "yes"
- - "no"
+ - 'yes'
+ - 'no'
description: To run the funtional test suite
FUNCTIONAL_TEST_SCOPE:
- value: "mr"
+ value: 'mr'
options:
- - "mr"
- - "nightly"
- - "weekly"
- - "pre-release"
- - "release"
- description: "Testsuite to run (only for FUNCTIONAL_TEST=yes)"
+ - 'mr'
+ - 'nightly'
+ - 'weekly'
+ - 'pre-release'
+ - 'release'
+ description: 'Testsuite to run (only for FUNCTIONAL_TEST=yes)'
FUNCTIONAL_TEST_REPEAT:
- value: "5"
- description: "Number of repetitions per test"
+ value: '5'
+ description: 'Number of repetitions per test'
FUNCTIONAL_TEST_TIME_LIMIT:
- value: "2700"
- description: "Timeout in seconds per test"
+ value: '2700'
+ description: 'Timeout in seconds per test'
FUNCTIONAL_TEST_CASES:
- value: "all"
+ value: 'all'
description: "Comma-separated list of test_cases to run. Use 'all' to run the full suite."
FUNCTIONAL_TEST_NAME:
- description: "Name of functional test run (only for pre-release and release)"
- value: "$$CI_COMMIT_SHA"
+ description: 'Name of functional test run (only for pre-release and release)'
+ value: '$$CI_COMMIT_SHA'
FUNCTIONAL_TEST_RECORD_CHECKPOINTS:
- value: "no"
- description: "Record golden checkpoints"
+ value: 'no'
+ description: 'Record golden checkpoints'
options:
- - "yes"
- - "no"
+ - 'yes'
+ - 'no'
CLUSTER_A100:
- value: "dgxa100_dracooci"
+ value: 'dgxa100_dracooci'
options:
- - "dgxa100_dracooci"
- - "dgxa100_dracooci-ord"
- description: "Cluster for A100 workloads"
+ - 'dgxa100_dracooci'
+ - 'dgxa100_dracooci-ord'
+ description: 'Cluster for A100 workloads'
CLUSTER_H100:
- value: "dgxh100_coreweave"
+ value: 'dgxh100_coreweave'
options:
- - "dgxh100_coreweave"
- - "dgxh100_eos"
- description: "Cluster for H100 workloads"
+ - 'dgxh100_coreweave'
+ - 'dgxh100_eos'
+ description: 'Cluster for H100 workloads'
+ CLUSTER_GB200:
+ value: 'dgxgb200_oci-hsg'
+ options:
+ - 'dgxgb200_oci-hsg'
+ description: 'Cluster for H100 workloads'
PUBLISH:
- value: "no"
+ value: 'no'
options:
- - "yes"
- - "no"
+ - 'yes'
+ - 'no'
description: Build and publish a wheel to PyPi
PUBLISH_COMMIT:
- value: "$$CI_COMMIT_SHA"
+ value: '$$CI_COMMIT_SHA'
description: Which commit to publish
PUBLISH_VERSION_BUMP_BRANCH:
- value: "$$CI_COMMIT_BRANCH"
+ value: '$$CI_COMMIT_BRANCH'
description: Which branch to target for version bump
PUBLISH_SCOPE:
- value: "code-freeze"
+ value: 'code-freeze'
options:
- - "code-freeze"
- - "release"
- - "review-reminder"
- - "upgrade-dependencies"
+ - 'code-freeze'
+ - 'release'
+ - 'review-reminder'
+ - 'upgrade-dependencies'
description: Type of publish (freeze or final release)
# CI wide variables
@@ -262,7 +267,7 @@ variables:
CI_MCORE_DEV_IMAGE: ${GITLAB_ENDPOINT}:5005/adlr/megatron-lm/mcore_ci_dev
CI_NEMO_IMAGE: ${GITLAB_ENDPOINT}:5005/adlr/megatron-lm/nemo_ci
UTILITY_IMAGE: ${GITLAB_ENDPOINT}:5005/adlr/megatron-lm/mcore_utility
- TE_GIT_REF: ""
+ TE_GIT_REF: ''
include:
- .gitlab/stages/00.pre.yml
diff --git a/.gitlab/scripts/build.sh b/.gitlab/scripts/build.sh
index e64434e834d..9bcf5d45712 100644
--- a/.gitlab/scripts/build.sh
+++ b/.gitlab/scripts/build.sh
@@ -20,17 +20,25 @@ docker buildx create --name container --driver=docker-container --use tls-enviro
ADDITIONAL_PARAMS=()
+CI_COMMIT_BRANCH="${CI_COMMIT_BRANCH:-$CI_MERGE_REQUEST_SOURCE_BRANCH_NAME}"
+
if [[ "$CI_COMMIT_BRANCH" == "ci-rebuild-mcore-nemo-image" || "$CI_COMMIT_BRANCH" == "main" || "$CI_COMMIT_BRANCH" == "dev" ]]; then
ADDITIONAL_PARAMS+=("--pull")
- ADDITIONAL_PARAMS+=("--cache-to type=registry,ref=${IMAGE}-buildcache:main,mode=max")
- ADDITIONAL_PARAMS+=("-t ${IMAGE}:${CI_COMMIT_BRANCH}")
-elif [[ -n "$CI_MERGE_REQUEST_IID" ]]; then
- ADDITIONAL_PARAMS+=("--cache-to type=registry,ref=${IMAGE}-buildcache:${CI_MERGE_REQUEST_IID},mode=max")
- ADDITIONAL_PARAMS+=("-t ${IMAGE}:${CI_MERGE_REQUEST_IID}")
+fi
+
+CI_COMMIT_BRANCH=$(echo "$CI_COMMIT_BRANCH" | tr '/' '-' | tr '[:upper:]' '[:lower:]' | sed 's/[^a-z0-9._-]/-/g')
+ADDITIONAL_PARAMS+=("--cache-to type=registry,ref=${IMAGE}-buildcache:${CI_COMMIT_BRANCH}-${PLATFORM},mode=max")
+ADDITIONAL_PARAMS+=("--cache-from type=registry,ref=${IMAGE}-buildcache:${CI_COMMIT_BRANCH}-${PLATFORM}")
+ADDITIONAL_PARAMS+=("-t ${IMAGE}:${CI_COMMIT_BRANCH}-${PLATFORM}")
+
+if [[ -n "$CI_MERGE_REQUEST_IID" ]]; then
+ ADDITIONAL_PARAMS+=("--cache-to type=registry,ref=${IMAGE}-buildcache:${CI_MERGE_REQUEST_IID}-${PLATFORM},mode=max")
+ ADDITIONAL_PARAMS+=("--cache-from type=registry,ref=${IMAGE}-buildcache:${CI_MERGE_REQUEST_IID}-${PLATFORM}")
+ ADDITIONAL_PARAMS+=("-t ${IMAGE}:${CI_MERGE_REQUEST_IID}-${PLATFORM}")
fi
if [[ "$CI_COMMIT_BRANCH" == "ci-nightly" ]]; then
- ADDITIONAL_PARAMS+=("-t ${IMAGE}:nightly")
+ ADDITIONAL_PARAMS+=("-t ${IMAGE}:nightly-${PLATFORM}")
fi
if [[ -n "$TE_GIT_REF" ]]; then
@@ -46,13 +54,11 @@ DOCKER_BUILDKIT=1 docker build \
--secret id=LOGGER_INDEX_URL \
--target $STAGE \
-f docker/$FILE \
- -t ${IMAGE}:${CI_PIPELINE_ID} \
+ -t ${IMAGE}:${CI_PIPELINE_ID}-${PLATFORM} \
--builder=container \
--build-arg JET_API_VERSION=$JET_API_VERSION \
- --cache-from type=registry,ref=${IMAGE}-buildcache:${CI_MERGE_REQUEST_IID} \
- --cache-from type=registry,ref=${IMAGE}-buildcache:dev \
- --cache-from type=registry,ref=${IMAGE}-buildcache:main \
--build-arg FROM_IMAGE_NAME=$BASE_IMAGE \
+ --provenance=false \
--push \
--progress plain \
${ADDITIONAL_PARAMS[@]} .
diff --git a/.gitlab/stages/00.pre.yml b/.gitlab/stages/00.pre.yml
index d3ac804e599..ff9e4e5178b 100644
--- a/.gitlab/stages/00.pre.yml
+++ b/.gitlab/stages/00.pre.yml
@@ -68,6 +68,7 @@ pre:create_ci_branches_dev:
- branch: ci-dev-rebuild-mcore-nemo-image
- branch: ci-dev-mr
- branch: ci-dev-nightly
+ - branch: ci-dev-weekly
- branch: ci-dev-upgrade-dependencies
tags:
- arch/amd64
diff --git a/.gitlab/stages/01.build.yml b/.gitlab/stages/01.build.yml
index d67225311f6..20252e7d045 100644
--- a/.gitlab/stages/01.build.yml
+++ b/.gitlab/stages/01.build.yml
@@ -9,10 +9,10 @@
extends: [.build_rules, .dind_rules]
stage: build
tags:
- - arch/amd64
+ - arch/${PLATFORM}
- origin/jet-fleet
- env/prod
- - ${TAG}
+ - purpose/builder-large
services:
- name: docker:24.0.5-dind
variables:
@@ -23,7 +23,6 @@
DOCKER_TLS_CERTDIR: "/certs"
DOCKER_TLS_VERIFY: 1
DOCKER_CERT_PATH: "$DOCKER_TLS_CERTDIR/client"
- TAG: purpose/builder-large
STAGE: jet
MCORE_BACKWARDS_REF: core_r0.14.0
KUBERNETES_SERVICE_MEMORY_REQUEST: 90Gi
@@ -48,7 +47,7 @@
reports:
dotenv: build.env
-test:build_image:
+test:pre_build_image:
extends: [.build_image]
parallel:
matrix:
@@ -56,13 +55,30 @@ test:build_image:
FILE: Dockerfile.ci.dev
IMAGE_TYPE: lts
BASE_IMAGE: nvcr.io/nvidia/pytorch:25.09-py3
+ PLATFORM: amd64
+ - IMAGE: CI_MCORE_LTS_IMAGE
+ FILE: Dockerfile.ci.dev
+ IMAGE_TYPE: lts
+ BASE_IMAGE: nvcr.io/nvidia/pytorch:25.09-py3
+ PLATFORM: arm64
- IMAGE: CI_MCORE_DEV_IMAGE
FILE: Dockerfile.ci.dev
IMAGE_TYPE: dev
- BASE_IMAGE: nvcr.io/nvidia/pytorch:25.09-py3
+ BASE_IMAGE: nvcr.io/nvidia/pytorch:25.11-py3
+ PLATFORM: amd64
+ - IMAGE: CI_MCORE_DEV_IMAGE
+ FILE: Dockerfile.ci.dev
+ IMAGE_TYPE: dev
+ BASE_IMAGE: nvcr.io/nvidia/pytorch:25.11-py3
+ PLATFORM: arm64
- IMAGE: UTILITY_IMAGE
FILE: Dockerfile.linting
BASE_IMAGE: python:3.10
+ PLATFORM: amd64
+ - IMAGE: UTILITY_IMAGE
+ FILE: Dockerfile.linting
+ BASE_IMAGE: python:3.10
+ PLATFORM: arm64
test:build_nemo_image:
extends: [.build_image]
@@ -70,6 +86,57 @@ test:build_nemo_image:
IMAGE: CI_NEMO_IMAGE
FILE: Dockerfile.ci.nemo
BASE_IMAGE: nvcr.io/nvidian/nemo:nightly
+ PLATFORM: amd64
rules:
- if: $FUNCTIONAL_TEST == "yes" || $INTEGRATION_TEST == "yes" || $CI_COMMIT_BRANCH == "ci-rebuild-mcore-nemo-image"
when: on_success
+
+test:build_image:
+ needs: [test:pre_build_image]
+ extends: [.build_rules, .dind_rules]
+ parallel:
+ matrix:
+ - IMAGE: CI_MCORE_LTS_IMAGE
+ - IMAGE: CI_MCORE_DEV_IMAGE
+ - IMAGE: UTILITY_IMAGE
+ stage: build
+ tags:
+ - arch/amd64
+ - origin/jet-fleet
+ - env/prod
+ - purpose/builder-large
+ services:
+ - name: docker:24.0.5-dind
+ variables:
+ HEALTHCHECK_TCP_PORT: "2376"
+ timeout: 180m
+ variables:
+ DOCKER_HOST: tcp://docker:2376
+ DOCKER_TLS_CERTDIR: "/certs"
+ DOCKER_TLS_VERIFY: 1
+ DOCKER_CERT_PATH: "$DOCKER_TLS_CERTDIR/client"
+ STAGE: jet
+ MCORE_BACKWARDS_REF: core_r0.14.0
+ KUBERNETES_SERVICE_MEMORY_REQUEST: 90Gi
+ KUBERNETES_SERVICE_MEMORY_LIMIT: 90Gi
+ SHARED_PATH: /builds/$CI_PROJECT_PATH/shared
+ script:
+ - |
+ set -x
+
+ env
+ eval "IMAGE=\$$IMAGE"
+
+ docker manifest create ${IMAGE}:${CI_PIPELINE_ID} \
+ ${IMAGE}:${CI_PIPELINE_ID}-amd64 \
+ ${IMAGE}:${CI_PIPELINE_ID}-arm64
+
+ docker manifest push ${IMAGE}:${CI_PIPELINE_ID}
+ - echo "MCORE_MR_COMMIT=$CI_COMMIT_SHA" | tee -a build.env
+ - echo "MCORE_BACKWARDS_COMMIT=$MCORE_BACKWARDS_COMMIT" | tee -a build.env
+ - cat build.env
+ retry:
+ max: 2
+ artifacts:
+ reports:
+ dotenv: build.env
diff --git a/.gitlab/stages/03.integration-tests.yml b/.gitlab/stages/03.integration-tests.yml
index 824721b9fb1..d28ecd8e137 100644
--- a/.gitlab/stages/03.integration-tests.yml
+++ b/.gitlab/stages/03.integration-tests.yml
@@ -43,6 +43,7 @@ integration:configure:
- |
A100_CLUSTER=$([[ "$CLUSTER_A100" != "" ]] && echo $CLUSTER_A100 || echo $DEFAULT_A100_CLUSTER)
H100_CLUSTER=$([[ "$CLUSTER_H100" != "" ]] && echo $CLUSTER_H100 || echo $DEFAULT_H100_CLUSTER)
+ GB200_CLUSTER=$([[ "$CLUSTER_GB200" != "" ]] && echo $CLUSTER_GB200 || echo $DEFAULT_GB200_CLUSTER)
- |
ARGS=(
"--scope $INTEGRATION_TEST_SCOPE"
@@ -88,12 +89,30 @@ integration:configure:
--platform dgx_h100 \
--cluster $H100_CLUSTER \
--output-path "functional-test-job-lts-H100.yaml"
+ - |
+ export PYTHONPATH=$(pwd)
+ python tests/test_utils/python_scripts/generate_jet_trigger_job.py \
+ ${ARGS[@]} \
+ --environment lts \
+ --platform dgx_gb2100 \
+ --cluster $GB200_CLUSTER \
+ --output-path "functional-test-job-lts-GB200.yaml"
+ - |
+ export PYTHONPATH=$(pwd)
+ python tests/test_utils/python_scripts/generate_jet_trigger_job.py \
+ ${ARGS[@]} \
+ --environment lts \
+ --platform dgx_gb200 \
+ --cluster $GB200_CLUSTER \
+ --output-path "functional-test-job-lts-GB200.yaml"
artifacts:
paths:
- functional-test-job-lts-A100.yaml
- functional-test-job-lts-H100.yaml
- functional-test-job-dev-H100.yaml
- functional-test-job-dev-A100.yaml
+ - functional-test-job-lts-GB200.yaml
+ - functional-test-job-dev-GB200.yaml
- tests/test_utils/local_recipes
.integration_run:
@@ -132,6 +151,12 @@ integration:run_lts_dgx_h100:
ENVIRONMENT: lts
CLUSTER: H100
+integration:run_lts_dgx_gb200:
+ extends: [.integration_run]
+ variables:
+ ENVIRONMENT: lts
+ CLUSTER: GB200
+
integration:run_dev_dgx_a100:
extends: [.integration_run]
variables:
@@ -143,3 +168,9 @@ integration:run_dev_dgx_h100:
variables:
ENVIRONMENT: dev
CLUSTER: H100
+
+integration:run_dev_dgx_gb200:
+ extends: [.integration_run]
+ variables:
+ ENVIRONMENT: dev
+ CLUSTER: GB200
diff --git a/.gitlab/stages/04.functional-tests.yml b/.gitlab/stages/04.functional-tests.yml
index e9aab319ab1..d32ff86a344 100644
--- a/.gitlab/stages/04.functional-tests.yml
+++ b/.gitlab/stages/04.functional-tests.yml
@@ -50,6 +50,7 @@ functional:configure:
- |
A100_CLUSTER=$([[ "$CLUSTER_A100" != "" ]] && echo $CLUSTER_A100 || echo $DEFAULT_A100_CLUSTER)
H100_CLUSTER=$([[ "$CLUSTER_H100" != "" ]] && echo $CLUSTER_H100 || echo $DEFAULT_H100_CLUSTER)
+ GB200_CLUSTER=$([[ "$CLUSTER_GB200" != "" ]] && echo $CLUSTER_GB200 || echo $DEFAULT_GB200_CLUSTER)
- |
RECORD_CHECKPOINTS=$([[ "$CI_MERGE_REQUEST_LABELS" == *"Record checkpoints"* || "$FUNCTIONAL_TEST_RECORD_CHECKPOINTS" == "yes" ]] && echo "true" || echo "false")
- |
@@ -113,12 +114,32 @@ functional:configure:
--cluster $H100_CLUSTER \
--output-path "functional-test-job-lts-H100.yaml" \
${RELEASE_ARGS[@]}
+ - |
+ export PYTHONPATH=$(pwd)
+ python tests/test_utils/python_scripts/generate_jet_trigger_job.py \
+ ${ARGS[@]} \
+ --environment dev \
+ --platform dgx_gb200 \
+ --cluster $GB200_CLUSTER \
+ --output-path "functional-test-job-dev-GB200.yaml" \
+ ${RELEASE_ARGS[@]}
+ - |
+ export PYTHONPATH=$(pwd)
+ python tests/test_utils/python_scripts/generate_jet_trigger_job.py \
+ ${ARGS[@]} \
+ --environment lts \
+ --platform dgx_gb200 \
+ --cluster $GB200_CLUSTER \
+ --output-path "functional-test-job-lts-GB200.yaml" \
+ ${RELEASE_ARGS[@]}
artifacts:
paths:
- functional-test-job-lts-A100.yaml
- functional-test-job-lts-H100.yaml
- functional-test-job-dev-A100.yaml
- functional-test-job-dev-H100.yaml
+ - functional-test-job-lts-GB200.yaml
+ - functional-test-job-dev-GB200.yaml
- tests/test_utils/local_recipes
.functional_run:
@@ -157,6 +178,12 @@ functional:run_lts_dgx_h100:
ENVIRONMENT: lts
CLUSTER: H100
+functional:run_lts_dgx_gb200:
+ extends: [.functional_run]
+ variables:
+ ENVIRONMENT: lts
+ CLUSTER: GB200
+
functional:run_dev_dgx_a100:
extends: [.functional_run]
variables:
@@ -169,6 +196,12 @@ functional:run_dev_dgx_h100:
ENVIRONMENT: dev
CLUSTER: H100
+functional:run_dev_dgx_gb200:
+ extends: [.functional_run]
+ variables:
+ ENVIRONMENT: dev
+ CLUSTER: GB200
+
functional:run_nemo:
extends: [.functional_tests_rules]
trigger:
@@ -178,7 +211,7 @@ functional:run_nemo:
inherit:
variables: true
variables:
- MCORE_COMMIT: $CI_COMMIT_SHA
+ MCORE_MR_COMMIT: $CI_COMMIT_SHA
TEST_NEMO2_MODULE: 'True'
ALLOW_FAILURE_DEPENDENCY: 'True'
TESTS_TO_RUN_ON_THIS_COMMIT: nightly
@@ -217,7 +250,7 @@ functional:x_notify:
- export RO_API_TOKEN=${PROJECT_ACCESS_TOKEN_MCORE}
- export GITLAB_ENDPOINT
- export CONTEXT=$FUNCTIONAL_TEST_SCOPE
- - export TAG_TEAM=$([[ "$CI_COMMIT_BRANCH" == "main" ]] && echo "1" || "0")
+ - export TAG_TEAM=$([[ "$CI_COMMIT_BRANCH" == "main" || "$CI_COMMIT_BRANCH" == "dev" ]] && echo "1" || "0")
- export TEAM_SLUG=$SLACK_ADMIN
- |
python tests/test_utils/python_scripts/notify.py \
@@ -231,7 +264,7 @@ functional:x_notify:
paths:
- scripts
rules:
- - if: ($CI_PIPELINE_SOURCE == "schedule" || $CI_COMMIT_BRANCH == "main") && $FUNCTIONAL_TEST == "yes"
+ - if: ($CI_PIPELINE_SOURCE == "schedule" || $CI_COMMIT_BRANCH == "main" || $CI_COMMIT_BRANCH == "dev") && $FUNCTIONAL_TEST == "yes"
when: always
- when: never
diff --git a/README.md b/README.md
index b3174a0b758..abb581c9b34 100644
--- a/README.md
+++ b/README.md
@@ -11,477 +11,66 @@ Megatron-LM & Megatron Core
-## ⚡ Quick Start
+> ## 🚨 **DEVELOPMENT BRANCH**
+> ⚠️ **EXPERIMENTAL FEATURES** - This is the **dev branch** with experimental features.
+>
+> **→ For releases and comprehensive documentation, visit the [main branch](https://github.com/NVIDIA/Megatron-LM)**
-```bash
-# 1. Install Megatron Core with required dependencies
-pip install --no-build-isolation megatron-core[mlm,dev]
+## ⚡ Quickstart
-# 2. Clone repository for examples
-git clone https://github.com/NVIDIA/Megatron-LM.git
+```bash
+# Clone the dev branch
+git clone -b dev https://github.com/NVIDIA/Megatron-LM.git
cd Megatron-LM
-pip install --no-build-isolation .[mlm,dev]
-```
-
-**→ [Complete Installation Guide](#installation)** - Docker, pip variants (dev,lts,etc.), and system requirements
-
-# Latest News
-- **[2025/12]** 🎉 **Megatron Core development has moved to GitHub!** All development and CI now happens in the open. We welcome community contributions.
-- **[2025/10]** **[Megatron Dev Branch](https://github.com/NVIDIA/Megatron-LM/tree/dev)** - early access branch with experimental features.
-- **[2025/10]** **[Megatron Bridge](https://github.com/NVIDIA-NeMo/Megatron-Bridge)** - Bidirectional converter for interoperability between Hugging Face and Megatron checkpoints, featuring production-ready recipes for popular models.
-- **[2025/08]** **[MoE Q3-Q4 2025 Roadmap](https://github.com/NVIDIA/Megatron-LM/issues/1729)** - Comprehensive roadmap for MoE features including DeepSeek-V3, Qwen3, advanced parallelism strategies, FP8 optimizations, and Blackwell performance enhancements.
-- **[2025/08]** **[GPT-OSS Model](https://github.com/NVIDIA/Megatron-LM/issues/1739)** - Advanced features including YaRN RoPE scaling, attention sinks, and custom activation functions are being integrated into Megatron Core.
-- **[2025/06]** **[Megatron MoE Model Zoo](https://github.com/yanring/Megatron-MoE-ModelZoo)** - Best practices and optimized configurations for training DeepSeek-V3, Mixtral, and Qwen3 MoE models with performance benchmarking and checkpoint conversion tools.
-- **[2025/05]** Megatron Core v0.11.0 brings new capabilities for multi-data center LLM training ([blog](https://developer.nvidia.com/blog/turbocharge-llm-training-across-long-haul-data-center-networks-with-nvidia-nemo-framework/)).
-
-
-Previous News
-
-- **[2024/07]** Megatron Core v0.7 improves scalability and training resiliency and adds support for multimodal training ([blog](https://developer.nvidia.com/blog/train-generative-ai-models-more-efficiently-with-new-nvidia-Megatron-Core-functionalities/)).
-- **[2024/06]** Megatron Core added supports for Mamba-based models. Check out our paper [An Empirical Study of Mamba-based Language Models](https://arxiv.org/pdf/2406.07887) and [code example](https://github.com/NVIDIA/Megatron-LM/tree/ssm/examples/mamba).
-- **[2024/01 Announcement]** NVIDIA has released the core capabilities in **Megatron-LM** into [**Megatron Core**](https://github.com/NVIDIA/Megatron-LM/tree/main/megatron/core) in this repository. Megatron Core expands upon Megatron-LM's GPU-optimized techniques with more cutting-edge innovations on system-level optimizations, featuring composable and modular APIs. Explore the [Megatron Core intro](#Megatron Core) for more details.
-
-
+# Install from source with dev dependencies (includes transformer_engine)
+pip install -e .[mlm,dev]
+```
Table of Contents
**Getting Started**
+- [⚡ Quick Start](#-quick-start)
+- [🧠 Dev Branch Philosophy](#-dev-branch-philosophy)
+- [📊 Performance & Benchmarking](#-performance--benchmarking)
+- [👥 Community & Support](#-community--support)
-- [Quick Start](#-quick-start)
-- [Latest News](#latest-news)
-- [Megatron Overview](#megatron-overview)
- - [Project Structure](#project-structure)
- - [Megatron-LM: Reference Implementation](#megatron-lm-reference-implementation)
- - [Megatron Core: Production Library](#megatron-core-production-library)
-- [Installation](#installation)
- - [Docker (Recommended)](#-docker-recommended)
- - [Pip Installation](#pip-installation)
- - [System Requirements](#system-requirements)
-
-**Core Features**
-
-- [Performance Benchmarking](#performance-benchmarking)
- - [Weak Scaling Results](#weak-scaling-results)
- - [Strong Scaling Results](#strong-scaling-results)
-- [Ecosystem Libraries](#ecosystem-libraries)
-
-**Training**
-
-- [Training](#training)
- - [Getting Started](#getting-started)
- - [Data Preparation](#data-preparation)
-- [Parallelism Strategies](#parallelism-strategies)
- - [Data Parallelism (DP)](#data-parallelism-dp)
- - [Tensor Parallelism (TP)](#tensor-parallelism-tp)
- - [Pipeline Parallelism (PP)](#pipeline-parallelism-pp)
- - [Context Parallelism (CP)](#context-parallelism-cp)
- - [Expert Parallelism (EP)](#expert-parallelism-ep)
- - [Parallelism Selection Guide](#parallelism-selection-guide)
-- [Performance Optimizations](#performance-optimizations)
-
-**Resources**
-
-- [Examples](./examples/) - Training scripts and tutorials
-- [Documentation](https://docs.nvidia.com/Megatron-Core/) - Official docs
-- [Roadmaps](#roadmaps) - Development roadmaps and feature tracking
-- [Community & Support](#community--support) - Get help and contribute
- - [Getting Help](#getting-help)
- - [Contributing](#contributing)
- - [Citation](#citation)
+**For Complete Documentation** → [Main Branch](https://github.com/NVIDIA/Megatron-LM) | [Official Docs](https://docs.nvidia.com/Megatron-Core/)
-# Megatron Overview
-
-## Project Structure
-
-```
-Megatron-LM/
-├── megatron/
-│ ├── core/ # Megatron Core (kernels, parallelism, building blocks)
-│ │ ├── models/ # Transformer models
-│ │ ├── transformer/ # Transformer building blocks
-│ │ ├── tensor_parallel/ # Tensor parallelism
-│ │ ├── pipeline_parallel/ # Pipeline parallelism
-│ │ ├── distributed/ # Distributed training (FSDP, DDP)
-│ │ ├── optimizer/ # Optimizers
-│ │ ├── datasets/ # Dataset loaders
-│ │ ├── inference/ # Inference engines
-│ │ └── export/ # Model export (e.g. TensorRT-LLM)
-│ ├── training/ # Training scripts
-│ ├── inference/ # Inference server
-│ ├── legacy/ # Legacy components
-│ └── post_training/ # Post-training (RLHF, etc.)
-├── examples/ # Ready-to-use training examples
-├── tools/ # Utility tools
-├── tests/ # Comprehensive test suite
-└── docs/ # Documentation
-```
-
-### Megatron-LM: Reference Implementation
-
-**Reference implementation** that includes Megatron Core plus everything needed to train models.
-
-**Best for:**
-
-- **Training state-of-the-art foundation models** at scale with cutting-edge performance on latest NVIDIA hardware
-- **Research teams** exploring new architectures and training techniques
-- **Learning distributed training** concepts and best practices
-- **Quick experimentation** with proven model configurations
-
-**What you get:**
-
-- Pre-configured training scripts for GPT, LLama, DeepSeek, Qwen, and more.
-- End-to-end examples from data prep to evaluation
-- Research-focused tools and utilities
-
-### Megatron Core: Composable Library
-
-**Composable library** with GPU-optimized building blocks for custom training frameworks.
-
-**Best for:**
-
-- **Framework developers** building on top of modular and optimized components
-- **Research teams** needing custom training loops, optimizers, or data pipelines
-- **ML engineers** requiring fault-tolerant training pipelines
-
-**What you get:**
-
-- Composable transformer building blocks (attention, MLP, etc.)
-- Advanced parallelism strategies (TP, PP, DP, EP, CP)
-- Pipeline schedules and distributed optimizers
-- Mixed precision support (FP16, BF16, FP8)
-- GPU-optimized kernels and memory management
-- High-performance dataloaders and dataset utilities
-- Model architectures (LLaMA, Qwen, GPT, Mixtral, Mamba, etc.)
-
-## Ecosystem Libraries
-
-**Libraries used by Megatron Core:**
-
-- **[Megatron Energon](https://github.com/NVIDIA/Megatron-Energon)** 📣 **NEW!** - Multi-modal data loader (text, images, video, audio) with distributed loading and dataset blending
-- **[Transformer Engine](https://github.com/NVIDIA/TransformerEngine)** - Optimized kernels and FP8 mixed precision support
-- **[Resiliency Extension (NVRx)](https://github.com/NVIDIA/nvidia-resiliency-ext)** - Fault tolerant training with failure detection and recovery
-
-**Libraries using Megatron Core:**
-
-- **[Megatron Bridge](https://github.com/NVIDIA-NeMo/Megatron-Bridge)** - Training library with bidirectional Hugging Face ↔ Megatron checkpoint conversion, flexible training loops, and production-ready recipes
-- **[NeMo RL](https://github.com/NVIDIA-NeMo/RL)** - Scalable toolkit for efficient reinforcement learning with RLHF, DPO, and other post-training methods
-- **[NeMo Framework](https://docs.nvidia.com/nemo-framework/user-guide/latest/overview.html)** - Enterprise framework with cloud-native support and end-to-end examples
-- **[Model Optimizer (ModelOpt)](https://github.com/NVIDIA/Model-Optimizer)** - Model optimization toolkit for quantization, pruning, distillation, speculative decoding, and more. Checkout end-to-end examples in [examples/post_training/modelopt](./examples/post_training/modelopt/).
-
-**Compatible with:** [Hugging Face Accelerate](https://github.com/huggingface/accelerate), [Colossal-AI](https://github.com/hpcaitech/ColossalAI), [DeepSpeed](https://github.com/microsoft/DeepSpeed)
-
-# Installation
-## 🐳 Docker (Recommended)
-We strongly recommend using the previous releases of [PyTorch NGC Container](https://catalog.ngc.nvidia.com/orgs/nvidia/containers/pytorch) rather than the latest one for optimal compatibility with Megatron Core release and testing. Our releases are always based on the previous month's NGC container, so this ensures compatibility and stability.
-**Note:** The NGC PyTorch container constraints the python environment globally via `PIP_CONSTRAINT`. In the following examples we will unset the variable.
-This container comes with all dependencies pre-installed with compatible versions and optimized configurations for NVIDIA GPUs:
-- PyTorch (latest stable version)
-- CUDA, cuDNN, NCCL (latest stable versions)
-- Support for FP8 on NVIDIA Hopper, Ada, and Blackwell GPUs
-- For best performance, use NVIDIA Turing GPU architecture generations and later
+## Dev Branch Philosophy
-```bash
-# Run container with mounted directories
-docker run --runtime --nvidia --gpus all -it --rm \
- -v /path/to/megatron:/workspace/megatron \
- -v /path/to/dataset:/workspace/dataset \
- -v /path/to/checkpoints:/workspace/checkpoints \
- -e PIP_CONSTRAINT= \
- nvcr.io/nvidia/pytorch:25.04-py3
-```
-
-## Pip Installation
-
-Megatron Core offers support for two NGC PyTorch containers:
-
-- `dev`: Moving head that supports the most recent upstream dependencies
-- `lts`: Long-term support of NGC PyTorch 24.01
-
-Both containers can be combined with `mlm` which adds package dependencies for Megatron-LM on top of Megatron Core.
-
-```bash
-# Install the latest release dependencies
-pip install "setuptools<80.0.0,>=77.0.0" "packaging>=24.2"
-pip install --no-build-isolation megatron-core[dev]
-# For running an M-LM application:
-pip install "setuptools<80.0.0,>=77.0.0" "packaging>=24.2"
-pip install --no-build-isolation megatron-core[mlm,dev]
-```
-
-```bash
-# Install packages for LTS support NGC PyTorch 24.01
-pip install "setuptools<80.0.0,>=77.0.0" "packaging>=24.2"
-pip install --no-build-isolation megatron-core[lts]
-# For running an M-LM application:
-pip install "setuptools<80.0.0,>=77.0.0" "packaging>=24.2"
-pip install --no-build-isolation megatron-core[mlm,lts]
-```
-
-For a version of Megatron Core with only torch, run:
-
-```bash
-pip install megatron-core
-```
-
-## System Requirements
-
-### Hardware Requirements
-
-- **FP8 Support**: NVIDIA Hopper, Ada, Blackwell GPUs
-- **Recommended**: NVIDIA Turing architecture or later
-
-### Software Requirements
+### Fast Iteration
+- **Streamlined Review**: 1 code owner + 1 dev approver (can delegate review) + CI/CD
-- **CUDA/cuDNN/NCCL**: Latest stable versions
-- **PyTorch**: Latest stable version
-- **Transformer Engine**: Latest stable version
-- **Python**: 3.12 recommended
+### Feature Lifecycle (Coming Soon)
+- **6-Month Timeline**: Experimental features must graduate to stable or be deprecated
+- **Migration Support**: Assistance provided for feature transitions
-# Performance Benchmarking
+### Stability Expectations
+- **Experimental Nature**: Features may change or be removed as development progresses
+- **Testing**: All features will pass convergence and performance validation before inclusion
+- **Support**: Dev branch issues should include `[DEV]` prefix
-For our latest performance benchmarking results, please refer to [NVIDIA NeMo Framework Performance Summary](https://docs.nvidia.com/nemo-framework/user-guide/latest/performance/performance_summary.html).
+## Performance & Benchmarking
-Our codebase efficiently trains models from 2B to 462B parameters across thousands of GPUs, achieving up to **47% Model FLOP Utilization (MFU)** on H100 clusters.
+- 🚀 [2025/11] [Optimizing DeepSeek-V3 Training Performance on NVIDIA GB200 NVL72](docs/discussions/deepseek-v3-gb200-optimization/deepseek-v3-gb200-optimization.md).
+- ⚡ [2025/11] [A Guide to Reproduce DeepSeek-V3 Pre-training Performance on GB200](docs/discussions/deepseek-v3-gb200-optimization/deepseek-v3-gb200-reproduce-guide.md).
-
-
-**Benchmark Configuration:**
-
-- **Vocabulary size**: 131,072 tokens
-- **Sequence length**: 4096 tokens
-- **Model scaling**: Varied hidden size, attention heads, and layers to achieve target parameter counts
-- **Communication optimizations**: Fine-grained overlapping with DP (`--overlap-grad-reduce`, `--overlap-param-gather`), TP (`--tp-comm-overlap`), and PP (enabled by default)
-
-**Key Results:**
-
-- **6144 H100 GPUs**: Successfully benchmarked 462B parameter model training
-- **Superlinear scaling**: MFU increases from 41% to 47-48% with model size
-- **End-to-end measurement**: Throughputs include all operations (data loading, optimizer steps, communication, logging)
-- **Production ready**: Full training pipeline with checkpointing and fault tolerance
-- *Note: Performance results measured without training to convergence*
-
-## Weak Scaling Results
-
-Our weak scaled results show superlinear scaling (MFU increases from 41% for the smallest model considered to 47-48% for the largest models); this is because larger GEMMs have higher arithmetic intensity and are consequently more efficient to execute.
-
-
-
-## Strong Scaling Results
-
-We also strong scaled the standard GPT-3 model (our version has slightly more than 175 billion parameters due to larger vocabulary size) from 96 H100 GPUs to 4608 GPUs, using the same batch size of 1152 sequences throughout. Communication becomes more exposed at larger scale, leading to a reduction in MFU from 47% to 42%.
-
-
-
-# Training
-
-## Getting Started
-
-### Simple Training Example
-
-```bash
-# Distributed training example (2 GPUs, mock data)
-torchrun --nproc_per_node=2 examples/run_simple_mcore_train_loop.py
-```
-
-### LLama-3 Training Example
-
-```bash
-# 8 GPUs, FP8 precision, mock data
-./examples/llama/train_llama3_8b_fp8.sh
-```
-
-## Data Preparation
-
-### JSONL Data Format
-
-```json
-{"text": "Your training text here..."}
-{"text": "Another training sample..."}
-```
-
-### Basic Preprocessing
-
-```bash
-python tools/preprocess_data.py \
- --input data.jsonl \
- --output-prefix processed_data \
- --tokenizer-type HuggingFaceTokenizer \
- --tokenizer-model /path/to/tokenizer.model \
- --workers 8 \
- --append-eod
-```
-
-### Key Arguments
-
-- `--input`: Path to input JSON/JSONL file
-- `--output-prefix`: Prefix for output binary files (.bin and .idx)
-- `--tokenizer-type`: Tokenizer type (`HuggingFaceTokenizer`, `GPT2BPETokenizer`, etc.)
-- `--tokenizer-model`: Path to tokenizer model file
-- `--workers`: Number of parallel workers for processing
-- `--append-eod`: Add end-of-document token
-
-
-
-# Parallelism Strategies
-
-## Data Parallelism (DP)
-
-### Standard Data Parallel
-
-```bash
-# Standard DDP - replicate model on each GPU
-torchrun --nproc_per_node=8 pretrain_gpt.py \
- --data-parallel-sharding-strategy no_shard
-```
-
-### Fully Sharded Data Parallel (FSDP)
-
-```bash
-# Megatron's optimized FSDP (~15% faster than PyTorch FSDP2)
---use-custom-fsdp
-
-# PyTorch FSDP2
---use-torch-fsdp2
-
-# Sharding strategies
---data-parallel-sharding-strategy optim # Shard optimizer states (ZeRO-1)
---data-parallel-sharding-strategy optim_grads # Shard gradients + optimizer (ZeRO-2)
---data-parallel-sharding-strategy optim_grads_params # Shard parameters + gradients + optimizer (ZeRO-3)
-```
-
-## Tensor Parallelism (TP)
-
-Split individual model layers across GPUs:
-
-```bash
---tensor-model-parallel-size 4 # 4-way tensor parallelism
---sequence-parallel # Enable sequence parallelism (recommended with TP)
-```
-
-## Pipeline Parallelism (PP)
-
-Split model depth across GPUs:
-
-```bash
---pipeline-model-parallel-size 8 # 8 pipeline stages
---virtual-pipeline-model-parallel-size 4 # Virtual pipeline for better load balancing
-```
-
-## Context Parallelism (CP)
-
-Split long sequences across GPUs for handling long contexts:
-
-```bash
---context-parallel-size 2 # 2-way context parallelism
---cp-comm-type p2p # Communication: p2p, a2a, allgather, a2a+p2p
---hierarchical-context-parallel-sizes 2 4 # Hierarchical context parallelism
-```
-
-## Expert Parallelism (EP)
-
-For Mixture of Experts (MoE) models:
-
-```bash
---expert-model-parallel-size 4 # 4-way expert parallelism
---num-experts 8 # 8 experts per MoE layer
---moe-grouped-gemm # Optimize expert computation
-```
-
-## Combining Parallelism Strategies
-
-### Parallelism Selection Guide
-
-Based on [NVIDIA NeMo production configurations](https://github.com/NVIDIA/NeMo/tree/main/scripts/performance/recommended_model_configs):
-
-| Model | Size | GPUs | TP | PP | CP | EP | Notes |
-|-------|------|------|----|----|----|----|-------|
-| **LLama-3** | 8B | 8 | 1 | 1 | 2 | 1 | CP for long seqlen (8K) |
-| **LLama-3** | 70B | 64 | 4 | 4 | 2 | 1 | TP+PP |
-| **LLama-3.1** | 405B | 1024 | 8 | 8 | 2 | 1 | 3D parallelism for scale |
-| **GPT-3** | 175B | 128-512 | 4 | 8 | 1 | 1 | Large model config |
-| **Mixtral** | 8x7B | 64 | 1 | 4 | 1 | 8 | EP for MoE |
-| **Mixtral** | 8x22B | 256 | 4 | 4 | 8 | 8 | Combined TP+EP for large MoE |
-| **DeepSeek-V3** | 671B | 1024 | 2 | 16 | 1 | 64 | Large MoE config |
-
-### MoE-Specific Requirements
-
-**Important**: When combining Expert Parallelism (EP) with Tensor Parallelism (TP), **Sequence Parallelism (SP) must be enabled**.
-
-## Performance Optimizations
-
-| Feature | Flag | Benefit |
-|---------|------|---------|
-| **FlashAttention** | `--attention-backend` | Faster attention and lower memory usage |
-| **FP8 Training** | `--fp8-hybrid` | Faster training |
-| **Activation Checkpointing** | `--recompute-activations` | Reduced memory usage |
-| **Data Parallelism Communication Overlap** | `--overlap-grad-reduce` | Faster distributed training |
-| **Distributed Optimizer** | `--use-distributed-optimizer` | Reduced checkpointing time |
-
-**→ [NVIDIA NeMo Framework Performance Tuning Guide](https://docs.nvidia.com/nemo-framework/user-guide/latest/performance/performance-guide.html#performance-tuning-guide)** - Comprehensive performance optimization guide covering advanced tuning techniques, communication overlaps, memory optimizations, and profiling options.
-
-### FlashAttention
-
-[FlashAttention](https://github.com/Dao-AILab/flash-attention) is a fast and memory-efficient attention algorithm. We recommend the default usage, which uses cuDNN for attention via Transformer Engine and provides up to 50% speedups on forward and 84% on backward propagation with FP8 kernels. The `flash-attn` package is also supported via `--use-flash-attn`.
-
-### Mixed Precision Training
-
-```bash
---fp16 # Standard FP16
---bf16 # BFloat16 (recommended for large models)
---fp8-hybrid # FP8 training (Hopper, Ada, and Blackwell GPUs)
-```
-
-### Activation Checkpointing and Recomputation
-
-```bash
-# For limited memory
---recompute-activations
-
-# For extreme memory constraints
---recompute-granularity full \
---recompute-method uniform
-```
-
-### Data Parallelism Communication Overlap
-
-```bash
---overlap-grad-reduce
---overlap-param-gather
-```
-
-### Distributed Optimizer
-
-```bash
---use-distributed-optimizer
-```
-
-# Roadmaps
-
-Stay up-to-date with our development roadmaps and planned features:
-
-- **[MoE Q3-Q4 2025 Roadmap](https://github.com/NVIDIA/Megatron-LM/issues/1729)** - Comprehensive MoE feature development including DeepSeek-V3, Qwen3, advanced parallelism, FP8 optimizations, and Blackwell enhancements
-- **[GPT-OSS Implementation Tracker](https://github.com/NVIDIA/Megatron-LM/issues/1739)** - Advanced features including YaRN RoPE scaling, attention sinks, and custom activation functions
-
-*More roadmap trackers will be added soon.*
-
-# Community & Support
-
-## Getting Help
+## Community & Support
+### Getting Help
- 📖 **[Documentation](https://docs.nvidia.com/Megatron-Core/)** - Official documentation
- 🐛 **[Issues](https://github.com/NVIDIA/Megatron-LM/issues)** - Bug reports and feature requests
-## Contributing
-
+### Contributing
We ❤️ contributions! Ways to contribute:
- 🐛 **Report bugs** - Help us improve reliability
@@ -491,8 +80,7 @@ We ❤️ contributions! Ways to contribute:
**→ [Contributing Guide](./CONTRIBUTING.md)**
-## Citation
-
+### Citation
```bibtex
@article{megatron-lm,
title={Megatron-LM: Training Multi-Billion Parameter Language Models Using Model Parallelism},
diff --git a/docker/.ngc_version.dev b/docker/.ngc_version.dev
index 6b72812b34f..8e8108b9a9a 100644
--- a/docker/.ngc_version.dev
+++ b/docker/.ngc_version.dev
@@ -1 +1 @@
-nvcr.io/nvidia/pytorch:25.09-py3
\ No newline at end of file
+nvcr.io/nvidia/pytorch:25.11-py3
\ No newline at end of file
diff --git a/docker/Dockerfile.ci.dev b/docker/Dockerfile.ci.dev
index 9ed7f88a0a5..4e1a4de55e8 100644
--- a/docker/Dockerfile.ci.dev
+++ b/docker/Dockerfile.ci.dev
@@ -1,3 +1,5 @@
+# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved.
+
# syntax=docker/dockerfile:1.3-labs
ARG FROM_IMAGE_NAME
@@ -15,10 +17,17 @@ ENV UV_LINK_MODE=copy
RUN bash -ex <<"EOF"
apt-get update
- apt-get install -y --no-install-recommends gettext python3-venv psmisc
+ apt-get install -y --no-install-recommends gettext python3-venv psmisc uuid-runtime
apt-get clean
python -m venv /opt/jet
- wget https://github.com/mikefarah/yq/releases/download/v${YQ_VERSION}/yq_linux_amd64 -O /usr/local/bin/yq
+ ARCH=$(uname -m)
+ case "${ARCH}" in \
+ "x86_64") YQ_ARCH=amd64 ;; \
+ "aarch64") YQ_ARCH=arm64 ;; \
+ "armv7l") YQ_ARCH=arm ;; \
+ *) echo "Unsupported architecture: ${ARCH}" && exit 1 ;; \
+ esac
+ wget https://github.com/mikefarah/yq/releases/download/v${YQ_VERSION}/yq_linux_${YQ_ARCH} -O /usr/local/bin/yq
chmod a+x /usr/local/bin/yq
curl -LsSf https://astral.sh/uv/${UV_VERSION}/install.sh | sh
EOF
@@ -55,14 +64,14 @@ EOF
COPY docker/patches/deepep.patch /workspace/deepep.patch
RUN bash -ex <<"EOF"
cd /workspace
- uv pip install nvidia-nvshmem-cu13
+ uv pip install nvidia-nvshmem-cu13==3.4.5
pushd /opt/venv/lib/python3.12/site-packages/nvidia/nvshmem/lib/
ln -s libnvshmem_host.so.3 libnvshmem_host.so
popd
- git clone --branch hybrid-ep https://github.com/deepseek-ai/DeepEP.git
+ git clone --branch hybrid-ep https://github.com/Autumn1998/DeepEP.git
pushd DeepEP
- git checkout 83e0d156807f31abed4ea55c2fa6eb4b62a11b82
+ git checkout df375b40f24e5c495e2db36e808125266661652c
patch -p1 < /workspace/deepep.patch
popd
TORCH_CUDA_ARCH_LIST="9.0 10.0 12.0" uv pip install --no-build-isolation -v DeepEP/.
@@ -71,6 +80,7 @@ EOF
COPY assets/ /opt/data/
ENV UV_PYTHON=$UV_PROJECT_ENVIRONMENT/bin/python
+COPY . /opt/megatron-lm/
##### For NVIDIANS only #####
FROM main as jet
@@ -89,6 +99,6 @@ RUN --mount=type=secret,id=JET_INDEX_URLS \
LOGGER_INDEX_URL=$(cat /run/secrets/LOGGER_INDEX_URL)
uv pip install --no-cache-dir --upgrade $LOGGER_INDEX_URL "one-logger"
uv pip install --no-cache-dir --upgrade "setuptools<80.0.0"
- uv pip install --no-cache-dir --upgrade $JET_INDEX_URLS "jet-client~=3.0"
+ uv pip install --no-cache-dir --upgrade $JET_INDEX_URLS "jet-client~=4.0"
EOF
###
diff --git a/docker/Dockerfile.ci.nemo b/docker/Dockerfile.ci.nemo
index 2369602f54d..93fe23bfd6f 100644
--- a/docker/Dockerfile.ci.nemo
+++ b/docker/Dockerfile.ci.nemo
@@ -1,3 +1,5 @@
+# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved.
+
# syntax=docker/dockerfile:1.3-labs
ARG FROM_IMAGE_NAME
@@ -14,7 +16,7 @@ FROM main as jet
ARG JET_API_VERSION
RUN --mount=type=secret,id=JET_INDEX_URLS \
JET_INDEX_URLS=$(cat /run/secrets/JET_INDEX_URLS) && \
- pip install --no-cache-dir jet-api==$JET_API_VERSION "jet-client~=3.0" --upgrade $JET_INDEX_URLS
+ pip install --no-cache-dir jet-api==$JET_API_VERSION "jet-client~=4.0" --upgrade $JET_INDEX_URLS
ENV PATH="$PATH:/opt/jet/bin"
###
diff --git a/docs/api-guide/custom_fsdp.md b/docs/api-guide/custom_fsdp.md
index e265de8ae4b..faa262ee7fa 100644
--- a/docs/api-guide/custom_fsdp.md
+++ b/docs/api-guide/custom_fsdp.md
@@ -13,6 +13,8 @@ Add these flag to enable MCore custom FSDP.
--use-distributed-optimizer
```
+For a practical guide covering required configurations, checkpoint conversion, and example scripts, see the [Megatron-FSDP User Guide](../../discussions/megatron-fsdp-user-guide/megatron-fsdp-user-guide.md).
+
## Key Features
- **Sharding Strategy**: Efficiently shards optimizer states, gradients, and parameters to reduce memory consumption.
diff --git a/docs/api-guide/fine_grained_activation_offloading.md b/docs/api-guide/fine_grained_activation_offloading.md
new file mode 100644
index 00000000000..53211d1d06c
--- /dev/null
+++ b/docs/api-guide/fine_grained_activation_offloading.md
@@ -0,0 +1,31 @@
+# Fine-grained Activation Offloading (collaborated with rednote)
+
+Memory capacity is more and more important with the rising of extreme sparse MoE models like DeepSeek-V3 and Qwen3-235B. Fine-grained recomputing reduces the memory footprint at the cost of extra recomputation, while offloading could utilize the host-device bandwidth to achieve nearly zero-overhead. Fine-grained Activation Offloading targets at offloading the activation at the granularity of specific modules, so that we can calibrate the amount of offloading activation to maximize the training throughput.
+
+Currently, the supported offloading modules are `"attn_norm", "core_attn", "attn_proj", "mlp_norm", "expert_fc1", "moe_act"`, which could work with fine-grained recomputation to release almost all activations of a transformer layer.
+
+**Features**
+* Support PP=1/PP/Interleaved PP
+* Compatible with fine-grained recomputation
+* Support FP8
+* Support MTP
+* Support mixed dense & moe layer
+* Support A2A Overlap
+* Support CUDA Graph
+ * (Temporary) cuda graph scope cannot contains the offloading modules
+
+**Usage**
+```bash
+# Enable fine-grained activation offloading
+--fine-grained-activation-offloading
+
+# Specify which modules are going to offload its input
+# Choices: "attn_norm", "core_attn", "attn_proj", "mlp_norm", "expert_fc1", "moe_act".
+--offload-modules expert_fc1
+```
+**Compatible with Fine-grained Recomputation**
+- For modules with minor perf overhead like layernorm or moe_act, use recomputing to reduce memory footprint;
+- For other modules, use offloading to reduce memory footprint;
+- Make sure the offloading/reloading could be overlapped with computing;
+
+
diff --git a/docs/discussions/README.md b/docs/discussions/README.md
new file mode 100644
index 00000000000..81b1a58d5b0
--- /dev/null
+++ b/docs/discussions/README.md
@@ -0,0 +1,28 @@
+# Megatron Discussions
+
+This directory contains in-depth guides, tutorials, and discussions about optimizing and using Megatron for various use cases.
+
+## Available Guides
+
+### Performance Optimization
+
+- **[A Guide to Reproduce DeepSeek-V3 Pre-training Performance on GB200](deepseek-v3-gb200-optimization/deepseek-v3-gb200-reproduce-guide.md)**
+
+ A detailed guide on how to reproduce the DeepSeek-V3 pre-training performance on GB200, incluing the dockerfile, package requirements and training scripts.
+
+### Training Guides
+
+- **[Megatron-FSDP User Guide](megatron-fsdp-user-guide/megatron-fsdp-user-guide.md)**
+
+ A practical guide to enable Megatron-FSDP training, including a quick-start example for DeepSeek-V3, required and recommended configurations, and instructions for checkpoint conversion from torch_dist to fsdp_dtensor.
+
+## Contributing
+
+If you'd like to contribute a guide or tutorial, please follow this structure:
+
+1. Create a new directory: `docs/discussions/your-guide-name/`
+2. Add your main guide: `docs/discussions/your-guide-name/your-guide-name.md`
+3. Create an images directory: `docs/discussions/your-guide-name/images/`
+4. Update this README.md with a link to your guide
+
+Each guide should be self-contained with its own images and supporting files.
diff --git a/docs/discussions/deepseek-v3-gb200-optimization/deepseek-v3-gb200-reproduce-guide.md b/docs/discussions/deepseek-v3-gb200-optimization/deepseek-v3-gb200-reproduce-guide.md
new file mode 100644
index 00000000000..61bd8289c66
--- /dev/null
+++ b/docs/discussions/deepseek-v3-gb200-optimization/deepseek-v3-gb200-reproduce-guide.md
@@ -0,0 +1,354 @@
+# A Guide to Reproduce DeepSeek-V3 Pre-training Performance on GB200
+
+## 1. Dockerfile
+
+Requirements:
+- Transformer Engine: We recommend using commit [d2945c6](https://github.com/NVIDIA/TransformerEngine/commit/d2945c6a571e3978677614d1fe08779966a5a4ef) with PR [2146](https://github.com/NVIDIA/TransformerEngine/pull/2146) and [2150](https://github.com/NVIDIA/TransformerEngine/pull/2150). You could prepare the branch by yourself, or use this [branch](https://github.com/hxbai/TransformerEngine/commits/dev_20251024/) based on TE v2.9 plus the above three commits/PRs.
+- cuDNN: v9.14 is required.
+- HybridEP: Install it from [here](https://github.com/deepseek-ai/DeepEP/commits/3f601f7ac1c062c46502646ff04c535013bfca00).
+
+Dockerfile for reference.
+
+```dockerfile
+FROM nvcr.io/nvidia/pytorch:25.09-py3 AS base
+
+ENV SHELL=/bin/bash
+
+# =========================
+# Install system packages
+# =========================
+RUN rm -rf /opt/megatron-lm && \
+ apt-get update && \
+ apt-get install -y sudo gdb bash-builtins git zsh autojump tmux curl gettext libfabric-dev && \
+ wget https://github.com/mikefarah/yq/releases/download/v4.27.5/yq_linux_arm64 -O /usr/bin/yq && \
+ chmod +x /usr/bin/yq
+
+# =========================
+# Install Python packages
+# =========================
+# NOTE: `unset PIP_CONSTRAINT` to install packages that do not meet the default constraint in the base image.
+# Some package requirements and related versions are from
+# https://github.com/NVIDIA/Megatron-LM/blob/core_v0.12.0/Dockerfile.linting.
+# https://github.com/NVIDIA/Megatron-LM/blob/core_v0.12.0/requirements_mlm.txt.
+# https://github.com/NVIDIA/Megatron-LM/blob/core_v0.12.0/requirements_ci.txt.
+RUN unset PIP_CONSTRAINT && pip install --no-cache-dir debugpy dm-tree torch_tb_profiler einops wandb \
+ sentencepiece tokenizers transformers torchvision ftfy modelcards datasets tqdm pydantic \
+ nvidia-pytriton py-spy yapf darker \
+ tiktoken flask-restful \
+ nltk wrapt pytest pytest_asyncio pytest-cov pytest_mock pytest-random-order \
+ black==24.4.2 isort==5.13.2 flake8==7.1.0 pylint==3.2.6 coverage mypy \
+ setuptools==69.5.1
+
+# =========================
+# Install cudnn 9.14.0.64 for correct mxfp8 quantization and layernorm fusion
+# =========================
+RUN apt-get update && \
+ wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2404/sbsa/cuda-keyring_1.1-1_all.deb && \
+ dpkg -i cuda-keyring_1.1-1_all.deb && \
+ apt-get update && \
+ apt-get -y install libcudnn9-cuda-13
+
+# =========================
+# Install latest TE
+# Use a specific commit instead of main to make it more stable.
+# This is based on release_v2.9 branch and contains some CPU and quantization optimizations.
+# =========================
+ARG COMMIT="7dd3914726abb79bc99ff5a5db1449458ed64151"
+ARG TE="git+https://github.com/hxbai/TransformerEngine.git@${COMMIT}"
+RUN pip install nvidia-mathdx==25.1.1 && \
+ unset PIP_CONSTRAINT && \
+ NVTE_CUDA_ARCHS="100" NVTE_BUILD_THREADS_PER_JOB=8 NVTE_FRAMEWORK=pytorch pip install --no-build-isolation --no-cache-dir $TE
+
+# =========================
+# Install HybridEP
+# =========================
+WORKDIR /home/
+RUN git clone --branch hybrid-ep https://github.com/deepseek-ai/DeepEP.git && \
+ cd DeepEP && git checkout 3f601f7ac1c062c46502646ff04c535013bfca00 && \
+ TORCH_CUDA_ARCH_LIST="10.0" pip install --no-build-isolation .
+
+# =========================
+# Clean cache
+# =========================
+RUN rm -rf /root/.cache /tmp/*
+```
+
+> [!Tip]
+>
+> If you prefer to use CUDA 12.9, please change the base container to `nvcr.io/nvidia/pytorch:25.06-py3` and the cuDNN to be installed to `libcudnn9-cuda-12`.
+
+## 2. Megatron-Core
+
+We recommend using the [dev branch](https://github.com/NVIDIA/Megatron-LM/tree/dev) after PR [1917](https://github.com/NVIDIA/Megatron-LM/pull/1917).
+
+```bash
+git clone https://github.com/NVIDIA/Megatron-LM.git && \
+cd Megatron-LM &&
+git checkout effebd81f410bc6566fffee6c320b6f8f762e06d
+```
+
+## 3. Cluster Configuration
+
+Since we're using EP 32 on NVL72, it's important to make sure
+
+> [!Important]
+> **Every 32 GB200 GPUs (8 nodes) are in the same NVL domain (or rack)**.
+
+Usually you can make it via your cluster workload manager. Taking Slurm as an example, you could pass `--segment 8` to the sbatch command to ensure that every segment of 8 nodes will be scheduled to a rack.
+
+## 4. Training scripts
+
+### Environment variables
+
+```bash
+CUDA_DEVICE_MAX_CONNECTIONS=1
+NVTE_FWD_LAYERNORM_SM_MARGIN=0
+NVTE_BWD_LAYERNORM_SM_MARGIN=0
+NVLINK_DOMAIN_SIZE=72
+NVTE_ALLOW_NONDETERMINISTIC_ALGO=1
+PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True
+NCCL_NVLS_ENABLE=0
+NVTE_FUSED_ATTN=1
+NVTE_NORM_FWD_USE_CUDNN=1
+NVTE_NORM_BWD_USE_CUDNN=1
+PYTHONWARNINGS=ignore
+NCCL_DEBUG=VERSION
+NCCL_GRAPH_REGISTER=0
+```
+
+### bindpcie
+
+Download [bindpcie](https://github.com/NVIDIA/mlperf-common/blob/main/client/bindpcie) to your workdir, make it executable,
+
+```bash
+wget https://raw.githubusercontent.com/NVIDIA/mlperf-common/refs/heads/main/client/bindpcie &&
+chmod 755 bindpcie
+```
+
+and then
+
+> [!Important]
+> **Place it at the beginning of your launch command in every process.**
+
+Taking Slurm as an example, your script should look like
+
+```bash
+#!/bin/bash
+
+#SBATCH [... sbatch args]
+
+srun [... srun args] /path/to/bindpcie /path/to/pretrain_gpt.py [... mcore arguments]
+```
+
+This is a very important step on GB200.
+
+### Launch script
+
+```bash
+/path/to/bindpcie \
+/path/to/megatron-lm/pretrain_gpt.py \
+--distributed-timeout-minutes 60 \
+--tensor-model-parallel-size 1 \
+--pipeline-model-parallel-size 8 \
+--expert-model-parallel-size 32 \
+--context-parallel-size 1 \
+--expert-tensor-parallel-size 1 \
+--use-distributed-optimizer \
+--overlap-grad-reduce \
+--overlap-param-gather \
+--use-mcore-models \
+--sequence-parallel \
+--use-flash-attn \
+--disable-bias-linear \
+--micro-batch-size 1 \
+--global-batch-size 2048 \
+--train-samples 585937500 \
+--exit-duration-in-mins 220 \
+--no-save-optim \
+--no-check-for-nan-in-loss-and-grad \
+--cross-entropy-loss-fusion \
+--cross-entropy-fusion-impl te \
+--manual-gc \
+--manual-gc-interval 10 \
+--enable-experimental \
+--transformer-impl transformer_engine \
+--seq-length 4096 \
+--data-cache-path /path/to/data_cache \
+--tokenizer-type HuggingFaceTokenizer \
+--tokenizer-model unsloth/DeepSeek-V3 \
+--data-path /path/to/data \
+--split 99,1,0 \
+--no-mmap-bin-files \
+--no-create-attention-mask-in-dataloader \
+--num-workers 6 \
+--num-layers 61 \
+--hidden-size 7168 \
+--ffn-hidden-size 18432 \
+--num-attention-heads 128 \
+--kv-channels 128 \
+--max-position-embeddings 4096 \
+--position-embedding-type rope \
+--rotary-base 10000 \
+--make-vocab-size-divisible-by 3232 \
+--normalization RMSNorm \
+--norm-epsilon 1e-6 \
+--swiglu \
+--untie-embeddings-and-output-weights \
+--multi-latent-attention \
+--attention-dropout 0.0 \
+--hidden-dropout 0.0 \
+--clip-grad 1.0 \
+--weight-decay 0.1 \
+--qk-layernorm \
+--lr-decay-samples 584765624 \
+--lr-warmup-samples 1536000 \
+--lr-warmup-init 3.9e-7 \
+--lr 3.9e-6 \
+--min-lr 3.9e-7 \
+--lr-decay-style cosine \
+--adam-beta1 0.9 \
+--adam-beta2 0.95 \
+--num-experts 256 \
+--moe-layer-freq ([0]*3+[1]*58) \
+--moe-ffn-hidden-size 2048 \
+--moe-shared-expert-intermediate-size 2048 \
+--moe-router-load-balancing-type seq_aux_loss \
+--moe-router-topk 8 \
+--moe-grouped-gemm \
+--moe-aux-loss-coeff 1e-4 \
+--moe-router-group-topk 4 \
+--moe-router-num-groups 8 \
+--moe-router-pre-softmax \
+--moe-router-padding-for-quantization \
+--moe-router-topk-scaling-factor 2.5 \
+--moe-router-score-function sigmoid \
+--moe-router-enable-expert-bias \
+--moe-router-bias-update-rate 1e-3 \
+--moe-router-dtype fp32 \
+--moe-permute-fusion \
+--moe-router-fusion \
+--q-lora-rank 1536 \
+--kv-lora-rank 512 \
+--qk-head-dim 128 \
+--qk-pos-emb-head-dim 64 \
+--v-head-dim 128 \
+--rotary-scaling-factor 40 \
+--mscale 1.0 \
+--mscale-all-dim 1.0 \
+--eval-iters 32 \
+--eval-interval 200 \
+--no-load-optim \
+--no-load-rng \
+--auto-detect-ckpt-format \
+--load None \
+--save /path/to/checkpoints \
+--save-interval 500 \
+--dist-ckpt-strictness log_all \
+--init-method-std 0.02 \
+--log-timers-to-tensorboard \
+--log-memory-to-tensorboard \
+--log-validation-ppl-to-tensorboard \
+--log-throughput \
+--log-interval 1 \
+--logging-level 40 \
+--tensorboard-dir /path/to/tensorboard \
+--wandb-project deepseek-v3-benchmarking-v0.15 \
+--wandb-exp-name DeepSeek-V3-TP1PP8EP32CP1VPP4-MBS1GBS2048-v0.15 \
+--bf16 \
+--enable-experimental \
+--recompute-granularity selective \
+--recompute-modules moe_act mlp \
+--cuda-graph-impl transformer_engine \
+--cuda-graph-scope attn moe_router moe_preprocess \
+--te-rng-tracker \
+--pipeline-model-parallel-layout "Et|(tt|)*30L" \
+--moe-router-force-load-balancing \
+--moe-token-dispatcher-type flex \
+--moe-flex-dispatcher-backend hybridep \
+--moe-hybridep-num-sms 32 \
+--fp8-recipe mxfp8 \
+--fp8-format e4m3 \
+--fp8-param-gather \
+--reuse-grad-buf-for-mxfp8-param-ag \
+--use-precision-aware-optimizer \
+--main-grads-dtype fp32 \
+--main-params-dtype fp32 \
+--exp-avg-dtype bf16 \
+--exp-avg-sq-dtype bf16 \
+```
+
+### Explanation of arguments
+
+The following arguments indicate key optimizations.
+
+- Pipeline parallel layout
+
+```bash
+--pipeline-model-parallel-layout "Et|(tt|)*30L"
+```
+
+`E` stands for embedding, `t` for transformer layer, `L` for Loss. So it's interpreted as a total of 32 stages, where the first stage is Embedding + 1 transformer layer, the last stage is Loss, and the middle 30 stages are 2 transformer layers.
+
+- Fine-grained recompute
+
+```bash
+--recompute-granularity selective \
+--recompute-modules moe_act mlp \
+```
+
+- Partial CUDA Graphs
+
+```bash
+--cuda-graph-impl transformer_engine \
+--cuda-graph-scope attn moe_router moe_preprocess \
+--te-rng-tracker \
+```
+
+- Force load balancing for performance benchmark
+
+```bash
+--moe-router-force-load-balancing \
+```
+
+- HybridEP
+
+```bash
+--moe-token-dispatcher-type flex \
+--moe-flex-dispatcher-backend hybridep \
+--moe-hybridep-num-sms 32 \
+```
+
+- MXFP8 recipe
+
+```bash
+--fp8-recipe mxfp8 \
+--fp8-format e4m3 \
+--fp8-param-gather \
+--reuse-grad-buf-for-mxfp8-param-ag \
+```
+
+- BF16 optimizer states
+
+```bash
+--use-precision-aware-optimizer \
+--main-grads-dtype fp32 \
+--main-params-dtype fp32 \
+--exp-avg-dtype bf16 \
+--exp-avg-sq-dtype bf16 \
+```
+
+- Kernel fusions
+
+```bash
+--cross-entropy-loss-fusion \
+--cross-entropy-fusion-impl te \
+--moe-permute-fusion \
+--moe-router-fusion \
+```
+
+- Manual GC to make ranks better synchronized
+
+```bash
+--manual-gc \
+--manual-gc-interval 10 \
+```
diff --git a/docs/discussions/deepseek-v3-gb200-optimization/images/image1.png b/docs/discussions/deepseek-v3-gb200-optimization/images/image1.png
new file mode 100644
index 00000000000..6e4dad685c4
Binary files /dev/null and b/docs/discussions/deepseek-v3-gb200-optimization/images/image1.png differ
diff --git a/docs/discussions/deepseek-v3-gb200-optimization/images/image2.png b/docs/discussions/deepseek-v3-gb200-optimization/images/image2.png
new file mode 100644
index 00000000000..920e3c57f94
Binary files /dev/null and b/docs/discussions/deepseek-v3-gb200-optimization/images/image2.png differ
diff --git a/docs/discussions/deepseek-v3-gb200-optimization/images/image3.png b/docs/discussions/deepseek-v3-gb200-optimization/images/image3.png
new file mode 100644
index 00000000000..f606dbfb744
Binary files /dev/null and b/docs/discussions/deepseek-v3-gb200-optimization/images/image3.png differ
diff --git a/docs/discussions/deepseek-v3-gb200-optimization/images/image4.png b/docs/discussions/deepseek-v3-gb200-optimization/images/image4.png
new file mode 100644
index 00000000000..04239401edd
Binary files /dev/null and b/docs/discussions/deepseek-v3-gb200-optimization/images/image4.png differ
diff --git a/docs/discussions/deepseek-v3-gb200-optimization/images/image5.png b/docs/discussions/deepseek-v3-gb200-optimization/images/image5.png
new file mode 100644
index 00000000000..0128fc7ae45
Binary files /dev/null and b/docs/discussions/deepseek-v3-gb200-optimization/images/image5.png differ
diff --git a/docs/discussions/deepseek-v3-gb200-optimization/images/image6.png b/docs/discussions/deepseek-v3-gb200-optimization/images/image6.png
new file mode 100644
index 00000000000..cb2ed2eb9ad
Binary files /dev/null and b/docs/discussions/deepseek-v3-gb200-optimization/images/image6.png differ
diff --git a/docs/discussions/deepseek-v3-gb200-optimization/images/image7.png b/docs/discussions/deepseek-v3-gb200-optimization/images/image7.png
new file mode 100644
index 00000000000..325d0fd4f52
Binary files /dev/null and b/docs/discussions/deepseek-v3-gb200-optimization/images/image7.png differ
diff --git a/docs/discussions/megatron-fsdp-user-guide/example-scripts/sbatch_checkpoint_convert.sh b/docs/discussions/megatron-fsdp-user-guide/example-scripts/sbatch_checkpoint_convert.sh
new file mode 100644
index 00000000000..9f302c93f8f
--- /dev/null
+++ b/docs/discussions/megatron-fsdp-user-guide/example-scripts/sbatch_checkpoint_convert.sh
@@ -0,0 +1,50 @@
+#!/bin/bash
+
+# Configuration: Set these paths before running the script
+MEGATRON_PATH=${MEGATRON_PATH:-"your_own_megatron_path"} # Path to Megatron-LM repository
+CONTAINER_IMAGE=${CONTAINER_IMAGE:-"your_own_container_image"} # Path to .sqsh or docker image url
+OUTPUT_PATH=${OUTPUT_PATH:-"your_own_output_path"} # Path for SLURM logs
+
+# Checkpoint conversion command
+# Note: Update the checkpoint paths in the command below
+RUN_CMD="
+cd ${MEGATRON_PATH};
+git rev-parse HEAD;
+export PYTHONPATH=${MEGATRON_PATH}:${PYTHONPATH};
+python3 tools/checkpoint/checkpoint_inspector.py \
+ convert-torch-dist-to-fsdp-dtensor --swiglu \
+ your_own_path_to_input_torch_dist_checkpoint \
+ your_own_path_to_output_fsdp_dtensor_checkpoint \
+ --param-to-param-group-map-json your_own_path_to_param_to_param_group_map.json"
+
+# SLURM settings
+SLURM_LOGS="${OUTPUT_PATH}/slurm_logs"
+mkdir -p ${SLURM_LOGS} || {
+ echo "Error: Failed to create SLURM logs directory ${SLURM_LOGS}"
+ exit 1
+}
+
+# Submit SLURM job
+# Note: Update SBATCH parameters below according to your cluster configuration
+set +e
+sbatch <&1 | tee ${SLURM_LOGS}/\${SLURM_JOB_ID}.log
+
+EOF
+set -e
diff --git a/docs/discussions/megatron-fsdp-user-guide/example-scripts/sbatch_mfsdp_deepseek_v3.sh b/docs/discussions/megatron-fsdp-user-guide/example-scripts/sbatch_mfsdp_deepseek_v3.sh
new file mode 100644
index 00000000000..7b93d25d943
--- /dev/null
+++ b/docs/discussions/megatron-fsdp-user-guide/example-scripts/sbatch_mfsdp_deepseek_v3.sh
@@ -0,0 +1,223 @@
+#!/bin/bash
+
+export NCCL_IB_SL=1
+export NCCL_IB_TIMEOUT=19
+export NVTE_FWD_LAYERNORM_SM_MARGIN=16
+export NVTE_BWD_LAYERNORM_SM_MARGIN=16
+export NCCL_P2P_NET_CHUNKSIZE=2097152
+export TORCH_NCCL_AVOID_RECORD_STREAMS=1
+export PYTHONWARNINGS=ignore
+export TRITON_CACHE_DIR=/tmp/triton_cache_$SLURM_NODEID
+
+# Configuration: Set these variables before running the script
+MEGATRON_PATH=${MEGATRON_PATH:-"your_own_megatron_path"} # Path to Megatron-LM repository
+CONTAINER_IMAGE=${CONTAINER_IMAGE:-"your_own_container_image"} # Path to .sqsh or docker image url
+OUTPUT_PATH=${OUTPUT_PATH:-"your_own_output_path"} # Path for output logs and checkpoints
+DATA_PATH=${DATA_PATH:-"your_own_data_path"}
+USE_MEGATRON_FSDP=${USE_MEGATRON_FSDP:-1}
+SHARDING_STRATEGY=${SHARDING_STRATEGY:-"optim_grads_params"}
+PROFILE=${PROFILE:-0}
+WANDB=${WANDB:-1}
+
+TP=${TP:-1}
+EP=${EP:-8}
+MBS=${MBS:-4}
+GBS=${GBS:-2048}
+COMMENT=${COMMENT:-"hybridep-selective-recompute"}
+
+PRETRAIN_ARGS=(
+ --distributed-timeout-minutes 60
+ --tensor-model-parallel-size ${TP}
+ --expert-model-parallel-size ${EP}
+ --expert-tensor-parallel-size 1
+ --context-parallel-size 1
+ --use-distributed-optimizer
+ --overlap-grad-reduce
+ --overlap-param-gather
+ --use-mcore-models
+ --sequence-parallel
+ --use-flash-attn
+ --disable-bias-linear
+ --micro-batch-size ${MBS}
+ --global-batch-size ${GBS}
+ --train-samples 585937500
+ --exit-duration-in-mins 220
+ --no-check-for-nan-in-loss-and-grad
+ --manual-gc
+ --manual-gc-interval 10
+ --recompute-granularity selective
+ --recompute-modules mlp moe mla_up_proj layernorm
+ --transformer-impl transformer_engine
+ --seq-length 4096
+ --data-cache-path ${OUTPUT_PATH}/cache
+ --tokenizer-type HuggingFaceTokenizer
+ --tokenizer-model deepseek-ai/DeepSeek-V3
+ --data-path ${DATA_PATH}
+ --split 99,1,0
+ --no-mmap-bin-files
+ --no-create-attention-mask-in-dataloader
+ --num-workers 6
+ --num-layers 61
+ --hidden-size 7168
+ --ffn-hidden-size 18432
+ --num-attention-heads 128
+ --kv-channels 128
+ --max-position-embeddings 4096
+ --position-embedding-type rope
+ --rotary-base 10000
+ --make-vocab-size-divisible-by 3232
+ --normalization RMSNorm
+ --norm-epsilon 1e-6
+ --swiglu
+ --untie-embeddings-and-output-weights
+ --multi-latent-attention
+ --attention-dropout 0.0
+ --hidden-dropout 0.0
+ --clip-grad 1.0
+ --weight-decay 0.1
+ --qk-layernorm
+ --lr-decay-samples 584765624
+ --lr-warmup-samples 1536000
+ --lr-warmup-init 3.9e-7
+ --lr 3.9e-6
+ --min-lr 3.9e-7
+ --lr-decay-style cosine
+ --adam-beta1 0.9
+ --adam-beta2 0.95
+ --num-experts 256
+ --moe-layer-freq [0]*3+[1]*58
+ --moe-ffn-hidden-size 2048
+ --moe-shared-expert-intermediate-size 2048
+ --moe-router-load-balancing-type seq_aux_loss
+ --moe-router-topk 8
+ --moe-token-dispatcher-type flex
+ --moe-flex-dispatcher-backend hybridep
+ --moe-router-pre-softmax
+ --moe-grouped-gemm
+ --moe-aux-loss-coeff 1e-4
+ --moe-router-group-topk 4
+ --moe-router-num-groups 8
+ --moe-router-topk-scaling-factor 2.5
+ --moe-router-score-function sigmoid
+ --moe-router-enable-expert-bias
+ --moe-router-bias-update-rate 1e-3
+ --moe-router-dtype fp32
+ --moe-permute-fusion
+ --moe-router-force-load-balancing
+ --q-lora-rank 1536
+ --kv-lora-rank 512
+ --qk-head-dim 128
+ --qk-pos-emb-head-dim 64
+ --v-head-dim 128
+ --rotary-scaling-factor 40
+ --mscale 1.0
+ --mscale-all-dim 1.0
+ --mtp-num-layers 1
+ --mtp-loss-scaling-factor 0.1
+ --eval-iters 32
+ --eval-interval 100
+ --auto-detect-ckpt-format
+ --load ${OUTPUT_PATH}/checkpoints
+ --save ${OUTPUT_PATH}/checkpoints
+ --save-interval 100
+ --dist-ckpt-strictness log_all
+ --init-method-std 0.02
+ --log-timers-to-tensorboard
+ --log-memory-to-tensorboard
+ --log-num-zeros-in-grad
+ --log-params-norm
+ --log-validation-ppl-to-tensorboard
+ --log-throughput
+ --log-interval 1
+ --logging-level 40
+ --tensorboard-dir ${OUTPUT_PATH}/tensorboard
+ --bf16
+ --enable-experimental
+)
+
+if [ "${USE_MEGATRON_FSDP}" = 1 ]; then
+ unset CUDA_DEVICE_MAX_CONNECTIONS
+ PRETRAIN_ARGS=(
+ "${PRETRAIN_ARGS[@]}"
+ --use-megatron-fsdp
+ --data-parallel-sharding-strategy ${SHARDING_STRATEGY}
+ --no-gradient-accumulation-fusion
+ --use-distributed-optimizer
+ --calculate-per-token-loss
+ --init-model-with-meta-device
+ --ckpt-format fsdp_dtensor
+ --grad-reduce-in-bf16
+ --fsdp-double-buffer
+ --use-nccl-ub
+ )
+fi
+
+# Profiling command
+if [ "${PROFILE}" = 1 ]; then
+ PROFILE_CMD="nsys profile --sample=none --cpuctxsw=none --trace=cuda,nvtx,cublas,cudnn \
+ --capture-range=cudaProfilerApi \
+ --capture-range-end=stop \
+ --cuda-graph-trace=node \
+ --cuda-memory-usage=true \
+ -f true -x true \
+ -o ${OUTPUT_PATH}/nsys/Megatron-FSDP-Deepseek-V3-TP${TP}EP${EP}-MBS${MBS}GBS${GBS}-${COMMENT}"
+ PRETRAIN_ARGS=(
+ "${PRETRAIN_ARGS[@]}"
+ --profile
+ --profile-step-start 10
+ --profile-step-end 12
+ --profile-ranks 0
+ )
+ echo "PROFILE_CMD="
+ echo $PROFILE_CMD
+else
+ PROFILE_CMD=""
+fi
+
+if [ "${WANDB}" = 1 ]; then
+ export WANDB_API_KEY=${WANDB_API_KEY:-"your_own_wandb_api_key"}
+ PRETRAIN_ARGS=(
+ "${PRETRAIN_ARGS[@]}"
+ --wandb-project your_own_wandb_project
+ --wandb-exp-name DeepSeek-V3-TP${TP}EP${EP}-MBS${MBS}GBS${GBS}-${COMMENT}
+ )
+fi
+
+TRAINING_CMD="
+cd ${MEGATRON_PATH};
+git rev-parse HEAD;
+export PYTHONPATH=${MEGATRON_PATH}:${PYTHONPATH};
+${PROFILE_CMD} python ${MEGATRON_PATH}/pretrain_gpt.py ${PRETRAIN_ARGS[@]}"
+
+# SLURM settings
+SLURM_LOGS="${OUTPUT_PATH}/slurm_logs"
+mkdir -p ${SLURM_LOGS} || {
+ echo "Error: Failed to create SLURM logs directory ${SLURM_LOGS}"
+ exit 1
+}
+
+# Submit SLURM job
+# Note: Update SBATCH parameters below according to your cluster configuration
+set +e
+sbatch <&1 | tee ${SLURM_LOGS}/\${SLURM_JOB_ID}.log
+
+EOF
+set -e
diff --git a/docs/discussions/megatron-fsdp-user-guide/megatron-fsdp-user-guide.md b/docs/discussions/megatron-fsdp-user-guide/megatron-fsdp-user-guide.md
new file mode 100644
index 00000000000..c2354ad07f0
--- /dev/null
+++ b/docs/discussions/megatron-fsdp-user-guide/megatron-fsdp-user-guide.md
@@ -0,0 +1,116 @@
+# Megatron-FSDP User Guide
+
+## Table of Contents
+
+- [Megatron-FSDP Quick Start](#megatron-fsdp-quick-start)
+- [Checkpoint Conversion from 3D-Parallel to Megatron-FSDP](#checkpoint-conversion-from-3d-parallel-to-megatron-fsdp)
+
+## Megatron-FSDP Quick Start
+
+We recommend using the latest [NVIDIA NeMo Framework Container](https://catalog.ngc.nvidia.com/orgs/nvidia/containers/nemo/tags), which provides a tested software stack and optimized performance.
+
+For your reference, we provide an example launch script for DeepSeek-V3: [`sbatch_mfsdp_deepseek_v3.sh`](./example-scripts/sbatch_mfsdp_deepseek_v3.sh).
+
+### Required Configurations
+
+To enable Megatron-FSDP, add the following required flags to your training script:
+
+```bash
+--use-megatron-fsdp
+--data-parallel-sharding-strategy optim_grads_params
+--no-gradient-accumulation-fusion
+--use-distributed-optimizer
+--ckpt-format fsdp_dtensor
+```
+
+### Recommended Configurations
+
+We also recommend adding the following configurations to further improve performance:
+
+```bash
+unset CUDA_DEVICE_MAX_CONNECTIONS
+```
+```bash
+--calculate-per-token-loss
+--init-model-with-meta-device
+--grad-reduce-in-bf16
+--fsdp-double-buffer
+--use-nccl-ub
+```
+
+💡 **Detailed explanations of these configurations are provided below.**
+
+#### 1. Disable `CUDA_DEVICE_MAX_CONNECTIONS`
+
+To ensure full parallelization of FSDP communication and computation, disable the CUDA_DEVICE_MAX_CONNECTIONS environment variable. This step avoids potential bubbles in the CUDA stream. (But it may slow down TP and CP to some extent.)
+
+#### 2. Add `--calculate-per-token-loss`
+
+For gradients sharding mode optimization, include the `--calculate-per-token-loss` flag in your training script. This improves performance by reducing the frequency of gradient scaling, which is also a sizable drain on SM resources.
+
+#### 3. Add `--init-model-with-meta-device`
+
+Allows model initialization using meta device, followed by layer-by-layer initialization of distributed model weight buffers via the `Module.reset_parameters` API, facilitating the initialization of extremely large models.
+
+#### 4. Add `--grad-reduce-in-bf16`
+
+Enables gradient reduction in BF16 precision instead of FP32, reducing communication volume and accelerating the backward pass.
+
+#### 5. Add `--fsdp-double-buffer`
+
+Uses persistently allocated double buffers for temporarily-defined memory needed in `MegatronFSDP` communications. While having persistent double buffers may increase peak VRAM utilization, it is necessary to register NCCL user buffers (`nccl_ub=True`) for `MegatronFSDP`. Currently, this is supported only for simple repetitive model structures such as GPT.
+
+- **Only effective when using Megatron-LM.**
+- Defaults to `False`. Automatically overridden to `True` when `nccl_ub` is enabled.
+
+#### 6. Add `--use-nccl-ub`
+
+Allocates and [registers NCCL user buffers](https://docs.nvidia.com/deeplearning/nccl/user-guide/docs/usage/bufferreg.html#) for param and grad buffers. This option enables an SM-efficient NCCL algorithm that could improve the performance of overlapped computations. This flag will be much more effective when used together with [SHARP](https://docs.nvidia.com/networking/display/sharpv3130) if the FSDP communication includes both NVL and IB domains. Enabling this option will cause additional memory overhead due to the requirement to enable the `fsdp_double_buffer` option.
+
+- **Only effective when using Megatron-LM.**
+- Defaults to `False`.
+- By default we try to use NCCL window (symmetric) registration if it is available. If not it falls back to conventional local registration.
+- **Incompatible with PyTorch's segmentable allocator:** Do not set `PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True` when using `--use-nccl-ub`, as this will cause a runtime error due to compatibility issues with the `torch.cuda.MemPool` API.
+
+## Checkpoint Conversion from 3D-Parallel to Megatron-FSDP
+
+Megatron-FSDP introduces `fsdp_dtensor`, a DTensor-based distributed checkpoint format that serves as its standard. To help you smoothly transition from 3D-Parallel to Megatron-FSDP, we provide a script for converting checkpoints from the `torch_dist` format to the `fsdp_dtensor` format. Using DeepSeek-V3 as an example, the detailed conversion process is described below.
+
+### Step 1: Generate 3D-Parallel Checkpoint with `param_to_param_group_map`
+
+Run your 3D-parallel + EP training script to generate a `torch_dist` checkpoint along with a directory containing `param_to_param_group_map` files. Add the following flag to your training script:
+
+```bash
+--dump-param-to-param-group-map /path/to/param_to_param_group_map
+```
+
+If you already have a `torch_dist` checkpoint, simply specify the `--dump-param-to-param-group-map /path/to/param_to_param_group_map` flag and run a very short experiment-this will create the `param_to_param_group_map` you need without full pretraining.
+
+### Step 2: Export `param_to_param_group_map` to a JSON File
+
+Convert the `param_to_param_group_map` into a JSON file for easier processing by running:
+
+```bash
+python tools/checkpoint/checkpoint_inspector.py print-torch-dcp-in-json /path/to/param_to_param_group_map
+```
+
+This will create a `param_to_param_group_map.json` file in the `/path/to/param_to_param_group_map` directory.
+
+### Step 3: Convert Checkpoint from `torch_dist` to `fsdp_dtensor`
+
+Convert your `torch_dist` checkpoint to the `fsdp_dtensor` format using the parameter to `param_to_param_group_map` JSON file:
+
+```bash
+torchrun --nproc_per_node=8 --nnodes=1 \
+ tools/checkpoint/checkpoint_inspector.py \
+ convert-torch-dist-to-fsdp-dtensor --swiglu \
+ /path/to/input_torch_dist_checkpoint \
+ /path/to/output_fsdp_dtensor_checkpoint \
+ --param-to-param-group-map-json /path/to/param_to_param_group_map.json
+```
+
+**Note:** For multi-node conversion tasks, please refer to the example script: [`sbatch_checkpoint_convert.sh`](./example-scripts/sbatch_checkpoint_convert.sh).
+
+### Step 4: Launch Megatron-FSDP Training
+
+Start your Megatron-FSDP training job using the converted `fsdp_dtensor` checkpoint.
\ No newline at end of file
diff --git a/docs/images/fine_grained_activation_offloading/offloading_and_recomputing.png b/docs/images/fine_grained_activation_offloading/offloading_and_recomputing.png
new file mode 100644
index 00000000000..6c8afa78bb1
Binary files /dev/null and b/docs/images/fine_grained_activation_offloading/offloading_and_recomputing.png differ
diff --git a/docs/source/api-guide/router_replay.md b/docs/source/api-guide/router_replay.md
new file mode 100644
index 00000000000..334a29c78a6
--- /dev/null
+++ b/docs/source/api-guide/router_replay.md
@@ -0,0 +1,176 @@
+# Design Document: MoE Router Replay Feature
+
+### 1. Overview
+
+This document provides a detailed description of the "Router Replay" feature implemented within the Megatron-LM Core for Mixture-of-Experts (MoE) models.
+
+This feature is designed to enhance determinism and analyzability in MoE model training and inference. It enables the model to load routing decisions from a predefined file and enforce their use during the forward pass, thereby bypassing the real-time routing computation.
+
+### 2. Motivation
+
+* **Determinism & Reproducibility**: In distributed training, MoE routing decisions can exhibit minor variations due to factors like floating-point precision. By replaying a fixed routing table, the MoE computation path is guaranteed to be identical across runs, which facilitates debugging and reproducing experimental results.
+* **Performance Profiling**: The router's own computation (e.g., logits calculation, top-k selection) incurs overhead. In replay mode, this part of the computation can be completely skipped, allowing for more precise isolation and profiling of performance bottlenecks within the Expert Layers themselves.
+* **Debugging Aid**: When issues arise in the model, fixing the routing decisions helps to isolate variables, making it easier to determine whether the problem lies with the routing mechanism or the expert computations.
+
+### 3. Design and Architecture
+
+The design follows the principles of being non-intrusive and on-demand, with the core idea of activating the replay logic only when explicitly requested by the user.
+
+* **Core Components**:
+ * `RouterReplay` (located in `megatron/core/transformer/moe/router_replay.py`): A utility class for replaying MoE routing decisions. When enabled via the `enable_routing_replay` flag, a separate instance of `RouterReplay` is created for each MoE layer's router. Each instance is responsible for loading routing data and providing the deterministic routing decisions for its corresponding layer during the forward pass.
+ * `enable_routing_replay` (located in `megatron/core/transformer/transformer_config.py`): A boolean global configuration flag that serves as the sole entry point for enabling this feature.
+
+* **Workflow**:
+ The feature supports different modes, such as recording and replaying, controlled by a `RouterReplayAction`.
+
+ 1. **Enabling the Feature**: The user sets `enable_routing_replay` to `True` in the model configuration.
+ 2. **Initialization**: When `enable_routing_replay` is true, each `TopKRouter` creates its own `RouterReplay` instance.
+ 3. **Mode Configuration**: The user must programmatically set the desired router replay action (e.g., `record`, `forward_replay`, `backward_replay`) on the `RouterReplay` instances.
+ 4. **Execution Flow (within a mini-batch)**:
+ * **Forward Pass**:
+ * For each micro-batch, the `topk_routing_with_score_function` checks the `router_replay_action`.
+ * **In `record` mode**: The dynamically computed `top-k` expert indices are captured and stored.
+ * **In `forward_replay` mode**: The function retrieves pre-loaded expert indices from `target_topk_idx`. These indices are used for the forward computation and are also appended to the `replay_backward_list` to prepare for the backward pass.
+ * **Backward Pass**:
+ * For each micro-batch (processed in reverse order in pipeline parallelism), the `router_replay_action` is checked again.
+ * **In `backward_replay` mode**: The function retrieves the expert indices for the corresponding micro-batch by popping them from the `replay_backward_list`. This mode is intended for training recomputation (e.g., activation checkpointing and pipeline recompute) so the same routing decisions are used during recompute/backward as in forward, ensuring determinism and correctness.
+
+### 4. Implementation Details
+
+The implementation cleanly separates the replay logic from the router's core computation.
+
+* **`megatron/core/transformer/transformer_config.py`**:
+ * Adds the configuration option `enable_routing_replay: bool = False`.
+
+* **`megatron/core/transformer/moe/moe_utils.py`**:
+ * Introduces the `RouterReplay` class to manage the state for recording and replaying routing decisions for a single MoE layer.
+ * `target_topk_idx`: An attribute holding the expert indices for the current micro-batch during forward replay mode.
+ * `recorded_topk_idx`: An attribute for storing the computed expert indices when in record mode.
+ * `replay_backward_list`: A list that accumulates the `top-k` indices used during the forward passes of a mini-batch. This list is consumed in FIFO order during the backward pass to ensure correctness under pipeline parallelism.
+ * `set_target_indices()`: A method to load the replay indices into `target_topk_idx` for the forward pass.
+ * `record_indices()`: A method to save the computed indices.
+ * The `topk_routing_with_score_function` is modified to contain the core logic. It checks the `router_replay_action` on the `router_replay` instance and accordingly performs one of the following actions: computes and records indices, replays indices from `target_topk_idx` (for forward), replays indices from `replay_backward_list` (for backward), or falls through to the default dynamic routing.
+
+#### Training recompute usage
+- During forward replay, `set_target_indices()` prepares `replay_backward_list` so each micro-batch’s indices are available for recomputation.
+- During recompute/backward, set action to `REPLAY_BACKWARD` so indices are consumed in FIFO order to mirror the forward sequence.
+
+### 5. Usage Guide
+
+1. **Enable & Instantiate**
+ - Create one `RouterReplay` instance per MoE router layer when building the model.
+ - Optionally use the global helpers to set/clear actions across all layers.
+2. **Record Routing Decisions**
+ - Set action: `RouterReplay.set_global_router_replay_action(RouterReplayAction.RECORD)`.
+ - Run the model; retrieve per-layer indices via `RouterReplay.get_recorded_data()` and persist.
+3. **Forward Replay**
+ - Load indices and distribute: `RouterReplay.set_replay_data(list_of_tensors)`.
+ - Set action: `RouterReplay.set_global_router_replay_action(RouterReplayAction.REPLAY_FORWARD)`.
+ - Run the model; dynamic top‑k is bypassed and target indices are used.
+4. **Backward Replay**
+ - For training recomputation (activation checkpointing or pipeline recompute), set action: `REPLAY_BACKWARD` during recomputation.
+ - Per micro‑batch indices are consumed from `replay_backward_list` in FIFO order.
+5. **Cleanup**
+ - Use `RouterReplay.clear_global_indices()`, `RouterReplay.clear_global_router_replay_action()`, and `RouterReplay.clear_global_router_replay_instances()` to restore default behavior and prevent memory leaks.
+
+#### Quick usage with `topk_routing_with_score_function`
+
+```python
+import torch
+from megatron.core.transformer.moe.router_replay import RouterReplay, RouterReplayAction
+from megatron.core.transformer.moe.moe_utils import topk_routing_with_score_function
+
+rr = RouterReplay()
+
+# Record
+RouterReplay.set_global_router_replay_action(RouterReplayAction.RECORD)
+logits = torch.randn(8, 16)
+probs_rec, routing_map_rec = topk_routing_with_score_function(
+ logits=logits, topk=2, use_pre_softmax=False, score_function="softmax", router_replay=rr,
+)
+recorded = rr.get_recorded_indices()
+torch.save(recorded, "/tmp/replay.pt")
+
+# Forward replay
+rr.clear_router_replay_action()
+rr.set_router_replay_action(RouterReplayAction.REPLAY_FORWARD)
+target = torch.load("/tmp/replay.pt")
+rr.set_target_indices(target)
+probs_rep, routing_map_rep = topk_routing_with_score_function(
+ logits=logits, topk=2, use_pre_softmax=False, score_function="softmax", router_replay=rr,
+)
+
+RouterReplay.clear_global_router_replay_action()
+RouterReplay.clear_global_indices()
+RouterReplay.clear_global_router_replay_instances()
+```
+
+### 6. Minimal Demo
+
+Here is a minimal code example showing how to use RouterReplay for recording and replaying:
+
+```python
+import torch
+import torch.distributed as dist
+from megatron.core.transformer.transformer_config import TransformerConfig
+from megatron.core.transformer.moe.router import TopKRouter
+from megatron.core.transformer.moe.router_replay import RouterReplay, RouterReplayAction
+
+
+# Initialize distributed training
+if not dist.is_initialized():
+ dist.init_process_group(backend="nccl")
+
+# Create a transformer config with RouterReplay enabled
+config = TransformerConfig(
+ num_experts=8,
+ expert_model_parallel_size=1,
+ num_top_k=2,
+ enable_routing_replay=True
+)
+
+# Create a TopKRouter instance
+router = TopKRouter(config)
+
+# Generate sample input (batch_size, sequence_length, hidden_size)
+logits = torch.randn(16, 32, 8).to(torch.cuda.current_device())
+
+# -----------------
+# 1. Recording Mode
+# -----------------
+print("=== Recording Mode ===")
+# Set global router replay action to RECORD
+RouterReplay.set_global_router_replay_action(RouterReplayAction.RECORD)
+
+# Perform routing
+routing_output = router.forward(logits)
+print(f"Recorded top-k indices shape: {routing_output.top_k_idx.shape}")
+
+# -----------------
+# 2. Forward Replay Mode
+# -----------------
+print("\n=== Forward Replay Mode ===")
+# Save recorded indices to a file
+torch.save(routing_output.top_k_idx, "/tmp/replay.pt")
+
+# Load indices from file and set as target for replay
+replay_indices = torch.load("/tmp/replay.pt")
+for router_instance in RouterReplay.global_router_replay_instances:
+ router_instance.target_topk_idx = replay_indices
+
+# Set global router replay action to REPLAY_FORWARD
+RouterReplay.set_global_router_replay_action(RouterReplayAction.REPLAY_FORWARD)
+
+# Perform routing again - this will use the replayed indices
+replay_routing_output = router.forward(logits)
+print(f"Replayed top-k indices shape: {replay_routing_output.top_k_idx.shape}")
+print(f"Are indices the same? {torch.equal(routing_output.top_k_idx, replay_routing_output.top_k_idx)}")
+
+
+# Clean up
+RouterReplay.clear_global_router_replay_action()
+RouterReplay.clear_global_indices()
+RouterReplay.clear_global_router_replay_instances()
+if dist.is_initialized():
+ dist.destroy_process_group()
+```
diff --git a/examples/inference/gpt/gpt_static_inference.py b/examples/inference/gpt/gpt_static_inference.py
index 2dcae5549a6..32a49191b19 100644
--- a/examples/inference/gpt/gpt_static_inference.py
+++ b/examples/inference/gpt/gpt_static_inference.py
@@ -104,7 +104,7 @@ def get_inference_engine(args: Namespace, model: MegatronModule) -> StaticInfere
text_generation_controller = TextGenerationController(
inference_wrapped_model=inference_wrapped_model, tokenizer=tokenizer
)
- return StaticInferenceEngine(text_generation_controller=text_generation_controller, legacy=args.use_legacy_static_engine)
+ return StaticInferenceEngine(text_generation_controller=text_generation_controller, legacy=True)
async def generate(
diff --git a/examples/rl/environments/countdown/countdown.py b/examples/rl/environments/countdown/countdown.py
index c5ad57bb72d..acfabc46681 100644
--- a/examples/rl/environments/countdown/countdown.py
+++ b/examples/rl/environments/countdown/countdown.py
@@ -1,3 +1,4 @@
+# Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# This file is adapted from code in https://github.com/Jiayi-Pan/TinyZero
import re
diff --git a/gpt_builders.py b/gpt_builders.py
index 4b86e30e597..0be64edaab6 100644
--- a/gpt_builders.py
+++ b/gpt_builders.py
@@ -7,6 +7,11 @@
get_gpt_layer_with_transformer_engine_spec,
get_gpt_layer_with_inference_spec,
get_gpt_mtp_block_spec,
+ get_gpt_decoder_layer_specs,
+)
+from megatron.core.models.gpt.experimental_attention_variant_module_specs import (
+ get_transformer_block_with_experimental_attention_variant_spec,
+ get_transformer_layer_with_experimental_attention_variant_spec,
)
from megatron.core.models.gpt.heterogeneous.heterogeneous_layer_specs import (
get_gpt_heterogeneous_layer_spec,
@@ -42,7 +47,13 @@ def gpt_builder(args, pre_process, post_process, vp_stage=None, config=None, pg_
else:
use_te = args.transformer_impl == "transformer_engine"
- if args.num_experts:
+ if args.experimental_attention_variant is not None:
+ transformer_layer_spec = (
+ get_transformer_block_with_experimental_attention_variant_spec(
+ config=config, vp_stage=vp_stage
+ )
+ )
+ elif args.num_experts:
assert not (config.transformer_impl == "inference_optimized")
# Define the decoder block spec
transformer_layer_spec = get_gpt_decoder_block_spec(
@@ -61,18 +72,29 @@ def gpt_builder(args, pre_process, post_process, vp_stage=None, config=None, pg_
mtp_block_spec = None
if args.mtp_num_layers is not None:
assert not (config.transformer_impl == "inference_optimized")
- if (
- hasattr(transformer_layer_spec, 'layer_specs')
- and len(transformer_layer_spec.layer_specs) == 0
- ):
- # Get the decoder layer spec explicitly if no decoder layer in the last stage,
- # Only happens with block spec (TransformerBlockSubmodules) when using MoE.
- transformer_layer_spec_for_mtp = _get_transformer_layer_spec(use_te, config)
+ # Get GPT decoder layer specs for the model.
+ if args.spec is not None:
+ mtp_transformer_layer_spec = import_module(args.spec)
else:
- transformer_layer_spec_for_mtp = transformer_layer_spec
+ # Define the decoder block spec
+ if args.experimental_attention_variant is not None:
+ decoder_layer_specs = (
+ get_transformer_layer_with_experimental_attention_variant_spec(
+ config=config
+ )
+ )
+ else:
+ decoder_layer_specs = get_gpt_decoder_layer_specs(
+ config,
+ use_transformer_engine=use_te,
+ normalization=args.normalization,
+ qk_l2_norm=args.qk_l2_norm,
+ )
+ mtp_transformer_layer_spec = decoder_layer_specs[-1]
+ # Use spec of the last layer in decoder block as spec of the transformer layer in MTP
mtp_block_spec = get_gpt_mtp_block_spec(
config,
- transformer_layer_spec_for_mtp,
+ mtp_transformer_layer_spec,
use_transformer_engine=use_te,
vp_stage=vp_stage,
)
@@ -101,12 +123,12 @@ def gpt_builder(args, pre_process, post_process, vp_stage=None, config=None, pg_
def _get_transformer_layer_spec(use_te, config):
"""Get transformer layer specification based on configuration.
-
+
Args:
use_te (bool): Whether to use Transformer Engine
args: Training arguments
config: Model configuration
-
+
Returns:
transformer_layer_spec: The transformer layer specification
"""
@@ -117,11 +139,13 @@ def _get_transformer_layer_spec(use_te, config):
args.moe_grouped_gemm,
args.qk_layernorm,
args.multi_latent_attention,
+ args.experimental_attention_variant,
moe_use_legacy_grouped_gemm=args.moe_use_legacy_grouped_gemm,
qk_l2_norm=args.qk_l2_norm,
use_kitchen=config.use_kitchen,
use_kitchen_attention=config.use_kitchen_attention,
kitchen_attention_backend=config.kitchen_attention_backend,
+ fallback_to_eager_attn=config.fallback_to_eager_attn,
)
elif config.transformer_impl == "inference_optimized":
return get_gpt_layer_with_inference_spec(
@@ -135,6 +159,7 @@ def _get_transformer_layer_spec(use_te, config):
args.moe_grouped_gemm,
args.qk_layernorm,
args.multi_latent_attention,
+ args.experimental_attention_variant,
moe_use_legacy_grouped_gemm=args.moe_use_legacy_grouped_gemm,
normalization=args.normalization,
use_kitchen=config.use_kitchen,
diff --git a/hello_world b/hello_world
new file mode 100644
index 00000000000..e69de29bb2d
diff --git a/megatron/core/datasets/Makefile b/megatron/core/datasets/Makefile
index e745f52399b..16f251bf903 100644
--- a/megatron/core/datasets/Makefile
+++ b/megatron/core/datasets/Makefile
@@ -1,4 +1,4 @@
-CXXFLAGS += -O3 -Wall -shared -std=c++11 -fPIC -fdiagnostics-color
+CXXFLAGS += -O3 -Wall -shared -std=c++17 -fPIC -fdiagnostics-color
CPPFLAGS += $(shell python3 -m pybind11 --includes)
LIBNAME = helpers_cpp
diff --git a/megatron/core/datasets/blended_megatron_dataset_builder.py b/megatron/core/datasets/blended_megatron_dataset_builder.py
index 952517bf5b4..5ad3f7389a2 100644
--- a/megatron/core/datasets/blended_megatron_dataset_builder.py
+++ b/megatron/core/datasets/blended_megatron_dataset_builder.py
@@ -1,4 +1,4 @@
-# Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved.
+# Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
import logging
import math
@@ -47,11 +47,13 @@ def __init__(
sizes: List[int],
is_built_on_rank: Callable,
config: BlendedMegatronDatasetConfig,
+ vp_stage: Optional[int] = None,
):
self.cls = cls
self.sizes = sizes
self.is_built_on_rank = is_built_on_rank
self.config = config
+ self.vp_stage = vp_stage
log_single_rank(
logger,
diff --git a/megatron/core/datasets/data_schedule.py b/megatron/core/datasets/data_schedule.py
new file mode 100644
index 00000000000..0f016473b6a
--- /dev/null
+++ b/megatron/core/datasets/data_schedule.py
@@ -0,0 +1,301 @@
+# Copyright (c) 2025 NVIDIA CORPORATION. All rights reserved.
+
+from typing import Any, List, Optional
+
+import torch
+
+from megatron.core import parallel_state
+from megatron.core.pipeline_parallel.hybrid_cp_schedule import BalancedCPScheduler
+from megatron.core.process_groups_config import ProcessGroupCollection
+
+
+class HybridCPDataLoaderWrapper:
+ """
+ A wrapper class that wraps around an existing data_iterator.
+ For every __next__ call,
+ 1. Each DP rank pulls a batch of packed samples.
+ 2. Extracts the sequence lengths of each sub-sample and all-gathers across the DP group.
+ 3. Schedules the sub-samples to the DPxCP ranks using the BalancedCPScheduler.
+ 4. Based on the schedule, reroutes the sub-samples to the correct rank using all-to-all.
+ 5. Returns the assigned sub-samples to this rank.
+
+ Args:
+ data_iterator: The original data_iterator to wrap around
+ config: The config object containing the max_seqlen_per_dp_cp_rank
+ dp_cp_group: Data parallel context parallel group.
+ """
+
+ def __init__(
+ self, data_iterator, config, pg_collection: Optional[ProcessGroupCollection] = None
+ ):
+ self.data_iterator = data_iterator
+ self.config = config
+ if pg_collection is None:
+ self.dp_cp_group = parallel_state.get_data_parallel_group(with_context_parallel=True)
+ self.dp_group = parallel_state.get_data_parallel_group()
+ self.tp_group = parallel_state.get_tensor_model_parallel_group()
+ else:
+ self.dp_cp_group = pg_collection.dp_cp
+ self.dp_group = pg_collection.dp
+ self.tp_group = pg_collection.tp
+ assert (
+ self.dp_cp_group is not None and self.dp_group is not None and self.tp_group is not None
+ ), "dp_cp_group, dp_group, tp_group must not be None when using hybrid context parallel"
+
+ self.cp_balancing_scheduler = BalancedCPScheduler(
+ max_seq_len_per_rank=self.config.max_seqlen_per_dp_cp_rank, dp_cp_group=self.dp_cp_group
+ )
+
+ self.total_hdp_gpus = self.dp_cp_group.size()
+
+ def __iter__(self):
+ """Return self as an iterator."""
+ return self
+
+ def get_global_seqlens(self, subsample_seqlens: torch.Tensor) -> List[int]:
+ """
+ Gathers the sequence lengths of all subsamples from all DP ranks.
+ Each DP rank loads the same number of microbatches but each microbatch
+ may have a different number of subsamples.
+
+ We find the number of subsamples each rank holds and then gather the
+ sequence lengths of all subsamples from all ranks.
+ """
+ # Collect the number of subsamples from all ranks
+ local_len = torch.tensor([subsample_seqlens.shape[0]], dtype=torch.int32).cuda()
+ dp_subsample_count = [torch.zeros_like(local_len) for _ in range(self.dp_group.size())]
+ torch.distributed.all_gather(dp_subsample_count, local_len, group=self.dp_group)
+
+ # Find the max number of subsamples across all ranks and pad subsample_seqlens to max length
+ dp_subsample_counts = torch.stack(dp_subsample_count, dim=0).cpu().view(-1)
+ max_sub_samples = int(dp_subsample_counts.max().item())
+
+ if local_len.item() < max_sub_samples:
+ subsample_seqlens_padded = torch.cat(
+ [
+ subsample_seqlens,
+ torch.zeros(max_sub_samples - local_len.item(), dtype=torch.int32).cuda(),
+ ],
+ dim=0,
+ )
+ else:
+ subsample_seqlens_padded = subsample_seqlens
+
+ # Gather the subsample_seqlens from all ranks
+ seqlens_gathered = [
+ torch.empty_like(subsample_seqlens_padded) for _ in range(self.dp_group.size())
+ ]
+ torch.distributed.all_gather(
+ seqlens_gathered, subsample_seqlens_padded, group=self.dp_group
+ )
+
+ # Trim each seqlens_gathered to the length of the correct sample
+ for dp_rank, seqlen in enumerate(seqlens_gathered):
+ seqlens_gathered[dp_rank] = seqlen[: dp_subsample_counts[dp_rank]]
+
+ seqlens_gathered = torch.cat(seqlens_gathered, dim=0)
+ seqlens_gathered = seqlens_gathered.cpu().tolist()
+
+ # Calculate the offsets to assign unique global ID to each subsample.
+ csum = torch.cumsum(dp_subsample_counts, dim=0, dtype=torch.int32)
+ offsets = torch.cat([torch.zeros(1, dtype=torch.int32), csum[:-1]], dim=0)
+
+ return seqlens_gathered, offsets
+
+ def get_global_id_seqlens(self, num_local_subsamples, offsets, seqlens_gathered):
+ """
+ Calculates the global ID for each subsample.
+
+ We assign a unique global ID to each subsample.
+
+ Returns:
+ global_id_seqlens: list of (global_id, seqlen) tuples for scheduling.
+ global_ids_this_rank: list of global IDs locally present on this rank.
+ """
+ dp_rank = self.dp_group.rank()
+ global_ids = torch.arange(len(seqlens_gathered), dtype=torch.int32).cuda()
+ # Create a list of (global_id, seqlen) tuples for scheduling
+ global_id_seqlens = [(i, seqlens_gathered[i]) for i in range(len(global_ids))]
+ # Get the global IDs locally present on this rank
+ global_ids_this_rank = global_ids[
+ offsets[dp_rank] : offsets[dp_rank] + num_local_subsamples
+ ]
+
+ return global_id_seqlens, global_ids_this_rank
+
+ def _gid_to_src_rank(self, gid: int, offsets: List[int]) -> int:
+ dp_src_rank = torch.bucketize(gid, offsets[1:] - 1)
+ # Since the torch.distributed.get_process_group_ranks
+ # provides the global rank, we need to consider TP
+ hdp_rank = (
+ torch.distributed.get_process_group_ranks(self.dp_group)[dp_src_rank]
+ // self.tp_group.size()
+ )
+ return hdp_rank
+
+ def reroute_samples_to_hdp_ranks(
+ self, batch, global_ids_this_rank, global_id_seqlens, sample_id_groups, offsets
+ ):
+ """
+ Reroutes the sub-samples to the correct rank after scheduling.
+
+ For each key in the batch dict, we perform an all-to-all communication
+ to transfer the data to the correct ranks.
+ Since all CP ranks within a DP group have the same data, we only need
+ to transfer data between matching CP ranks.
+ """
+ gid2local_id = {int(gid): i for i, gid in enumerate(global_ids_this_rank)}
+ hdp_rank = self.dp_cp_group.rank()
+ dp_ranks = torch.distributed.get_process_group_ranks(self.dp_group)
+ # Here we actually want to get the DP group's rank within the HDP group,
+ # we need to consider TP
+ dp_ranks = [r // self.tp_group.size() for r in dp_ranks]
+
+ data_keys = batch[0].keys()
+
+ # Create the send plan
+ combined_sample_id_groups: List[List[int]] = [[] for _ in range(self.total_hdp_gpus)]
+
+ for d in range(self.total_hdp_gpus):
+ for sample_id_group in sample_id_groups:
+ combined_sample_id_groups[d].extend(sample_id_group[d])
+
+ for dest_rank in range(self.total_hdp_gpus):
+ combined_sample_id_groups[dest_rank].sort()
+
+ # Filter out samples that are not present on this rank
+ send_ids_sorted = [
+ gid
+ for d in dp_ranks
+ for gid in combined_sample_id_groups[d]
+ if gid in global_ids_this_rank
+ ]
+ # send_counts = [len(combined_sample_id_groups[d]) for d in range(self.total_hdp_gpus)]
+
+ send_lens_split = [0] * self.total_hdp_gpus
+ for dest_rank in range(self.total_hdp_gpus):
+ if dest_rank in dp_ranks:
+ send_lens_split[dest_rank] = sum(
+ [
+ global_id_seqlens[gid][1]
+ for gid in combined_sample_id_groups[dest_rank]
+ if gid in global_ids_this_rank
+ ]
+ )
+ else:
+ # We only need to share local data with DP ranks that have different data.
+ send_lens_split[dest_rank] = 0
+
+ # Create the recv plan
+ recv_sample_id_groups = [[] for _ in range(self.total_hdp_gpus)]
+ for gid in combined_sample_id_groups[hdp_rank]:
+ src_rank = self._gid_to_src_rank(gid, offsets)
+ recv_sample_id_groups[src_rank].append(gid)
+
+ recv_lens_split = [0] * self.total_hdp_gpus
+ for src_rank in range(self.total_hdp_gpus):
+ recv_lens_split[src_rank] = sum(
+ [global_id_seqlens[gid][1] for gid in recv_sample_id_groups[src_rank]]
+ )
+
+ recv_ids_sorted = [
+ gid for d in range(self.total_hdp_gpus) for gid in recv_sample_id_groups[d]
+ ]
+ recv_counts = [len(recv_sample_id_groups[d]) for d in range(self.total_hdp_gpus)]
+
+ recv_samples = [{k: None for k in data_keys} for _ in range(sum(recv_counts))]
+
+ def _pack_sample_by_key(key: str) -> torch.Tensor:
+ flattened_tensors = []
+ for gid in send_ids_sorted:
+ t = batch[gid2local_id[gid]][key].to(torch.cuda.current_device(), non_blocking=True)
+ flattened_tensors.append(t)
+ return (
+ torch.cat(flattened_tensors, dim=0)
+ if flattened_tensors
+ else torch.empty(0, device=torch.cuda.current_device(), dtype=batch[0][key].dtype)
+ )
+
+ def _unpack_sample_by_key(key: str, recv_tensor: torch.Tensor):
+ cursor = 0
+ for i, gid in enumerate(recv_ids_sorted):
+ sample_len = global_id_seqlens[gid][1]
+ recv_samples[i][key] = recv_tensor[cursor : cursor + sample_len]
+ cursor += sample_len
+
+ for key in data_keys:
+ send_tensor = _pack_sample_by_key(key)
+ recv_tensor = torch.empty(
+ sum(recv_lens_split), device=torch.cuda.current_device(), dtype=send_tensor.dtype
+ )
+ torch.distributed.all_to_all_single(
+ output=recv_tensor,
+ input=send_tensor,
+ output_split_sizes=recv_lens_split,
+ input_split_sizes=send_lens_split,
+ group=self.dp_cp_group,
+ )
+ _unpack_sample_by_key(key, recv_tensor)
+
+ recv_sample_with_id = {
+ recv_id: recv_samples[i] for i, recv_id in enumerate(recv_ids_sorted)
+ }
+ return recv_sample_with_id
+
+ def unpack_batch(self, batch):
+ """
+ Unpacks the packed samples into a list of sub-samples.
+ Since each sub-sample may be routed to different DPxCP ranks,
+ we unpack the sample here to avoid unnecessarily transferring
+ the entire packed sample.
+ """
+ batch_unpacked = []
+ for sample in batch:
+ for sub_sample in range(sample["cu_seqlens"].shape[0] - 1):
+ sub_sample_dict = {}
+ start_idx = sample["cu_seqlens"][sub_sample]
+ end_idx = sample["cu_seqlens"][sub_sample + 1]
+ if end_idx - start_idx == 0:
+ continue
+ for key in sample.keys():
+ if key in ["cu_seqlens", "batch_idx", "max_seqlen"]:
+ continue
+ sub_sample_dict[key] = sample[key][start_idx:end_idx]
+ batch_unpacked.append(sub_sample_dict)
+ return batch_unpacked
+
+ def __next__(self) -> Any:
+ """
+ Get the next item from the dataset, pull scheduling metadata and return it.
+ """
+ if self.data_iterator is None:
+ # TP0 reads from data_iterator, others receive via broadcast.
+ return None, None
+ else:
+ batch = next(self.data_iterator)
+ subsample_seqlens = []
+ for sample in batch:
+ subsample_seqlens.extend(
+ [
+ int(sample["cu_seqlens"][i + 1] - sample["cu_seqlens"][i])
+ for i in range(0, sample["cu_seqlens"].shape[0] - 1)
+ ]
+ )
+ subsample_seqlens = torch.tensor(subsample_seqlens, dtype=torch.int32).cuda()
+ subsample_seqlens = subsample_seqlens[subsample_seqlens != 0]
+
+ seqlens_gathered, offsets = self.get_global_seqlens(subsample_seqlens)
+
+ global_id_seqlens, global_ids_this_rank = self.get_global_id_seqlens(
+ subsample_seqlens.shape[0], offsets, seqlens_gathered
+ )
+
+ groups, sample_id_groups = self.cp_balancing_scheduler.get_groups_and_subsamples(
+ global_id_seqlens, self.config
+ )
+
+ batch = self.unpack_batch(batch)
+ samples_this_rank_with_id = self.reroute_samples_to_hdp_ranks(
+ batch, global_ids_this_rank, global_id_seqlens, sample_id_groups, offsets
+ )
+ return samples_this_rank_with_id, sample_id_groups
diff --git a/megatron/core/datasets/gpt_dataset.py b/megatron/core/datasets/gpt_dataset.py
index a8e00d5ab75..a2d39a6d688 100644
--- a/megatron/core/datasets/gpt_dataset.py
+++ b/megatron/core/datasets/gpt_dataset.py
@@ -49,6 +49,24 @@ class GPTDatasetConfig(BlendedMegatronDatasetConfig):
object_storage_cache_path: Optional[str] = None
"""Path for caching indices for s3 or msc dataloading."""
+ context_parallel_size: int = 1
+ """Option to enable context parallelism"""
+
+ data_parallel_size: int = 1
+ """Option to enable data parallelism"""
+
+ sequence_parallel_size: int = 0
+ """Option to indicate the sequence parallelism size when using TP
+ Set to 0 if sequence parallel is not enabled regardless of TP size.
+ """
+
+ hybrid_context_parallel: bool = False
+ """Option to enable hybrid context parallelism. When setting this to True,
+ each sample should be divisible by the data parallel size * context parallel size * 2.
+ If sequence parallel is enabled, it should be divisible by the
+ data parallel size * context parallel size * sequence parallel size * 2.
+ """
+
def __post_init__(self) -> None:
"""Do asserts and set fields post init"""
super().__post_init__()
diff --git a/megatron/core/datasets/retro/config/config.py b/megatron/core/datasets/retro/config/config.py
index ac9ca841242..73f34a47545 100644
--- a/megatron/core/datasets/retro/config/config.py
+++ b/megatron/core/datasets/retro/config/config.py
@@ -5,6 +5,7 @@
from dataclasses import dataclass
from megatron.core.transformer import TransformerConfig
+from megatron.core.utils import experimental_api
from .bert_embedders import RetroBertEmbedders
from .gpt_chunk_datasets import RetroGPTChunkDatasets
@@ -12,7 +13,9 @@
@dataclass
+@experimental_api
class RetroPreprocessingConfig(TransformerConfig):
+ # pylint: disable=line-too-long
"""Configuration object for Retro preprocessing.
*Note* : Arguments prefixed with '--retro-gpt-*' or '--retro-bert-*' are
diff --git a/megatron/core/distributed/distributed_data_parallel.py b/megatron/core/distributed/distributed_data_parallel.py
index df1d7ae94db..e831d7cf4ec 100644
--- a/megatron/core/distributed/distributed_data_parallel.py
+++ b/megatron/core/distributed/distributed_data_parallel.py
@@ -6,7 +6,6 @@
import torch
-from .. import parallel_state
from ..config_logger import has_config_logger_enabled, log_config_to_disk
from ..fp8_utils import is_float8tensor, post_all_gather_processing
from ..process_groups_config import ProcessGroupCollection
@@ -55,10 +54,15 @@ def __init__(
# If using very large dp_sizes, make buckets larger to ensure that chunks used in NCCL
# ring-reduce implementations are large enough to remain bandwidth-bound rather than
# latency-bound.
+ # Setup process groups, handling both None and provided pg_collection values.
+ process_group_dict = ProcessGroupCollection.setup_process_groups_for_ddp(
+ pg_collection, config, ddp_config
+ )
+
+ # If bucket_size is not provided as an input, use sane default based on dp_group size.
+ dp_group = process_group_dict['dp_group']
if ddp_config.bucket_size is None:
- ddp_config.bucket_size = max(
- 40000000, 1000000 * parallel_state.get_data_parallel_world_size()
- )
+ ddp_config.bucket_size = max(40000000, 1000000 * dp_group.size())
# Set bucket_size to infinity if overlap_grad_reduce is False.
if not ddp_config.overlap_grad_reduce:
ddp_config.bucket_size = None
@@ -70,45 +74,19 @@ def __init__(
f'Setting up DistributedDataParallel with config {self.ddp_config}',
)
- if pg_collection is None:
- self.dp_group = parallel_state.get_data_parallel_group(
- with_context_parallel=False, partial_data_parallel=False
- )
- self.dp_cp_group = parallel_state.get_data_parallel_group(
- with_context_parallel=True, partial_data_parallel=False
- )
- self.intra_dp_cp_group = parallel_state.get_data_parallel_group(
- with_context_parallel=True, partial_data_parallel=True
- )
- self.expt_dp_group = parallel_state.get_expert_data_parallel_group()
- self.intra_expt_dp_group = parallel_state.get_expert_data_parallel_group(
- partial_expert_data_parallel=True
- )
- if self.ddp_config.num_distributed_optimizer_instances > 1:
- self.inter_dist_opt_group = (
- parallel_state.get_inter_distributed_optimizer_instance_group()
- )
- self.tp_group = parallel_state.get_tensor_model_parallel_group()
- self.pp_group = parallel_state.get_pipeline_model_parallel_group()
- self.ep_group = parallel_state.get_expert_model_parallel_group()
- else:
- # Setup process groups using DDP-specific helper method
- process_groups = ProcessGroupCollection.setup_process_groups_for_ddp(
- pg_collection, config, self.ddp_config
- )
-
- self.dp_group = process_groups['dp_group']
- self.dp_cp_group = process_groups['dp_cp_group']
- self.intra_dp_cp_group = process_groups['intra_dp_cp_group']
- self.expt_dp_group = process_groups['expt_dp_group']
- self.intra_expt_dp_group = process_groups['intra_expt_dp_group']
- self.tp_group = process_groups['tp_group']
- self.pp_group = process_groups['pp_group']
- self.ep_group = process_groups['ep_group']
-
- # Set inter_dist_opt_group if multiple optimizer instances
- if self.ddp_config.num_distributed_optimizer_instances > 1:
- self.inter_dist_opt_group = process_groups['inter_dist_opt_group']
+ # Assign all required process groups
+ self.dp_group = process_group_dict['dp_group']
+ self.dp_cp_group = process_group_dict['dp_cp_group']
+ self.intra_dp_cp_group = process_group_dict['intra_dp_cp_group']
+ self.expt_dp_group = process_group_dict['expt_dp_group']
+ self.intra_expt_dp_group = process_group_dict['intra_expt_dp_group']
+ self.tp_group = process_group_dict['tp_group']
+ self.pp_group = process_group_dict['pp_group']
+ self.ep_group = process_group_dict['ep_group']
+
+ # Set inter_dist_opt_group if multiple optimizer instances
+ if self.ddp_config.num_distributed_optimizer_instances > 1:
+ self.inter_dist_opt_group = process_group_dict['inter_dist_opt_group']
# Turn off bucketing if we are on a pipeline stage that is not the first (since
# data-parallel communication on these stages is not on the critical path), or if
diff --git a/megatron/core/distributed/distributed_data_parallel_config.py b/megatron/core/distributed/distributed_data_parallel_config.py
index e2a026d836f..eaec971c79c 100644
--- a/megatron/core/distributed/distributed_data_parallel_config.py
+++ b/megatron/core/distributed/distributed_data_parallel_config.py
@@ -137,6 +137,14 @@ class DistributedDataParallelConfig:
when nccl_ub is set.
"""
+ fsdp_manual_registration: bool = False
+ """If true, manually register the FSDP communication buffers to NCCL user buffer.
+ This option is only effective when use_megatron_fsdp and nccl_ub is set.
+ For symmetric registration with large models, the registration itself can take
+ a significant amount of time. This option minimizes the number of registration calls
+ to minimize the registration time.
+ """
+
delay_wgrad_compute: bool = False
"""Delay the weight gradient computation to improve batch-level communication overlapping"""
@@ -146,14 +154,6 @@ def __post_init__(self):
"""Check the validity of the config."""
if self.reuse_grad_buf_for_mxfp8_param_ag:
assert self.fp8_param_gather, "Reuse grad buffer only when keeping params in MXFP8."
- # Using mxfp8 param without overlap param gather and overlap grad reduce will cause NaN.
- # TODO: Remove this assertion when the issue is fixed.
- assert (
- self.overlap_param_gather
- ), "--overlap-param-gather is required when using mxfp8 params"
- assert (
- self.overlap_grad_reduce
- ), "--overlap-grad-reduce is required when using mxfp8 params"
if self.nccl_ub:
if 'expandable_segments:True' in os.getenv('PYTORCH_CUDA_ALLOC_CONF', '').split(','):
diff --git a/megatron/core/distributed/finalize_model_grads.py b/megatron/core/distributed/finalize_model_grads.py
index 55663acdc10..ddaeb7e8d84 100644
--- a/megatron/core/distributed/finalize_model_grads.py
+++ b/megatron/core/distributed/finalize_model_grads.py
@@ -1,4 +1,4 @@
-# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved.
+# Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
from functools import partial
from typing import Callable, List, Optional, Union
@@ -193,7 +193,11 @@ def _allreduce_word_embedding_grads(
pp_group = parallel_state.get_pipeline_model_parallel_group()
_allreduce_embedding_grad(
- model, embd_group, pp_group, partial(_get_shared_word_embedding_weight, config=config)
+ model,
+ embd_group,
+ pp_group,
+ partial(_get_shared_word_embedding_weight, config=config),
+ config=config,
)
@@ -203,6 +207,7 @@ def _allreduce_embedding_grad(
pp_group: torch.distributed.ProcessGroup,
weight_getter: Callable[[torch.nn.Module], Optional[torch.nn.Parameter]],
skip_if_none: bool = True,
+ config: TransformerConfig = None,
):
"""Unified helper to all-reduce embedding parameters across pipeline stages.
@@ -229,6 +234,9 @@ def _allreduce_embedding_grad(
model_module = model[0]
elif is_pp_last_stage(pp_group):
model_module = model[-1]
+ elif getattr(config, 'mtp_num_layers', None) is not None and config.mtp_num_layers > 0:
+ # Embedding for MTP layers is in the last virtual pipeline model parallel stage.
+ model_module = model[-1]
else: # We do not support an interleaved schedule for models with encoders yet.
model_module = model[0]
diff --git a/megatron/core/distributed/fsdp/mcore_fsdp_adapter.py b/megatron/core/distributed/fsdp/mcore_fsdp_adapter.py
index 7432a7f9a36..d6384e70488 100644
--- a/megatron/core/distributed/fsdp/mcore_fsdp_adapter.py
+++ b/megatron/core/distributed/fsdp/mcore_fsdp_adapter.py
@@ -111,6 +111,9 @@ def __init__(
dist_index=self.megatron_fsdp_dist_index,
calculate_per_token_loss=config.calculate_per_token_loss,
init_model_with_meta_device=config.init_model_with_meta_device,
+ enable_fine_grained_param_gather_hook=(
+ config.fp8_recipe == "mxfp8" and ddp_config.fp8_param_gather
+ ),
),
)
self.param_and_grad_buffer = self.module.param_and_grad_buffer
@@ -123,6 +126,7 @@ def __init__(
self.broadcast_params = self.module.broadcast_params
self.module.state_dict_for_save_checkpoint = self.module.state_dict
self.state_dict_for_save_checkpoint = self.state_dict
+ self.module.config = config
self.sync_rng_states_across_tp_group()
diff --git a/megatron/core/distributed/fsdp/src/README.md b/megatron/core/distributed/fsdp/src/README.md
index 404d8cbc89a..d51797fd51d 100644
--- a/megatron/core/distributed/fsdp/src/README.md
+++ b/megatron/core/distributed/fsdp/src/README.md
@@ -280,13 +280,16 @@ optimizer.load_state_dict(ckpt_state_dict["optimizer"])
- **Only effective when using Megatron-LM.**
- Defaults to `False`.
- `nccl_ub` will allocate and register the NCCL userbuffer for param and grad buffers. This option enables an SM-efficient NCCL algorithm that could improve the performance of overlapped computations. This flag will be much more effective when used together with SHARP if the FSDP communication includes both NVL and IB domains. Enabling this option will cause additional memory overhead due to the requirement to enable the `fsdp_double_buffer` option.
- - **Only effective when using Megatron-LM.**
+ - **Only effective when using with Megatron-Core.**
- Defaults to `False`.
- By default we try to use NCCL window (symmetric) registration if it is available. If not it falls back to conventional local registraion.
+- `fsdp_manual_registration` will manually register the FSDP communication buffers with the NCCL user buffer. For symmetric registration with large models, the registration itself can take a significant amount of time. This option minimizes the number of registration calls to reduce the registration time. However, with this option enabled, you need to manually call the `ParamAndGradBuffer.manual_buffer_registration()` function after the first iteration. This is already implemented in the Megatron-LM training loop. In other use cases, users are expected to call this function themselves.
+ - **Only effective when using with Megatron-Core.**
+ - This option is only effective when `nccl_ub` is enabled.
+ - Defaults to `False`.
- `disable_symmetric_registration` will disable NCCL window (i.e. symmetric) registraion when using `nccl_ub`.
- Dafaults to `False`.
- `fsdp_double_buffer` will use persistently allocated double buffers for temporarily-defined memory needed in `MegatronFSDP` communications. Having persistent double buffers may increase peak VRAM utilization, but is required to register NCCL user buffers (`nccl_ub=True`) for `MegatronFSDP`. Currently, this is only supported for simple repetitive model structures such as GPT.
- - **Only effective when using Megatron-LM.**
- Defaults to `False`. Automatically overridden to `True` when `nccl_ub` is enabled.
- `preproc_state_dict_for_dcp_ckpt` adds `model.state_dict()` and `optimizer.state_dict()` post-hooks that modify the model and optimizer state in preparation for `torch.distributed.checkpoint.{save,load}` ([Torch DCP](https://docs.pytorch.org/docs/stable/distributed.checkpoint.html)) checkpointing. Specifically, it adds `__create_write_items__` and `__create_chunk_list__` methods to Tensors utilized by Torch DCP to redistribute parameters when saving and loading model and optimizer checkpoints. Can be deactivated should the user need a custom distributed checkpointing strategy.
- Defaults to `True`.
diff --git a/megatron/core/distributed/fsdp/src/megatron_fsdp/distributed_data_parallel_config.py b/megatron/core/distributed/fsdp/src/megatron_fsdp/distributed_data_parallel_config.py
index 5151ecabfb5..f0c817e1f80 100644
--- a/megatron/core/distributed/fsdp/src/megatron_fsdp/distributed_data_parallel_config.py
+++ b/megatron/core/distributed/fsdp/src/megatron_fsdp/distributed_data_parallel_config.py
@@ -131,20 +131,20 @@ class DistributedDataParallelConfig:
when nccl_ub is set.
"""
+ fsdp_manual_registration: bool = False
+ """If true, manually register the FSDP communication buffers to NCCL user buffer.
+ This option is only effective when use_megatron_fsdp and nccl_ub is set.
+ For symmetric registration with large models, the registration itself can take
+ a significant amount of time. This option minimizes the number of registration calls
+ to minimize the registration time.
+ """
+
def __post_init__(self):
import os
"""Check the validity of the config."""
if self.reuse_grad_buf_for_mxfp8_param_ag:
assert self.fp8_param_gather, "Reuse grad buffer only when keeping params in MXFP8."
- # Using mxfp8 param without overlap param gather and overlap grad reduce will cause NaN.
- # TODO: Remove this assertion when the issue is fixed.
- assert (
- self.overlap_param_gather
- ), "--overlap-param-gather is required when using mxfp8 params"
- assert (
- self.overlap_grad_reduce
- ), "--overlap-grad-reduce is required when using mxfp8 params"
if self.nccl_ub:
if 'expandable_segments:True' in os.getenv('PYTORCH_CUDA_ALLOC_CONF', '').split(','):
diff --git a/megatron/core/distributed/fsdp/src/megatron_fsdp/megatron_fsdp.py b/megatron/core/distributed/fsdp/src/megatron_fsdp/megatron_fsdp.py
index 5e953e8c6c2..e2cbccf4356 100644
--- a/megatron/core/distributed/fsdp/src/megatron_fsdp/megatron_fsdp.py
+++ b/megatron/core/distributed/fsdp/src/megatron_fsdp/megatron_fsdp.py
@@ -23,6 +23,20 @@
import torch.nn as nn
from torch.utils._pytree import tree_flatten, tree_map, tree_unflatten
+from .mixed_precision import (
+ fp8_create_transpose_cache,
+ fp8_discard_transpose_cache,
+ is_float8tensor,
+)
+from .param_and_grad_buffer import (
+ AllGatherPipeline,
+ BucketingPolicy,
+ GradReducePipeline,
+ ParamAndGradBuffer,
+ PrefetchOrder,
+ override_sharded_param_methods_with_safety_checks,
+ to_local_if_dtensor,
+)
from .utils import FSDPDistributedIndex
logger = logging.getLogger(__name__)
@@ -34,23 +48,12 @@
from megatron.core.distributed.distributed_data_parallel_config import (
DistributedDataParallelConfig,
)
- from megatron.core.fp8_utils import is_float8tensor
from megatron.core.utils import is_submodule
except ImportError:
# Megatron-LM is not installed, use Megatron-FSDP as a standalone module.
logger.info("Megatron Core is not installed, Megatron-FSDP will run without Megatron Core.")
from .distributed_data_parallel_config import DistributedDataParallelConfig
- from .utils import is_float8tensor, is_submodule
-
-from .param_and_grad_buffer import (
- AllGatherPipeline,
- BucketingPolicy,
- GradReducePipeline,
- ParamAndGradBuffer,
- PrefetchOrder,
- override_sharded_param_methods_with_safety_checks,
- to_local_if_dtensor,
-)
+ from .utils import is_submodule
class TrainingState(Enum):
@@ -168,6 +171,7 @@ def __init__(
nccl_ub: bool = False,
fsdp_double_buffer: bool = False,
disable_symmetric_registration: bool = False,
+ enable_fine_grained_param_gather_hook: bool = False,
):
super().__init__()
# If device is not specified, use the current device.
@@ -217,6 +221,7 @@ def __init__(
self.calculate_per_token_loss = calculate_per_token_loss
self.init_model_with_meta_device = init_model_with_meta_device
+ self.enable_fine_grained_param_gather_hook = enable_fine_grained_param_gather_hook
# Whether to constantly synchronize the model every training iteration,
# which defaults to False to overlap communication with computation
@@ -406,6 +411,7 @@ def all_gather_and_wait_parameters_ready(
prefetch=True,
prefetch_order=PrefetchOrder.FORWARD_PASS_ORDER,
wait_bucket_ready=True,
+ bwd=False,
):
"""
All-gather parameters across the data parallel group and wait for
@@ -432,11 +438,14 @@ def all_gather_and_wait_parameters_ready(
and self.ddp_config.outer_dp_sharding_strategy != "no_shard"
and (self.microbatch_count == 0 or self.model_auto_sync)
),
+ bwd=bwd,
)
if wait_bucket_ready:
for param in params:
bucket_id = self.param_and_grad_buffer.param_to_param_group[param]
- ag_pipeline.wait_bucket_ready(bucket_id)
+ ag_pipeline.wait_bucket_ready(bucket_id, bwd)
+ if bwd and is_float8tensor(param):
+ fp8_create_transpose_cache(param)
for param in params:
# This setting is needed to make FSDP store the weight object when used
@@ -495,19 +504,17 @@ def _register_fsdp_hooks(self, root_module):
"""
fsdp_unit_modules = self.fsdp_unit_modules
- def release_module_parameters(module, *unused):
+ def release_module_parameters(module, bwd, *unused):
for param in module.parameters():
bucket_id = self.param_and_grad_buffer.param_to_param_group[param]
- self.all_gather_pipeline.release_bucket(bucket_id)
-
+ self.all_gather_pipeline.release_bucket(bucket_id, bwd)
if not self.ddp_config.keep_fp8_transpose_cache:
release_params_fp8_transpose_cache(module.parameters())
def release_params_fp8_transpose_cache(params):
for param in params:
if is_float8tensor(param):
- param._transpose_invalid = True
- param._transpose = None
+ fp8_discard_transpose_cache(param)
def _grad_acc(param):
"""
@@ -564,12 +571,15 @@ def _post_backward(module, *unused):
if self.ddp_config.data_parallel_sharding_strategy == "optim_grads_params":
# Deallocate the module parameters after the backward pass,
# because we have our data-parallel gradients computed.
- release_module_parameters(module)
+ release_module_parameters(module, bwd=True)
module._training_state = TrainingState.IDLE
param_list = list(module.parameters())
else:
param_list = list(module.parameters(recurse=False))
+ if self.enable_fine_grained_param_gather_hook:
+ param_list = list(module.parameters(recurse=False))
+
# If the parameter is shared, we do not accumulate gradients
# here, as the gradients will be accumulated in the
# root post-backward hook.
@@ -621,6 +631,9 @@ def _pre_forward_param_unshard(
# to allocate as little memory as possible for this forward pass.
param_list = list(module.parameters(recurse=False))
+ if self.enable_fine_grained_param_gather_hook:
+ param_list = list(module.parameters(recurse=False))
+
# All-gather the parameters before the forward pass.
self.all_gather_and_wait_parameters_ready(
params=param_list,
@@ -720,7 +733,7 @@ def _root_post_backward(*unused):
if self.model_auto_sync:
self.finish_grad_sync()
- def _pre_backward(module: nn.Module, *unused):
+ def _pre_backward_param_unshard(module: nn.Module, *unused):
"""
Sub-module pre-backward hook to all-gather the module parameters
before the backward pass.
@@ -729,11 +742,19 @@ def _pre_backward(module: nn.Module, *unused):
# and unsharding operations when performing activation recomputation
# / gradient checkpointing.
module._training_state = TrainingState.PRE_BACKWARD
+
if isinstance(module, tuple(fsdp_unit_modules)):
- # All-gather / unshard the module parameters before the backward pass.
- self.all_gather_and_wait_parameters_ready(
- list(module.parameters()), prefetch_order=PrefetchOrder.BACKWARD_PASS_ORDER
- )
+ param_list = list(module.parameters())
+ else:
+ param_list = list(module.parameters(recurse=False))
+
+ if self.enable_fine_grained_param_gather_hook:
+ param_list = list(module.parameters(recurse=False))
+
+ # All-gather / unshard the module parameters before the backward pass.
+ self.all_gather_and_wait_parameters_ready(
+ param_list, prefetch_order=PrefetchOrder.BACKWARD_PASS_ORDER, bwd=True
+ )
self._root_pre_backward_hook_issued = False
@@ -760,7 +781,9 @@ def _root_pre_backward(module: nn.Module, *unused):
for bucket_id in range(ag_pipeline.num_buckets):
group = self.param_and_grad_buffer.parameter_groups[bucket_id]
if group.fsdp_unit_id is not None:
- ag_pipeline.bucket_can_be_released[bucket_id] = True
+ ag_pipeline.bucket_can_be_released[
+ ag_pipeline.get_bucket_key(bucket_id, bwd=False)
+ ] = True
# Track parameters that require gradient reduction and optimization.
self._params_require_handle_grad = set()
for param_group in self.param_and_grad_buffer.parameter_groups:
@@ -782,8 +805,12 @@ def _post_forward(module: nn.Module, input: Any, output: Any):
# during activation recomputation / gradient checkpointing.
return output
+ assert isinstance(
+ module, tuple(fsdp_unit_modules)
+ ), "_post_forward hook should only be registered on FSDP unit modules."
+
# Release the module parameters after the forward pass to save memory.
- release_module_parameters(module)
+ release_module_parameters(module, bwd=False)
module._training_state = TrainingState.IDLE
return output
@@ -824,21 +851,55 @@ def forward_hook(_module, inputs, output):
# on the output tensor(s).
return module.register_forward_hook(forward_hook)
+ def _register_pre_forward_param_unshard_hook(module):
+ """
+ Register the forward pre-hook to unshard parameters before the forward pass.
+ If we are not sharding anything, we do not have a model weight buffer and thus
+ have nothing to all-gather / un-shard.
+ """
+ if self.ddp_config.data_parallel_sharding_strategy != "no_shard":
+ self.forward_pre_hooks[f"{module._get_name()} parameter unshard"] = (
+ module.register_forward_pre_hook(
+ _pre_forward_param_unshard, prepend=True, with_kwargs=True
+ )
+ )
+
+ def _register_pre_backward_param_unshard_hook(module):
+ """
+ Register the backward pre-hook to unshard FSDP unit module parameters
+ immediately before the backward pass via attaching a gradient-triggered
+ hook to the output tensor(s) of a module during a post-forward hook.
+ """
+ self.backward_pre_hooks[f"all-gather {module._get_name()} parameters"] = (
+ create_custom_backward_hook(module, _pre_backward_param_unshard)
+ )
+
+ def _register_grad_acc_and_reduce_hook(module):
+ """
+ Register the post-backward hook to deallocate model parameters and
+ reduce-scatter gradients immediately after the module backward pass
+ has completed to conserve memory for the subsequent backward pass.
+ """
+ self.forward_pre_hooks[f"module {name} register post-backward hook"] = (
+ module.register_forward_pre_hook(
+ functools.partial(_register_post_backward_hook, _post_backward),
+ with_kwargs=True,
+ )
+ )
+
fsdp_modules = []
for name, module in root_module.named_modules():
+ if self.enable_fine_grained_param_gather_hook:
+ _register_pre_forward_param_unshard_hook(module)
+ _register_pre_backward_param_unshard_hook(module)
+ _register_grad_acc_and_reduce_hook(module)
+
# Skip if the module is already registered in fsdp_modules.
if any(is_submodule(module, fsdp_module) for fsdp_module in fsdp_modules):
continue
- # Register the forward pre-hook to unshard parameters before the forward pass.
- # If we are not sharding anything, we do not have a model weight buffer and thus
- # have nothing to all-gather / un-shard.
- if self.ddp_config.data_parallel_sharding_strategy != "no_shard":
- self.forward_pre_hooks[f"module {name} parameter unshard"] = (
- module.register_forward_pre_hook(
- _pre_forward_param_unshard, prepend=True, with_kwargs=True
- )
- )
+ if not self.enable_fine_grained_param_gather_hook:
+ _register_pre_forward_param_unshard_hook(module)
if isinstance(module, tuple(fsdp_unit_modules)):
fsdp_modules.append(module)
@@ -849,12 +910,8 @@ def forward_hook(_module, inputs, output):
module.register_forward_hook(_post_forward, prepend=False)
)
- # Register the backward pre-hook to unshard FSDP unit module parameters
- # immediately before the backward pass via attaching a gradient-triggered
- # hook to the output tensor(s) of a module during a post-forward hook.
- self.backward_pre_hooks[f"all-gather module {name} parameters"] = (
- create_custom_backward_hook(module, _pre_backward)
- )
+ if not self.enable_fine_grained_param_gather_hook:
+ _register_pre_backward_param_unshard_hook(module)
elif (
not self.ddp_config.keep_fp8_transpose_cache
and self.ddp_config.data_parallel_sharding_strategy == "optim_grads_params"
@@ -867,15 +924,8 @@ def forward_hook(_module, inputs, output):
module.register_forward_hook(_release_module_fp8_transpose_cache, prepend=False)
)
- # Register the post-backward hook to deallocate model parameters and
- # reduce-scatter gradients immediately after the module backward pass
- # has completed to conserve memory for the subsequent backward pass.
- self.forward_pre_hooks[f"module {name} register post-backward hook"] = (
- module.register_forward_pre_hook(
- functools.partial(_register_post_backward_hook, _post_backward),
- with_kwargs=True,
- )
- )
+ if not self.enable_fine_grained_param_gather_hook:
+ _register_grad_acc_and_reduce_hook(module)
# Register root module pre- and post-backward hooks in cases where the
# forward function of root module is not called, but rather the forward
@@ -992,7 +1042,7 @@ def start_param_sync(self, *unused, force_sync: bool = False, force_dispatch: bo
else:
self.synchronize_param_gather()
for bucket_id in range(self.all_gather_pipeline.num_buckets):
- self.all_gather_pipeline.async_bucket_gather(bucket_id=bucket_id)
+ self.all_gather_pipeline.async_bucket_gather(bucket_id=bucket_id, bwd=False)
group = self.param_and_grad_buffer.parameter_groups[bucket_id]
if group.model_weight_buffer is None:
continue
@@ -1000,9 +1050,10 @@ def start_param_sync(self, *unused, force_sync: bool = False, force_dispatch: bo
if group.model_weight_buffer.is_data_distributed:
# If model weight is sharded, we wait for the all-gather to complete and
# then release the bucket immediately to save memory usage.
- self.all_gather_pipeline.wait_bucket_ready(bucket_id)
+ self.all_gather_pipeline.wait_bucket_ready(bucket_id, False)
+
for bucket_id in range(self.all_gather_pipeline.num_buckets):
- self.all_gather_pipeline.wait_bucket_ready(bucket_id)
+ self.all_gather_pipeline.wait_bucket_ready(bucket_id, False)
def start_grad_sync(self, *unused):
"""
diff --git a/megatron/core/distributed/fsdp/src/megatron_fsdp/mixed_precision.py b/megatron/core/distributed/fsdp/src/megatron_fsdp/mixed_precision.py
new file mode 100644
index 00000000000..d7156bea5c6
--- /dev/null
+++ b/megatron/core/distributed/fsdp/src/megatron_fsdp/mixed_precision.py
@@ -0,0 +1,334 @@
+# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+from importlib.metadata import version
+from typing import List, Optional, Tuple
+
+import torch
+from packaging.version import Version as PkgVersion
+
+logger = logging.getLogger(__name__)
+
+# Detect if Transformer Engine is installed
+try:
+ import transformer_engine # pylint: disable=W0611
+ from transformer_engine.pytorch.module.base import TransformerEngineBaseModule
+
+ HAVE_TE = True
+except (ImportError, ModuleNotFoundError):
+ TransformerEngineBaseModule = None
+ HAVE_TE = False
+ logger.info("Using Megatron-FSDP without Transformer Engine.")
+
+# Detect the Transformer Engine version
+try:
+ import transformer_engine as te
+
+ if hasattr(te, "__version__"):
+ TE_VERSION = PkgVersion(str(te.__version__))
+ else:
+ TE_VERSION = PkgVersion(version("transformer-engine"))
+except:
+ TE_VERSION = None
+
+# Detect the FP8 tensor class
+try:
+ from transformer_engine.pytorch.tensor import QuantizedTensor
+
+ HAVE_TE_FP8_TENSOR_CLASS = True
+ FP8_TENSOR_CLASS = QuantizedTensor
+except:
+ try:
+ from transformer_engine.pytorch.float8_tensor import Float8Tensor
+
+ HAVE_TE_FP8_TENSOR_CLASS = True
+ FP8_TENSOR_CLASS = Float8Tensor
+ except:
+ HAVE_TE_FP8_TENSOR_CLASS = False
+
+# Detect the MXFP8 tensor class
+try:
+ from transformer_engine.pytorch.tensor.mxfp8_tensor import MXFP8Tensor
+
+ HAVE_TE_MXFP8TENSOR = True
+except:
+ HAVE_TE_MXFP8TENSOR = False
+
+# Detect the Blockwise FP8 tensor class
+try:
+ from transformer_engine.pytorch.tensor.float8_blockwise_tensor import Float8BlockwiseQTensor
+
+ HAVE_TE_BLOCKWISE_FP8TENSOR = True
+except:
+ HAVE_TE_BLOCKWISE_FP8TENSOR = False
+
+# Detect the "cast_master_weights_to_fp8" function of Transformer Engine
+try:
+ from transformer_engine.pytorch.tensor.utils import cast_master_weights_to_fp8
+
+ HAVE_TE_CAST_MASTER_WEIGHTS_TO_FP8 = True
+except:
+ HAVE_TE_CAST_MASTER_WEIGHTS_TO_FP8 = False
+
+ # Try to import multi_tensor_apply, used in the fallback of fp8 quantization.
+ try:
+ from transformer_engine.pytorch.optimizers import multi_tensor_applier, multi_tensor_scale
+
+ multi_tensor_scale_impl = multi_tensor_scale
+ except ImportError:
+ try:
+ import amp_C
+ from apex.multi_tensor_apply import multi_tensor_applier
+
+ multi_tensor_scale_impl = amp_C.multi_tensor_scale
+ except ImportError:
+ import warnings
+
+ warnings.warn(
+ "Transformer Engine and Apex are not installed. "
+ "Falling back to local implementations of "
+ "multi_tensor_applier and multi_tensor_scale"
+ )
+
+ def local_multi_tensor_applier(op, noop_flag_buffer, tensor_lists, *args):
+ """Multi tensor op applier"""
+ return op(2048 * 32, noop_flag_buffer, tensor_lists, *args)
+
+ def local_multi_tensor_scale(chunk_size, noop_flag, tensor_lists, scale):
+ """Works as a drop-in replacement for amp_C.multi_tensor_scale."""
+ for src, dst in zip(tensor_lists[0], tensor_lists[1]):
+ dst.copy_(src * scale)
+
+ multi_tensor_applier = local_multi_tensor_applier
+ multi_tensor_scale_impl = local_multi_tensor_scale
+
+ def _multi_tensor_copy_this_to_that(
+ this: List[torch.Tensor],
+ that: List[torch.Tensor],
+ overflow_buf: Optional[torch.Tensor] = None,
+ ):
+ """
+ Use multi-tensor-applier to copy values from one list to another.
+ We don't have a bfloat16 implementation so for now if the overflow_buf
+ is not provided, we default back to simple loop copy to be compatible
+ with bfloat16.
+ """
+ if overflow_buf is not None:
+ overflow_buf.fill_(0)
+ # Scaling with factor `1.0` is equivalent to copy.
+ multi_tensor_applier(multi_tensor_scale_impl, overflow_buf, [this, that], 1.0)
+ else:
+ for this_, that_ in zip(this, that):
+ that_.copy_(this_)
+
+
+# Detect the "post_all_gather_processing" function of Transformer Engine
+try:
+ from transformer_engine.pytorch.tensor.utils import post_all_gather_processing
+
+ HAVE_TE_POST_ALL_GATHER_PROCESSING = True
+except:
+ HAVE_TE_POST_ALL_GATHER_PROCESSING = False
+
+
+def is_te_min_version(vers, check_equality=True):
+ """Check if minimum version of `transformer-engine` is installed."""
+ if not isinstance(TE_VERSION, PkgVersion):
+ return False
+
+ if check_equality:
+ return TE_VERSION >= PkgVersion(vers)
+ else:
+ return TE_VERSION > PkgVersion(vers)
+
+
+def is_float8tensor(tensor: torch.Tensor) -> bool:
+ """Check if a tensor is a FP8 tensor."""
+ return HAVE_TE and isinstance(tensor, FP8_TENSOR_CLASS)
+
+
+def is_blockwise_float8tensor(tensor: torch.Tensor) -> bool:
+ """Check if a tensor is a Blockwise FP8 tensor."""
+ return HAVE_TE_BLOCKWISE_FP8TENSOR and isinstance(tensor, Float8BlockwiseQTensor)
+
+
+def fp8_need_transpose_data(tensor: torch.Tensor) -> bool:
+ """Check if a FP8 tensor needs transpose data."""
+ return HAVE_TE_MXFP8TENSOR and isinstance(tensor, MXFP8Tensor)
+
+
+def fp8_need_transpose_data_for_meta_device_init(module: TransformerEngineBaseModule) -> bool:
+ """Check if a FP8 tensor needs transpose data, for meta device init scenario."""
+ return HAVE_TE_MXFP8TENSOR and module.fp8_meta["recipe"].mxfp8()
+
+
+def fp8_discard_transpose_cache(tensor: torch.Tensor) -> None:
+ """Discard the transpose cache of a FP8 tensor."""
+ assert is_float8tensor(tensor), f"Type {type(tensor)} is not a FP8 tensor"
+
+ if hasattr(tensor, "_transpose_invalid"):
+ tensor._transpose_invalid = True
+ tensor._transpose = None
+ elif not fp8_need_transpose_data(tensor):
+ tensor.update_usage(rowwise_usage=True, columnwise_usage=False)
+
+
+def fp8_create_transpose_cache(tensors: List[torch.Tensor]) -> None:
+ """Create the transpose cache of a FP8 tensor."""
+ if HAVE_TE_POST_ALL_GATHER_PROCESSING:
+ post_all_gather_processing(tensors)
+ else:
+ _fp8_create_transpose_cache_fallback(tensors)
+
+
+def _fp8_create_transpose_cache_fallback(tensors: List[torch.Tensor]) -> None:
+ if not isinstance(tensors, list):
+ tensors = [tensors]
+ for tensor in tensors:
+ assert is_float8tensor(tensor), f"Type {type(tensor)} is not a FP8 tensor"
+ if hasattr(tensor, "_create_transpose"):
+ tensor._create_transpose()
+ else:
+ tensor._create_columnwise()
+
+
+def fp8_set_raw_data(tensor: torch.Tensor, data: torch.Tensor, set_transpose: bool = False) -> None:
+ """Set the raw data of a Transformer Engine Float8Tensor."""
+ assert is_float8tensor(tensor), f"Type {type(tensor)} is not a FP8 tensor"
+
+ if set_transpose:
+ assert fp8_need_transpose_data(tensor), f"Type {type(tensor)} does not need transpose data"
+ data_attr = "_columnwise_data"
+ else:
+ data_attr = "_rowwise_data" if hasattr(tensor, "_rowwise_data") else "_data"
+
+ old_data = getattr(tensor, data_attr)
+ if old_data is not None:
+ assert (
+ old_data.dtype == data.dtype
+ ), f"The data types of raw data don't match {old_data.dtype} vs {data.dtype}"
+ assert (
+ old_data.shape == data.shape
+ ), f"Shape {old_data.shape} of old_data doesn't match {data.shape} of new_data"
+ setattr(tensor, data_attr, data)
+
+
+def fp8_get_raw_data(tensor: torch.Tensor, get_transpose: bool = False) -> torch.Tensor:
+ """Get the underlying raw storage of a FP8 tensor."""
+ assert is_float8tensor(tensor), f"Type {type(tensor)} is not a FP8 tensor"
+
+ if get_transpose:
+ assert fp8_need_transpose_data(tensor), f"Type {type(tensor)} does not need transpose data"
+ data_attr = "_columnwise_data"
+ else:
+ data_attr = "_rowwise_data" if hasattr(tensor, "_rowwise_data") else "_data"
+
+ return getattr(tensor, data_attr)
+
+
+def fp8_dequantize(tensor: torch.Tensor) -> torch.Tensor:
+ """Dequantize a FP8 tensor to a higher precision."""
+ assert is_float8tensor(tensor), f"Type {type(tensor)} is not a FP8 tensor"
+ assert is_te_min_version(
+ "2.0"
+ ), "Transformer Engine >= 2.0 is required for dequantizing parameters."
+ return tensor.dequantize()
+
+
+def fp8_quantize(
+ model_params: List[torch.Tensor],
+ main_params: List[torch.Tensor],
+ start_offsets: List[int],
+ data_parallel_group: torch.distributed.ProcessGroup,
+ fsdp_shard_model_params: List[Tuple[torch.Tensor, Optional[torch.Tensor]]],
+) -> None:
+ """Quantize sharded parameters to FP8."""
+ if len(model_params) == 0:
+ return
+ fsdp_shard_model_params = [x[0] if x[1] is None else x for x in fsdp_shard_model_params]
+
+ if HAVE_TE_CAST_MASTER_WEIGHTS_TO_FP8:
+ cast_master_weights_to_fp8(
+ model_params, main_params, start_offsets, data_parallel_group, fsdp_shard_model_params
+ )
+ else:
+ _fp8_quantize_fallback(
+ model_params, main_params, start_offsets, data_parallel_group, fsdp_shard_model_params
+ )
+
+
+def _fp8_quantize_fallback(
+ model_params: List[torch.Tensor],
+ main_params: List[torch.Tensor],
+ start_offsets: List[int],
+ data_parallel_group: torch.distributed.ProcessGroup,
+ fsdp_shard_model_params: List[Tuple[torch.Tensor, Optional[torch.Tensor]]],
+) -> None:
+ for model_param, main_param, start_offset, fsdp_shard_model_param in zip(
+ model_params, main_params, start_offsets, fsdp_shard_model_params
+ ):
+ if main_param is None:
+ continue
+
+ if fsdp_shard_model_param is not None:
+ shard_model_param = fsdp_shard_model_param
+ else:
+ shard_model_param = model_param._data.view(-1)[
+ start_offset : start_offset + main_param.numel()
+ ]
+
+ quantizer = model_param._quantizer
+ # When not using fp8 params, the main_param (fp32) is first cast to bf16/fp16, and then
+ # cast to fp8 during forward. This logic keeps numerical consistency with bf16 params.
+ main_param = main_param.to(model_param.dtype)
+ out = Float8Tensor(
+ shape=main_param.size(),
+ dtype=model_param.dtype,
+ requires_grad=False,
+ data=shard_model_param,
+ fp8_scale_inv=model_param._scale_inv,
+ fp8_dtype=model_param._fp8_dtype,
+ quantizer=quantizer,
+ )
+ quantizer.update_quantized(main_param, out)
+
+ amaxes = []
+ scales = []
+ scale_invs = []
+ for model_param in model_params:
+ quantizer = model_param._quantizer
+ amaxes.append(quantizer.amax.view(1))
+ scales.append(quantizer.scale.view(1))
+ scale_invs.append(model_param._scale_inv.view(1))
+ model_param._reset_caches()
+
+ dummy_overflow_buf = torch.tensor([0], dtype=torch.int, device="cuda")
+
+ # Update scaling factors.
+ packed_scales = torch.empty(len(scales), dtype=torch.float32, device=scales[0].device)
+ packed_scale_views = [packed_scales[i].view(1) for i in range(len(scales))]
+ _multi_tensor_copy_this_to_that(scales, packed_scale_views, dummy_overflow_buf)
+ torch.reciprocal(packed_scales, out=packed_scales)
+ _multi_tensor_copy_this_to_that(packed_scale_views, scale_invs, dummy_overflow_buf)
+
+ # Reduce amaxes.
+ # Note: Assume each param has a separate amax.
+ packed_amaxes = torch.empty(len(amaxes), dtype=torch.float32, device=amaxes[0].device)
+ packed_amax_views = [packed_amaxes[i].view(1) for i in range(len(amaxes))]
+ _multi_tensor_copy_this_to_that(amaxes, packed_amax_views, dummy_overflow_buf)
+ torch.distributed.all_reduce(
+ packed_amaxes, op=torch.distributed.ReduceOp.MAX, group=data_parallel_group
+ )
+ _multi_tensor_copy_this_to_that(packed_amax_views, amaxes, dummy_overflow_buf)
diff --git a/megatron/core/distributed/fsdp/src/megatron_fsdp/param_and_grad_buffer.py b/megatron/core/distributed/fsdp/src/megatron_fsdp/param_and_grad_buffer.py
index 627ae9a1311..04ea09970f4 100644
--- a/megatron/core/distributed/fsdp/src/megatron_fsdp/param_and_grad_buffer.py
+++ b/megatron/core/distributed/fsdp/src/megatron_fsdp/param_and_grad_buffer.py
@@ -32,6 +32,17 @@
from torch.distributed import _coalescing_manager
from torch.distributed.tensor import DTensor, Replicate, Shard
+from .mixed_precision import (
+ fp8_discard_transpose_cache,
+ fp8_get_raw_data,
+ fp8_need_transpose_data,
+ fp8_need_transpose_data_for_meta_device_init,
+ fp8_quantize,
+ fp8_set_raw_data,
+ is_blockwise_float8tensor,
+ is_float8tensor,
+ is_te_min_version,
+)
from .uneven_dtensor import update_uneven_dtensor_chunk_metadata, validate_uneven_dtensor
from .utils import (
_MODEL_PARALLEL_RNG_TRACKER_NAME,
@@ -50,27 +61,15 @@
from megatron.core.distributed.distributed_data_parallel_config import (
DistributedDataParallelConfig,
)
- from megatron.core.fp8_utils import (
- is_float8tensor,
- modify_underlying_storage,
- quantize_param_shard,
- )
from megatron.core.tensor_parallel import get_cuda_rng_tracker
- from megatron.core.utils import is_submodule, is_te_min_version
+ from megatron.core.utils import is_submodule
logger.info("Detected Megatron Core, using Megatron-FSDP with Megatron.")
except ImportError:
# Megatron-LM is not installed, use Megatron-FSDP as a standalone module.
from .distributed_data_parallel_config import DistributedDataParallelConfig
- from .utils import (
- get_cuda_rng_tracker,
- is_float8tensor,
- is_submodule,
- is_te_min_version,
- modify_underlying_storage,
- quantize_param_shard,
- )
+ from .utils import get_cuda_rng_tracker, is_submodule
logger.info("Megatron Core is not installed, Megatron-FSDP will run without Megatron Core.")
@@ -816,7 +815,7 @@ def __init__(
data_parallel_group: Optional[torch.distributed.ProcessGroup] = None,
dp_rank: Optional[int] = None,
temporary_bucket_allocator: Optional[TemporaryBucketAllocator] = None,
- is_dtype_float8: bool = False,
+ is_transpose_buffer: bool = False,
gradient_scaling_factor: Optional[float] = None,
chunk_size_factor: int = 1,
mem_alloc_context: Optional[Callable] = None,
@@ -849,7 +848,7 @@ def __init__(
self.temporary_bucket_allocator = (
temporary_bucket_allocator if temporary_bucket_allocator else TemporaryBucketAllocator()
)
- self.is_dtype_float8 = is_dtype_float8
+ self.is_transpose_buffer = is_transpose_buffer
self.gradient_scaling_factor = gradient_scaling_factor
self.mem_alloc_context = mem_alloc_context if mem_alloc_context else nullcontext
@@ -945,11 +944,11 @@ def fetch_bucket(
for p in self.params:
item_id = self.param_idx[p]
p = to_local_if_dtensor(p)
+ data = self.get_item_from_bucket(bucket, item_id).view(p.shape)
if is_float8tensor(p):
- p._data = self.get_item_from_bucket(bucket, item_id).view(p.shape)
+ fp8_set_raw_data(p, data, self.is_transpose_buffer)
else:
- p.data = self.get_item_from_bucket(bucket, item_id).view(p.shape)
-
+ p.data = data
return bucket
def free_bucket_storage(self):
@@ -1118,6 +1117,9 @@ def set_item(self, item_id: int, item_data: torch.Tensor) -> None:
# When fully sharded, we need to get the slice of the item to be stored in this shard.
# Otherwise, we can just flatten the entire item since this buffer contains
# the entire bucket.
+ if is_float8tensor(item_data):
+ item_data = fp8_get_raw_data(item_data, self.is_transpose_buffer)
+
if self.is_data_distributed:
# Get the coordinates of the slice of the item that is contained in this shard.
slice_start, slice_end = self._get_item_slice_in_shard(item_id)
@@ -1224,6 +1226,8 @@ class ParameterGroup:
Factor determining chunk size for grouped parameter processing.
model_weight_buffer (Optional[DataParallelBuffer]):
Buffer used to store model weights for data-parallel operations.
+ transpose_weight_buffer (Optional[DataParallelBuffer]):
+ Buffer used to store transpose weights for data-parallel operations.
main_weight_buffer (Optional[DataParallelBuffer]):
Buffer used to store main model weights for data-parallel operations.
main_grad_buffer (Optional[DataParallelBuffer]):
@@ -1243,6 +1247,7 @@ class ParameterGroup:
fsdp_unit_id: Optional[int] = None
chunk_size_factor: int = 1
model_weight_buffer: Optional[DataParallelBuffer] = None
+ transpose_weight_buffer: Optional[DataParallelBuffer] = None
main_weight_buffer: Optional[DataParallelBuffer] = None
main_grad_buffer: Optional[DataParallelBuffer] = None
hsdp_wbuf: Optional[DataParallelBuffer] = None
@@ -1313,12 +1318,10 @@ def _does_param_require_new_bucket(param):
parameter_groups = []
for name, param in module.named_parameters():
# We need this information to correctly dynamically allocate Tensors!
+ is_fp8 = is_float8tensor(param)
+ is_fp8_meta_device_init = meta_device_init_fp8_params.get(name, (False, False))[0]
param_attrs = dict(
- dtype=(
- "float8"
- if is_float8tensor(param) or meta_device_init_fp8_params.get(name, False)
- else param.dtype
- ),
+ dtype="float8" if (is_fp8 or is_fp8_meta_device_init) else param.dtype,
is_expert_param=is_expert_parameter(name, param),
requires_grad=param.requires_grad,
fsdp_unit_id=None,
@@ -1566,6 +1569,7 @@ def __init__(
reset_parameters_for_meta_device_init_module
)
self.ubr_groups = None
+ self.already_registered = False
# User buffer registration related settings
if self.ddp_config.nccl_ub:
assert nccl_allocator is not None, (
@@ -1640,7 +1644,10 @@ def __init__(
# to determine whether this parameter is fp8 or not.
fp8_meta_index = m.param_init_meta[name].fp8_meta_index
if m.primary_weights_in_fp8 and fp8_meta_index is not None:
- meta_device_init_fp8_params[self.param_to_name[param]] = True
+ meta_device_init_fp8_params[self.param_to_name[param]] = (
+ True,
+ fp8_need_transpose_data_for_meta_device_init(m),
+ )
# Get the parameter groups.
(self.parameter_groups, self.param_to_param_group, self.bucket_to_bucket_group) = (
@@ -1669,6 +1676,10 @@ def get_mem_alloc_context(self, groups=None, symmetric=True):
groups = [self.dist_index.get_fsdp_group(is_expert_parallel=False)]
if NCCL_ALLOCATOR == "MCORE":
+ if self.ddp_config.fsdp_manual_registration:
+ return functools.partial(
+ nccl_allocator.MemPoolAllocatorWithoutRegistration, NCCL_MEMORY_POOL
+ )
if len(groups) == 1:
# register buffers to the default group directly using nccl memory allocator
mem_alloc_context = functools.partial(
@@ -1685,6 +1696,12 @@ def get_mem_alloc_context(self, groups=None, symmetric=True):
symmetric=symmetric,
)
elif NCCL_ALLOCATOR == "APEX":
+ if self.ddp_config.fsdp_manual_registration:
+ logging.warning(
+ "FSDP manual registration is not supported for APEX NCCL allocator."
+ "falling back to default registration. "
+ "Please use Megatron Core NCCL allocator for manual registration."
+ )
if symmetric:
logging.warning(
"Symmetric registration is not supported for APEX NCCL allocator."
@@ -1708,6 +1725,39 @@ def get_mem_alloc_context(self, groups=None, symmetric=True):
else:
return nullcontext
+ def manual_buffer_registration(self):
+ """
+ Manually register the FSDP communication buffers to NCCL user buffer.
+ """
+ assert self.ddp_config.nccl_ub, "NCCL UBR is not enabled"
+ assert self.ddp_config.fsdp_double_buffer, "FSDP double buffer is not enabled"
+ assert self.ddp_config.fsdp_manual_registration, "FSDP manual registration is not enabled"
+ assert not self.already_registered, "Mem pool is already registered"
+
+ self.already_registered = True
+
+ global NCCL_MEMORY_POOL
+ torch.cuda.synchronize()
+ torch.distributed.barrier(async_op=False)
+ torch.cuda.synchronize()
+
+ for group in self.ubr_groups:
+ if torch.distributed.get_rank() == 0:
+ logging.info(
+ f"[MCORE][FSDP][Manual REG] Registering mem pool to group {group},"
+ f"group.group_desc:{group.group_desc}, group.size(): {group.size()}"
+ )
+ nccl_allocator.register_mem_pool(
+ NCCL_MEMORY_POOL,
+ group,
+ symmetric=not self.ddp_config.disable_symmetric_registration,
+ )
+ if torch.distributed.get_rank() == 0:
+ logging.info(
+ f"[MCORE][FSDP][Manual REG] Registered mem pool to group {group},"
+ f"group.group_desc:{group.group_desc}, group.size(): {group.size()}"
+ )
+
def _log_parameter_groups(self):
"""Compact log of FSDP parameter groups and their parameters."""
@@ -1724,6 +1774,7 @@ def _bytes_to_mb(bytes_val: int) -> str:
numel = sum(to_local_if_dtensor(p).shape.numel() for p in group.params)
buffers = {
"weight": group.model_weight_buffer,
+ "transpose_weight": group.transpose_weight_buffer,
"main_weight": group.main_weight_buffer,
"grad": group.main_grad_buffer,
}
@@ -1793,12 +1844,18 @@ def _init_each_parameter_group_buffers(self, meta_device_init_fp8_params):
self.weight_alloc = FixedPoolAllocator(
name="fsdp_params", fsdp_param_groups=self.parameter_groups, size=UB_BUFFER_NUM
)
+ self.transpose_weight_alloc = FixedPoolAllocator(
+ name="fsdp_fp8_transpose_params",
+ fsdp_param_groups=self.parameter_groups,
+ size=UB_BUFFER_NUM,
+ )
self.main_grad_alloc = FixedPoolAllocator(
name="fsdp_grads", fsdp_param_groups=self.parameter_groups, size=UB_BUFFER_NUM
)
self.double_buf_units = self.weight_alloc.fsdp_double_buffer_units
else:
self.weight_alloc = StorageResizeBasedBucketAllocator()
+ self.transpose_weight_alloc = StorageResizeBasedBucketAllocator()
self.main_grad_alloc = None
self.double_buf_units = []
@@ -1838,8 +1895,9 @@ def _init_each_parameter_group_buffers(self, meta_device_init_fp8_params):
)
# Check if the parameter group is FP8.
one_param = group.params[0]
- is_dtype_float8 = is_float8tensor(one_param) or meta_device_init_fp8_params.get(
- self.param_to_name[one_param], False
+ is_dtype_float8 = (
+ is_float8tensor(one_param)
+ or meta_device_init_fp8_params.get(self.param_to_name[one_param], (False, False))[0]
)
if is_dtype_float8:
param_dtype = torch.uint8
@@ -1848,6 +1906,16 @@ def _init_each_parameter_group_buffers(self, meta_device_init_fp8_params):
param_dtype = group.params[0].dtype
grad_dtype = param_dtype
+ # Check if the parameter group needs a transpose buffer for model weights.
+ # Currently, only mxfp8 needs it.
+ need_transpose_data = is_float8tensor(one_param) and fp8_need_transpose_data(one_param)
+ need_transpose_data_for_meta_device_init = meta_device_init_fp8_params.get(
+ self.param_to_name[one_param], (False, False)
+ )[1]
+ should_create_transpose_weight_buffer = (
+ need_transpose_data or need_transpose_data_for_meta_device_init
+ )
+
# Check if the parameter group requires a grad buffer or main weight buffer.
should_create_grad_buffer_or_main_weight_buffer = (
not self.only_create_grad_buffer_and_main_weight_buffer_for_param_requires_grad
@@ -1864,13 +1932,29 @@ def _init_each_parameter_group_buffers(self, meta_device_init_fp8_params):
dtype=param_dtype,
device=self.device,
data_parallel_group=main_buf_dp_group,
- is_dtype_float8=is_dtype_float8,
+ is_transpose_buffer=False,
temporary_bucket_allocator=self.weight_alloc,
bucket_id=group_id,
chunk_size_factor=group.chunk_size_factor,
mem_alloc_context=self.mem_alloc_context,
**main_buf_extra_kwargs,
)
+ if should_create_transpose_weight_buffer:
+ group.transpose_weight_buffer = DataParallelBuffer(
+ self.ddp_config,
+ group.params,
+ is_data_distributed=is_model_weight_buffer_distributed
+ and main_buf_dp_group.size() > 1,
+ dtype=param_dtype,
+ device=self.device,
+ data_parallel_group=main_buf_dp_group,
+ is_transpose_buffer=True,
+ temporary_bucket_allocator=self.transpose_weight_alloc,
+ bucket_id=group_id,
+ chunk_size_factor=group.chunk_size_factor,
+ mem_alloc_context=self.mem_alloc_context,
+ **main_buf_extra_kwargs,
+ )
# Initialize the main weight buffer.
if should_create_grad_buffer_or_main_weight_buffer and preserve_fp32_weights:
@@ -1902,7 +1986,7 @@ def _init_each_parameter_group_buffers(self, meta_device_init_fp8_params):
dtype=torch.float32 if grad_reduce_in_fp32 else grad_dtype,
device=self.device,
data_parallel_group=main_buf_dp_group,
- is_dtype_float8=False,
+ is_transpose_buffer=False,
temporary_bucket_allocator=self.main_grad_alloc,
gradient_scaling_factor=gradient_scaling_factor,
bucket_id=group_id,
@@ -1926,7 +2010,7 @@ def _init_each_parameter_group_buffers(self, meta_device_init_fp8_params):
dtype=wbuf.dtype,
device=wbuf.device,
data_parallel_group=hsdp_buf_dp_group,
- is_dtype_float8=wbuf.is_dtype_float8,
+ is_transpose_buffer=False,
temporary_bucket_allocator=self.weight_alloc,
bucket_id=group_id,
chunk_size_factor=group.chunk_size_factor,
@@ -1942,6 +2026,9 @@ def _init_each_parameter_group_buffers(self, meta_device_init_fp8_params):
),
)
+ if group.transpose_weight_buffer is not None:
+ raise NotImplementedError("HSDP for transpose buffer is not implemented yet")
+
if should_create_grad_buffer_or_main_weight_buffer:
# Initialize the HSDP grad buffer.
gbuf = group.main_grad_buffer
@@ -1953,7 +2040,7 @@ def _init_each_parameter_group_buffers(self, meta_device_init_fp8_params):
dtype=gbuf.dtype,
device=gbuf.device,
data_parallel_group=hsdp_buf_dp_group,
- is_dtype_float8=gbuf.is_dtype_float8,
+ is_transpose_buffer=False,
temporary_bucket_allocator=self.main_grad_alloc,
gradient_scaling_factor=gradient_scaling_factor,
bucket_id=group_id,
@@ -2036,6 +2123,20 @@ def _init_each_parameter_group_buffers(self, meta_device_init_fp8_params):
torch.empty(wbuf.data_size, dtype=wbuf.dtype, device=self.device)
)
bucket = wbuf.fetch_bucket()
+
+ tbuf = group.transpose_weight_buffer
+ if tbuf:
+ with self.mem_alloc_context():
+ if group.hsdp_wbuf:
+ raise NotImplementedError(
+ "HSDP for transpose buffer is not implemented yet"
+ )
+ else:
+ tbuf.init_data(
+ torch.empty(tbuf.data_size, dtype=tbuf.dtype, device=self.device)
+ )
+ transpose_bucket = tbuf.fetch_bucket()
+
mbuf = group.main_weight_buffer
if mbuf:
# Manually instantiate an empty tensor into the main weight buffer.
@@ -2089,25 +2190,41 @@ def _init_each_parameter_group_buffers(self, meta_device_init_fp8_params):
if not self.ddp_config.keep_fp8_transpose_cache:
for _param in m.parameters(recurse=False):
if is_float8tensor(_param):
- _param._transpose_invalid = True
- _param._transpose = None
+ fp8_discard_transpose_cache(_param)
# Raise error if a meta parameter still exists after initialization.
assert not p.is_meta, (self.param_to_name[p], module_reset_flag)
+ p_local = to_local_if_dtensor(p)
+
# Copy the model weight parameter tensor into the buffer.
# When distributed, this shards and preserves the data across all ranks.
- wbuf.set_item(item_id, to_local_if_dtensor(p))
+ wbuf.set_item(item_id, p_local)
+ if tbuf:
+ tbuf.set_item(item_id, p_local)
# Retrieve the newly allocated parameter data from the global bucket.
# Attach the bucket-allocated parameter data to the module parameter,
# to use the bucket-allocated data for autograd and NCCL.
- new_param_data = wbuf.get_item_from_bucket(bucket, item_id).view(
- to_local_if_dtensor(p).shape
- )
- if is_float8tensor(p):
- # Needed to instantiate FP8 parameters. Requires installing
- # TransformerEngine.
- modify_underlying_storage(p, new_param_data)
+ new_param_data = wbuf.get_item_from_bucket(bucket, item_id).view(p_local.shape)
+ if tbuf:
+ new_transpose_data = tbuf.get_item_from_bucket(
+ transpose_bucket, item_id
+ ).view(p_local.shape)
+ else:
+ new_transpose_data = None
+
+ if is_float8tensor(p_local):
+ old_param_data = fp8_get_raw_data(p_local)
+ assert old_param_data._base is None
+ new_param_data.detach().copy_(old_param_data)
+ fp8_set_raw_data(p_local, new_param_data)
+ del old_param_data
+ if new_transpose_data is not None:
+ old_transpose_data = fp8_get_raw_data(p_local, True)
+ assert old_transpose_data._base is None
+ new_transpose_data.detach().copy_(old_transpose_data)
+ fp8_set_raw_data(p_local, new_transpose_data, True)
+ del old_transpose_data
elif isinstance(p, DTensor):
old_param_data = p._local_tensor.data
p._local_tensor.data = new_param_data
@@ -2145,7 +2262,12 @@ def _init_each_parameter_group_buffers(self, meta_device_init_fp8_params):
# the (high-precision) main weight buffer.
# Nothing else needs to be done, because the main weights
# do not require autograd operations, only possibly sharding.
- mbuf.set_item(item_id, to_local_if_dtensor(p))
+ p_local = to_local_if_dtensor(p)
+ assert not is_float8tensor(p_local), (
+ self.param_to_name[p],
+ "fp8 param should use get_high_precision_init_val method.",
+ )
+ mbuf.set_item(item_id, p_local)
if wbuf and wbuf.is_data_distributed:
# Free the memory backing the temporarily-allocated bucket associated
@@ -2157,6 +2279,9 @@ def _init_each_parameter_group_buffers(self, meta_device_init_fp8_params):
# before forward activations and gradients are allocated in training.
wbuf.free_bucket_storage()
+ if tbuf and tbuf.is_data_distributed:
+ tbuf.free_bucket_storage()
+
# Allocate the main_weight buffer and main_grad buffer data in one buffer.
if self.buffer_all_in_one:
with self.mem_alloc_context():
@@ -2280,6 +2405,7 @@ def _reset_parameters(self, old_params, new_params):
group.params[item_id] = new_p
for buf in [
group.model_weight_buffer,
+ group.transpose_weight_buffer,
group.main_weight_buffer,
group.main_grad_buffer,
group.hsdp_wbuf,
@@ -2327,6 +2453,7 @@ def _init_distributed_params(self):
dist_main_weight = {}
for pg in self.parameter_groups:
wbuf = pg.model_weight_buffer
+ tbuf = pg.transpose_weight_buffer
mbuf = pg.main_weight_buffer
for item_id, orig_param in enumerate(pg.params):
param_name = self.param_to_name[orig_param]
@@ -2354,6 +2481,7 @@ def _init_distributed_params(self):
)
dist_main_weight[param_name] = dist_param
elif wbuf:
+ assert tbuf is None, "Transpose buffer should only exist when main params exist"
dist_param = make_fsdp_dtensor(
local_tensor=wbuf.get_item(item_id, only_shard=sharded_optimizer_state),
param=orig_param,
@@ -2473,8 +2601,9 @@ def update_main_grads(self):
item_id, only_shard=sharded_optimizer_state
)
if group.main_weight_buffer is not None:
- # Convert the gradient to the main weight buffer dtype.
- optimizer_grad = optimizer_grad.to(param.dtype)
+ if not getattr(self, "use_precision_aware_optimizer", False):
+ # Convert the gradient to the main weight buffer dtype.
+ optimizer_grad = optimizer_grad.to(param.dtype)
if name not in self.dist_main_grad:
# Register the gradient as a distributed tensor.
@@ -2522,9 +2651,54 @@ def copy_main_weights_to_model_weights(self):
expert_param_quantize_kwargs = copy.deepcopy(dense_param_quantize_kwargs)
data_parallel_group = None
expert_data_parallel_group = None
+ clear_quantize_kwargs = lambda kwargs: [d.clear() for d in kwargs.values()]
+
+ def _fp8_quantize_params(dense_param_quantize_kwargs, expert_param_quantize_kwargs):
+ if len(dense_param_quantize_kwargs["model_params"]) > 0:
+ # If we have FP8 parameters, we need to quantize them.
+ fp8_quantize(data_parallel_group=data_parallel_group, **dense_param_quantize_kwargs)
+
+ if len(expert_param_quantize_kwargs["model_params"]) > 0:
+ # If we have FP8 expert parameters, we need to quantize them.
+ fp8_quantize(
+ data_parallel_group=expert_data_parallel_group, **expert_param_quantize_kwargs
+ )
+
+ clear_quantize_kwargs(dense_param_quantize_kwargs)
+ clear_quantize_kwargs(expert_param_quantize_kwargs)
+
+ # Special handling of blockwise FP8
+ BATCH_QUANT_MEMORY_LIMIT_BYTES = 5 * 1024**3 # 5 GB
+ blockwise_fp8_weight_buffers = []
+ blockwise_fp8_param_buffers = []
+
+ def _batch_quantize_blockwise_fp8_params(
+ dense_param_quantize_kwargs, expert_param_quantize_kwargs, blockwise_fp8_param_buffers
+ ):
+ if len(blockwise_fp8_param_buffers) == 0:
+ return
+
+ # Copy original param shards into their blockwise FP8 working buffers
+ for bufs in blockwise_fp8_param_buffers:
+ bufs["bucket_param"].copy_(bufs["param"])
+
+ # Apply FP8 quantization to blockwise FP8 parameters
+ _fp8_quantize_params(dense_param_quantize_kwargs, expert_param_quantize_kwargs)
+
+ # Copy quantized params back from working buffers to original param tensors
+ for bufs in blockwise_fp8_param_buffers:
+ bufs["param"].copy_(bufs["bucket_param"])
+ blockwise_fp8_param_buffers.clear()
+
+ # Free bucket storage for blockwise FP8 weight buffers
+ for wbuf in blockwise_fp8_weight_buffers:
+ wbuf.free_bucket_storage()
+ blockwise_fp8_weight_buffers.clear()
+
for pg in self.parameter_groups:
mbuf = pg.main_weight_buffer
wbuf = pg.model_weight_buffer
+ tbuf = pg.transpose_weight_buffer
if mbuf is None:
continue
@@ -2540,44 +2714,88 @@ def copy_main_weights_to_model_weights(self):
shard_offsets_in_fp8 = quantize_func_kwargs["start_offsets"]
shard_model_params = quantize_func_kwargs["fsdp_shard_model_params"]
+ has_blockwise_fp8_param = False
for param in pg.params:
item_id = mbuf.param_idx[param]
if wbuf:
if wbuf.is_data_distributed or mbuf.is_data_distributed:
model_param = wbuf.get_item(item_id, only_shard=True)
+ if tbuf:
+ transpose_param = tbuf.get_item(item_id, only_shard=True)
+ else:
+ transpose_param = None
main_weight = mbuf.get_item(item_id, only_shard=True)
else:
model_param = wbuf.get_item(item_id)
+ if tbuf:
+ transpose_param = tbuf.get_item(item_id)
+ else:
+ transpose_param = None
main_weight = mbuf.get_item(item_id)
else:
assert not mbuf.is_data_distributed
model_param = to_local_if_dtensor(param)
main_weight = mbuf.get_item(item_id)
+ if is_blockwise_float8tensor(param):
+ fp8_params.append(param)
+ if model_param.numel() == 0:
+ shard_fp32_from_fp8.append(None)
+ shard_offsets_in_fp8.append(None)
+ shard_model_params.append([None, None])
+ else:
+ shard_fp32_from_fp8.append(main_weight)
+ shard_offsets_in_fp8.append(wbuf.locate_item_in_global_item(item_id)[0])
+ bucket = wbuf.fetch_bucket()
+ b_model_param = wbuf.get_item_from_bucket(bucket, item_id)[
+ slice(*wbuf.locate_item_in_global_item(item_id))
+ ]
+ assert (
+ transpose_param is None
+ ), "Blockwise FP8 does not support transpose param."
+ shard_model_params.append([b_model_param, None])
+ assert b_model_param.numel() == model_param.numel(), (
+ f"Blockwise FP8 bucket param numel {b_model_param.numel()} does"
+ f" not match model param numel {model_param.numel()}"
+ f" name: {self.param_to_name[param]}"
+ )
+ blockwise_fp8_param_buffers.append(
+ {"bucket_param": b_model_param, "param": model_param}
+ )
+ has_blockwise_fp8_param = True
+ continue
+
if is_float8tensor(param):
fp8_params.append(param)
if model_param.numel() == 0:
shard_fp32_from_fp8.append(None)
shard_offsets_in_fp8.append(None)
- shard_model_params.append(None)
+ shard_model_params.append([None, None])
else:
shard_fp32_from_fp8.append(main_weight)
shard_offsets_in_fp8.append(wbuf.locate_item_in_global_item(item_id)[0])
- shard_model_params.append(model_param)
+ shard_model_params.append([model_param, transpose_param])
continue
if model_param.numel() > 0:
model_param.data.copy_(main_weight.view(model_param.shape))
- if len(dense_param_quantize_kwargs["model_params"]) > 0:
- # If we have FP8 parameters, we need to quantize them.
- dense_param_quantize_kwargs["data_parallel_group"] = data_parallel_group
- quantize_param_shard(**dense_param_quantize_kwargs)
+ if has_blockwise_fp8_param:
+ blockwise_fp8_weight_buffers.append(wbuf)
+ if (
+ sum([wbuf.bucket_index.size for wbuf in blockwise_fp8_weight_buffers])
+ > BATCH_QUANT_MEMORY_LIMIT_BYTES
+ ):
+ _batch_quantize_blockwise_fp8_params(
+ dense_param_quantize_kwargs,
+ expert_param_quantize_kwargs,
+ blockwise_fp8_param_buffers,
+ )
- if len(expert_param_quantize_kwargs["model_params"]) > 0:
- # If we have FP8 expert parameters, we need to quantize them.
- expert_param_quantize_kwargs["data_parallel_group"] = expert_data_parallel_group
- quantize_param_shard(**expert_param_quantize_kwargs)
+ _batch_quantize_blockwise_fp8_params(
+ dense_param_quantize_kwargs, expert_param_quantize_kwargs, blockwise_fp8_param_buffers
+ )
+ _fp8_quantize_params(dense_param_quantize_kwargs, expert_param_quantize_kwargs)
@torch.no_grad()
def copy_model_weights_to_main_weights(self):
@@ -2595,6 +2813,7 @@ def copy_model_weights_to_main_weights(self):
f"Master weight buffer size {mbuf.data.numel()} does not match "
f"model weight buffer size {copyin_data.numel()}"
)
+ # TODO(mxfp8): Make sure it's not a fp8 buf?
mbuf.data.copy_(copyin_data.data)
def all_gather_parameters(self, async_op: bool = True):
@@ -2612,15 +2831,18 @@ def all_gather_parameters(self, async_op: bool = True):
all_gather_ops = []
for g in self.parameter_groups:
- shard = g.model_weight_buffer.get_shard_from_local_buffer()
- all_gather_handler = torch.distributed.all_gather_into_tensor(
- output_tensor=g.model_weight_buffer.data,
- input_tensor=shard,
- group=g.model_weight_buffer.data_parallel_group,
- async_op=async_op,
- )
- if async_op:
- all_gather_ops.append(all_gather_handler)
+ for buf in [g.model_weight_buffer, g.transpose_weight_buffer]:
+ if buf is None:
+ continue
+ shard = buf.get_shard_from_local_buffer()
+ all_gather_handler = torch.distributed.all_gather_into_tensor(
+ output_tensor=buf.data,
+ input_tensor=shard,
+ group=buf.data_parallel_group,
+ async_op=async_op,
+ )
+ if async_op:
+ all_gather_ops.append(all_gather_handler)
for op in all_gather_ops:
op.wait()
@@ -2641,7 +2863,7 @@ def reduce_scatter_gradients(self, async_op: bool = True):
reduce_scatter_ops = []
for g in self.parameter_groups:
gbuf = g.main_grad_buffer
- if gbuf is not None:
+ if gbuf is None:
continue
scaling_factor = gbuf.gradient_scaling_factor
reduce_op = gradient_reduce_preprocessing(gbuf.data, scaling_factor, self.ddp_config)
@@ -3091,9 +3313,16 @@ def __init__(
# Track the status of all-gather operations for each bucket.
self.param_gather_event_map = {}
# All buckets are initially deallocated / empty after initialization of ParamAndGradBuffer.
- self.bucket_status = {i: BucketStatus.EMPTY for i in range(self.buffer.num_buckets)}
+ self.bucket_status = {}
+ for i in range(self.buffer.num_buckets):
+ for bwd in [False, True]:
+ self.bucket_status[self.get_bucket_key(i, bwd)] = BucketStatus.EMPTY
+
# Track whether each bucket can be deallocated.
- self.bucket_can_be_released = {i: False for i in range(self.buffer.num_buckets)}
+ self.bucket_can_be_released = {}
+ for i in range(self.buffer.num_buckets):
+ for bwd in [False, True]:
+ self.bucket_can_be_released[self.get_bucket_key(i, bwd)] = False
# Map each bucket to the bucket group it belongs to by enumerated ID.
# Made to collect a subset of buckets in the same bucket group.
@@ -3118,6 +3347,13 @@ def __init__(
# all-gather parameters across groups.
self.outer_fsdp_group_param_gather_stream = torch.cuda.Stream()
+ def get_bucket_key(self, bucket_id, bwd):
+ """Get the key for the bucket."""
+ has_transpose_buffer = (
+ self.buffer.parameter_groups[bucket_id].transpose_weight_buffer is not None
+ )
+ return (bucket_id, has_transpose_buffer and bwd)
+
@property
def num_buckets(self):
"""Return the number of buckets."""
@@ -3134,10 +3370,11 @@ def reset(self):
UserWarning,
)
while len(self.param_gather_event_map) > 0:
- bucket_id = next(iter(self.param_gather_event_map))
- self.wait_bucket_ready(bucket_id)
+ (bucket_id, bwd) = next(iter(self.param_gather_event_map))
+ self.wait_bucket_ready(bucket_id, bwd)
for bucket_id in range(self.num_buckets):
- self.bucket_can_be_released[bucket_id] = True
+ for bwd in [False, True]:
+ self.bucket_can_be_released[self.get_bucket_key(bucket_id, bwd)] = True
self.recycle_unused_buckets()
assert all([status is BucketStatus.EMPTY for status in self.bucket_status.values()]), (
@@ -3159,6 +3396,7 @@ def all_gather_params(
suggested_AG_prefetch_size: Optional[int] = None,
async_param_gather: bool = True,
outer_fsdp_group_param_gather: bool = False,
+ bwd: bool = False,
):
"""All-gather the params. If prefetch is enabled, prefetch next buckets
in the order of `prefetch_order`.
@@ -3193,7 +3431,7 @@ def all_gather_params(
# Do not release the buckets that are being all-gathered.
for bucket_id in ag_buckets:
- self.bucket_can_be_released[bucket_id] = False
+ self.bucket_can_be_released[self.get_bucket_key(bucket_id, bwd)] = False
# If prefetch is enabled, we will add prefetch buckets to ag_buckets.
if prefetch:
@@ -3265,7 +3503,11 @@ def need_skip_prefetch(bucket_id):
bucket_id = next_bucket_id(ag_buckets)
# Only all-gather on buckets that have not been allocated yet.
- ag_buckets = [i for i in ag_buckets if self.bucket_status[i] == BucketStatus.EMPTY]
+ ag_buckets = [
+ bucket_id
+ for bucket_id in ag_buckets
+ if self.bucket_status[self.get_bucket_key(bucket_id, bwd)] == BucketStatus.EMPTY
+ ]
if len(ag_buckets) == 0:
return
@@ -3284,6 +3526,7 @@ def need_skip_prefetch(bucket_id):
self.ag_stream if self.ag_stream is not None else torch.cuda.current_stream()
)
if outer_fsdp_group_param_gather:
+ # TODO(mxfp8): Support hsdp
self.outer_fsdp_group_param_gather_stream.wait_stream(torch.cuda.current_stream())
with torch.cuda.stream(self.outer_fsdp_group_param_gather_stream):
outer_fsdp_group = self.buffer.dist_index.get_outer_fsdp_group()
@@ -3311,12 +3554,13 @@ def need_skip_prefetch(bucket_id):
for bucket_id in buckets:
# All-gather the module weights from each FSDP buffer shard
# into an allocated bucket containing unsharded weights.
- self.async_bucket_gather(bucket_id)
+ self.async_bucket_gather(bucket_id, bwd)
# Replace the parameter all-gather event with coalescing event.
for bucket_id in buckets:
- _, mark_bucket_ready_to_use = self.param_gather_event_map[bucket_id]
- self.param_gather_event_map[bucket_id] = (
+ bucket_key = self.get_bucket_key(bucket_id, bwd)
+ _, mark_bucket_ready_to_use = self.param_gather_event_map[bucket_key]
+ self.param_gather_event_map[bucket_key] = (
coalescing_event,
mark_bucket_ready_to_use,
)
@@ -3324,14 +3568,16 @@ def need_skip_prefetch(bucket_id):
# Wait for all-gather to finish
if not async_param_gather:
for bucket_id in buckets:
- self.wait_bucket_ready(bucket_id)
+ self.wait_bucket_ready(bucket_id, bwd)
- def wait_bucket_ready(self, bucket_id, empty_ok=False):
+ def wait_bucket_ready(self, bucket_id, bwd, empty_ok=False):
"""Wait for the bucket to be ready."""
- if self.bucket_status[bucket_id] == BucketStatus.READY_TO_USE:
+ bucket_key = self.get_bucket_key(bucket_id, bwd)
+
+ if self.bucket_status[bucket_key] == BucketStatus.READY_TO_USE:
# Already ready to use.
return
- if self.bucket_status[bucket_id] == BucketStatus.EMPTY:
+ if self.bucket_status[bucket_key] == BucketStatus.EMPTY:
if empty_ok:
return
# Bucket shouldn't be empty, this implies that the bucket
@@ -3339,48 +3585,64 @@ def wait_bucket_ready(self, bucket_id, empty_ok=False):
raise ValueError(f"Bucket {bucket_id} is empty.")
# Wait for asynchronous / overlapped NCCL operations to complete.
- param_gather_event, mark_bucket_ready_to_use = self.param_gather_event_map.pop(bucket_id)
+ param_gather_event, mark_bucket_ready_to_use = self.param_gather_event_map.pop(bucket_key)
param_gather_event.wait()
mark_bucket_ready_to_use()
@torch.no_grad()
- def release_bucket(self, bucket_id: int):
+ def release_bucket(self, bucket_id, bwd):
"""Release the bucket."""
- if self.bucket_status[bucket_id] == BucketStatus.EMPTY:
+ # TODO(mxfp8): In some cases, there won't be ag before bwd?
+ bucket_key = self.get_bucket_key(bucket_id, bwd)
+
+ if self.bucket_status[bucket_key] == BucketStatus.EMPTY:
return
- self.wait_bucket_ready(bucket_id, empty_ok=True)
- if self.bucket_status[bucket_id] == BucketStatus.COMMUNICATING:
+ self.wait_bucket_ready(bucket_id, bwd, empty_ok=True)
+ if self.bucket_status[bucket_key] == BucketStatus.COMMUNICATING:
raise ValueError(f"Bucket {bucket_id} is communicating.")
- wbuf = self.buffer.parameter_groups[bucket_id].model_weight_buffer
- wbuf.free_bucket_storage()
- self.bucket_status[bucket_id] = BucketStatus.EMPTY
+ if bwd and self.buffer.parameter_groups[bucket_id].transpose_weight_buffer is not None:
+ buf = self.buffer.parameter_groups[bucket_id].transpose_weight_buffer
+ else:
+ buf = self.buffer.parameter_groups[bucket_id].model_weight_buffer
+
+ buf.free_bucket_storage()
+ self.bucket_status[bucket_key] = BucketStatus.EMPTY
def recycle_unused_buckets(self):
"""Recycle the unused buckets."""
- for bucket_id, can_be_released in self.bucket_can_be_released.items():
+ for bucket_key, can_be_released in self.bucket_can_be_released.items():
if can_be_released:
- self.release_bucket(bucket_id)
- self.bucket_can_be_released[bucket_id] = False
+ bucket_id, is_transpose_weight = bucket_key[0], bucket_key[1]
+ self.release_bucket(bucket_id, is_transpose_weight)
+ self.bucket_can_be_released[bucket_key] = False
- def get_fsdp_buffer(self, bucket_id: int) -> DataParallelBuffer:
+ def get_fsdp_buffer(self, bucket_id: int, bwd=False) -> DataParallelBuffer:
"""Get the FSDP buffer with the given bucket ID."""
param_group = self.buffer.parameter_groups[bucket_id]
if self.buffer.ddp_config.outer_dp_sharding_strategy != "no_shard":
- return param_group.hsdp_wbuf
- return param_group.model_weight_buffer
+ if bwd and param_group.transpose_weight_buffer is not None:
+ raise RuntimeError("Transpose buffer is not supported for HSDP")
+ else:
+ return param_group.hsdp_wbuf
+ if bwd and param_group.transpose_weight_buffer is not None:
+ return param_group.transpose_weight_buffer
+ else:
+ return param_group.model_weight_buffer
@torch.no_grad()
- def async_bucket_gather(self, bucket_id: int) -> None:
+ def async_bucket_gather(self, bucket_id, bwd) -> None:
"""All-gather the bucket and set the items."""
- self.bucket_can_be_released[bucket_id] = False
- if self.bucket_status[bucket_id] != BucketStatus.EMPTY:
+ bucket_key = self.get_bucket_key(bucket_id, bwd)
+
+ self.bucket_can_be_released[bucket_key] = False
+ if self.bucket_status[bucket_key] != BucketStatus.EMPTY:
return
- self.bucket_status[bucket_id] = BucketStatus.COMMUNICATING
+ self.bucket_status[bucket_key] = BucketStatus.COMMUNICATING
- wbuf = self.get_fsdp_buffer(bucket_id)
+ wbuf = self.get_fsdp_buffer(bucket_id, bwd)
# Lazy release the unused buckets.
self.recycle_unused_buckets()
@@ -3395,18 +3657,21 @@ def async_bucket_gather(self, bucket_id: int) -> None:
async_op=True,
)
- def get_closure(bucket_id):
+ def get_closure(bucket_id, bwd):
@torch.no_grad()
def mark_bucket_ready_to_use():
# Mark the bucket as ready to use - all NCCL operations are complete.
- self.bucket_status[bucket_id] = BucketStatus.READY_TO_USE
+ self.bucket_status[self.get_bucket_key(bucket_id, bwd)] = BucketStatus.READY_TO_USE
return mark_bucket_ready_to_use
- mark_bucket_ready_to_use = get_closure(bucket_id)
+ mark_bucket_ready_to_use = get_closure(bucket_id, bwd)
# Track the async all-gather operation for the bucket.
- self.param_gather_event_map[bucket_id] = (param_gather_event, mark_bucket_ready_to_use)
+ self.param_gather_event_map[self.get_bucket_key(bucket_id, bwd)] = (
+ param_gather_event,
+ mark_bucket_ready_to_use,
+ )
@torch.no_grad()
@@ -3499,15 +3764,13 @@ def override_sharded_param_methods_with_safety_checks(params, all_gather_pipelin
def override_sharded_param_to_function_closure(p, to_function):
def override_sharded_param_to_function(*args, **kwargs):
- bucket_id = all_gather_pipeline.buffer.param_to_param_group[p]
- status = all_gather_pipeline.bucket_status[bucket_id]
- if status == BucketStatus.READY_TO_USE:
- return to_function(*args, **kwargs)
- raise RuntimeError(
- "This parameter is already shard by MCore FSDP and the "
- "shared-state parameter does not support 'to' function."
- "please define the dtype and device of the parameter before FSDP wrap."
- )
+ if p._typed_storage()._size() == 0:
+ warnings.warn(
+ "The parameter may be sharded by Megatron-FSDP, "
+ "no actual 'to' operation is performed."
+ )
+ return torch.empty([])
+ return to_function(*args, **kwargs)
return override_sharded_param_to_function
@@ -3515,15 +3778,13 @@ def override_sharded_param_to_function(*args, **kwargs):
def override_sharded_param_cpu_function_closure(p, cpu_function):
def override_sharded_param_cpu_function(*args, **kwargs):
- bucket_id = all_gather_pipeline.buffer.param_to_param_group[p]
- status = all_gather_pipeline.bucket_status[bucket_id]
- if status == BucketStatus.READY_TO_USE:
- return cpu_function(*args, **kwargs)
- warnings.warn(
- "The parameters are sharded by MCore FSDP, and no actual cpu "
- "operation is performed."
- )
- return torch.empty([], device="cpu")
+ if p._typed_storage()._size() == 0:
+ warnings.warn(
+ "The parameter may be sharded by Megatron-FSDP, "
+ "no actual 'cpu' operation is performed."
+ )
+ return torch.empty([], device="cpu")
+ return cpu_function(*args, **kwargs)
return override_sharded_param_cpu_function
diff --git a/megatron/core/distributed/fsdp/src/megatron_fsdp/utils.py b/megatron/core/distributed/fsdp/src/megatron_fsdp/utils.py
index 0008f1ab59a..01523929ae1 100644
--- a/megatron/core/distributed/fsdp/src/megatron_fsdp/utils.py
+++ b/megatron/core/distributed/fsdp/src/megatron_fsdp/utils.py
@@ -19,7 +19,7 @@
from contextlib import nullcontext
from functools import reduce
from importlib.metadata import version
-from typing import Callable, List, Optional, Sequence, Union
+from typing import Callable, Optional, Sequence, Union
try:
import einops
@@ -78,52 +78,6 @@ def is_te_min_version(vers, check_equality=True):
return te_version > PkgVersion(vers)
-# Check if Transformer Engine has class for fp8 tensors.
-try:
- if is_te_min_version("2.0"):
- # In TE2.x, QuantizedTensor is the base class for all different type of fp8 tensors,
- # including fp8 tensor for delayed scaling, current scaling and mxfp8, etc.
- from transformer_engine.pytorch.tensor import QuantizedTensor as FP8_TENSOR_CLASS
- else:
- from transformer_engine.pytorch.float8_tensor import Float8Tensor as FP8_TENSOR_CLASS
-
- HAVE_TE_FP8_TENSOR_CLASS = True
-except (ImportError, ModuleNotFoundError):
- # FP8 tensor class not found
- HAVE_TE_FP8_TENSOR_CLASS = False
-
-try:
- from transformer_engine.pytorch.optimizers import multi_tensor_applier, multi_tensor_scale
-
- multi_tensor_scale_impl = multi_tensor_scale
-except ImportError:
- try:
- import amp_C
- from apex.multi_tensor_apply import multi_tensor_applier
-
- multi_tensor_scale_impl = amp_C.multi_tensor_scale
- except ImportError:
- import warnings
-
- warnings.warn(
- "Transformer Engine and Apex are not installed. "
- "Falling back to local implementations of "
- "multi_tensor_applier and multi_tensor_scale"
- )
-
- def local_multi_tensor_applier(op, noop_flag_buffer, tensor_lists, *args):
- """Multi tensor op applier"""
- return op(2048 * 32, noop_flag_buffer, tensor_lists, *args)
-
- def local_multi_tensor_scale(chunk_size, noop_flag, tensor_lists, scale):
- """Works as a drop-in replacement for amp_C.multi_tensor_scale."""
- for src, dst in zip(tensor_lists[0], tensor_lists[1]):
- dst.copy_(src * scale)
-
- multi_tensor_applier = local_multi_tensor_applier
- multi_tensor_scale_impl = local_multi_tensor_scale
-
-
def is_submodule(module, parent_module, strict=True):
"""
Check if a module is a submodule of another module.
@@ -137,18 +91,6 @@ def is_submodule(module, parent_module, strict=True):
return False
-def is_float8tensor(tensor: torch.Tensor) -> bool:
- """Check if a tensor is a Transformer Engine Float8Tensor.
-
- Note that in TE2.x, in order to support more recipes, the design of the fp8 tensor class has
- changed. Now Float8Tensor is only used for current scaling and delayed scaling. And mxfp8
- and blockwise scaling have their own fp8 tensor classes. These different fp8 tensor classes
- are both inherited from QuantizedTensor. So, for TE1.x, FP8_TENSOR_CLASS is Float8Tensor,
- and for TE2.x, FP8_TENSOR_CLASS is QuantizedTensor.
- """
- return HAVE_TE_FP8_TENSOR_CLASS and isinstance(tensor, FP8_TENSOR_CLASS)
-
-
def get_mesh_names(
device_mesh: Optional[DeviceMesh] = None, only_submesh_dims: bool = False
) -> list[str]:
@@ -210,198 +152,6 @@ def contains_submesh(
return all(submesh_name in device_mesh_names for submesh_name in submesh_names)
-def _multi_tensor_copy_this_to_that(
- this: List[torch.Tensor], that: List[torch.Tensor], overflow_buf: Optional[torch.Tensor] = None
-):
- """
- Use multi-tensor-applier to copy values from one list to another.
- We don't have a bfloat16 implementation so for now if the overflow_buf
- is not provided, we default back to simple loop copy to be compatible
- with bfloat16.
- """
- if overflow_buf is not None:
- overflow_buf.fill_(0)
- # Scaling with factor `1.0` is equivalent to copy.
- multi_tensor_applier(multi_tensor_scale_impl, overflow_buf, [this, that], 1.0)
- else:
- for this_, that_ in zip(this, that):
- that_.copy_(this_)
-
-
-"""
-The code below abstracts the functionalities needed for implementing "--fp8-param-gather" into
-several functions. It provides different implementations for each function based on different
-versions of TE, ensuring compatibility across various TE versions.
-
-Currently, there are three functions:
- - modify_underlying_storage
- This function is used in DDP to place all parameters into a contiguous buffer. For
- non-fp8 tensors, replacing their data is simple, just using code like
- "tensor.data = new_data". However, for fp8 tensors, their raw data is not stored in the
- ".data" attribute, and it varies with different TE versions and different recipes. This
- function provides a unified interface to replace the underlying storage of a fp8 tensor.
- - quantize_param_shard
- This function is used in dist-opt to cast fp32 main params to fp8 params. For non-fp8
- params, this casting is as simple as "bf16_params.copy_(fp32_main_params)"; but for fp8
- params, the casting logic varies with different TE versions and different recipes. This
- function provides a unified interface to cast fp32 main params to fp8 params, and also
- updates the necessary attributes (like amax, scale, scale_inv or transpose cache) of the
- fp8 model params.
- - correct_amax_history_if_needed
- This function is used to correct the amax history of fp8 tensors. In TE1.x, some inplace
- copy operations will write unwanted values to the amax_history of fp8 tensors. This function
- corrects the amax_history back. For TE2.x, it's an empty function.
- Only useful for delayed scaling.
-"""
-if HAVE_TE and is_te_min_version("2.2"):
- # Supported TE versions: 2.2+
- from transformer_engine.pytorch.tensor import QuantizedTensor
-
- def _modify_underlying_storage_impl(
- fp8_tensor: QuantizedTensor, new_raw_data: torch.Tensor
- ) -> None:
- from transformer_engine.pytorch.tensor.utils import replace_raw_data
-
- replace_raw_data(fp8_tensor, new_raw_data)
-
- def _quantize_param_shard_impl(
- model_params: List[QuantizedTensor],
- main_params: List[torch.Tensor],
- start_offsets: List[int],
- data_parallel_group: ProcessGroup,
- fsdp_shard_model_params: Optional[List[torch.Tensor]] = None,
- ) -> None:
- if len(model_params) == 0:
- return
-
- from transformer_engine.pytorch.tensor.utils import cast_master_weights_to_fp8
-
- args = [model_params, main_params, start_offsets, data_parallel_group]
- if fsdp_shard_model_params is not None:
- if get_te_version() == PkgVersion("2.3.0.dev0+5fdd7bb") or is_te_min_version("2.3.0"):
- args.append(fsdp_shard_model_params)
- else:
- raise NotImplementedError(
- f"FSDP with --fp8-param-gather is not supported in TE v{get_te_version()}"
- )
- cast_master_weights_to_fp8(*args)
-
-elif HAVE_TE and is_te_min_version("2.0"):
- # Supported TE versions: 2.0
- from transformer_engine.pytorch.tensor import QuantizedTensor
- from transformer_engine.pytorch.tensor.float8_tensor import Float8Tensor
-
- def _modify_underlying_storage_impl(
- fp8_tensor: QuantizedTensor, new_raw_data: torch.Tensor
- ) -> None:
- old_raw_data = fp8_tensor._data
- assert old_raw_data.dtype == new_raw_data.dtype
- new_raw_data.detach().copy_(old_raw_data)
- fp8_tensor._data = new_raw_data
- del old_raw_data
-
- def _quantize_param_shard_impl(
- model_params: List[QuantizedTensor],
- main_params: List[torch.Tensor],
- start_offsets: List[int],
- data_parallel_group: ProcessGroup,
- fsdp_shard_model_params: Optional[List[torch.Tensor]] = None,
- ) -> None:
- if len(model_params) == 0:
- return
-
- if fsdp_shard_model_params is None:
- fsdp_shard_model_params = [None] * len(model_params)
-
- for model_param, main_param, start_offset, fsdp_shard_model_param in zip(
- model_params, main_params, start_offsets, fsdp_shard_model_params
- ):
- if main_param is None:
- continue
-
- if fsdp_shard_model_param is not None:
- shard_model_param = fsdp_shard_model_param
- else:
- shard_model_param = model_param._data.view(-1)[
- start_offset : start_offset + main_param.numel()
- ]
-
- quantizer = model_param._quantizer
- # When not using --fp8-param-gather, the main_param (fp32) is first cast to bf16/fp16,
- # and then cast to fp8 during forward.
- # Although it's not necessary when --fp8-param-gather is enabled, we still keep this
- # logic to keep numerical consistency. So here cast the main_param to model_param.dtype.
- main_param = main_param.to(model_param.dtype)
- out = Float8Tensor(
- shape=main_param.size(),
- dtype=model_param.dtype,
- requires_grad=False,
- data=shard_model_param,
- fp8_scale_inv=model_param._scale_inv,
- fp8_dtype=model_param._fp8_dtype,
- quantizer=quantizer,
- )
- quantizer.update_quantized(main_param, out)
-
- amaxes = []
- scales = []
- scale_invs = []
- for model_param in model_params:
- quantizer = model_param._quantizer
- amaxes.append(quantizer.amax.view(1))
- scales.append(quantizer.scale.view(1))
- scale_invs.append(model_param._scale_inv.view(1))
- model_param._reset_caches()
-
- dummy_overflow_buf = torch.tensor([0], dtype=torch.int, device="cuda")
-
- # Update scaling factors.
- packed_scales = torch.empty(len(scales), dtype=torch.float32, device=scales[0].device)
- packed_scale_views = [packed_scales[i].view(1) for i in range(len(scales))]
- _multi_tensor_copy_this_to_that(scales, packed_scale_views, dummy_overflow_buf)
- torch.reciprocal(packed_scales, out=packed_scales)
- _multi_tensor_copy_this_to_that(packed_scale_views, scale_invs, dummy_overflow_buf)
-
- # Reduce amaxes.
- # Note: Assume each param has a separate amax.
- packed_amaxes = torch.empty(len(amaxes), dtype=torch.float32, device=amaxes[0].device)
- packed_amax_views = [packed_amaxes[i].view(1) for i in range(len(amaxes))]
- _multi_tensor_copy_this_to_that(amaxes, packed_amax_views, dummy_overflow_buf)
- torch.distributed.all_reduce(
- packed_amaxes, op=torch.distributed.ReduceOp.MAX, group=data_parallel_group
- )
- _multi_tensor_copy_this_to_that(packed_amax_views, amaxes, dummy_overflow_buf)
-
-else:
- # Fallback impl if TE version is invalid or TE is not installed.
- def _modify_underlying_storage_impl(*args, **kwargs):
- raise RuntimeError(
- "Invalid Transformer Engine version for FP8 distributed optimizer, "
- "please install Transformer Engine 2.0+ or install Megatron-Core"
- )
-
- def _quantize_param_shard_impl(*args, **kwargs):
- raise RuntimeError(
- "Invalid Transformer Engine version for FP8 distributed optimizer, "
- "please install Transformer Engine 2.0+ or install Megatron-Core"
- )
-
-
-def modify_underlying_storage(tensor: torch.Tensor, new_raw_data: torch.Tensor):
- """Replace the underlying raw data of a tensor with new data."""
- _modify_underlying_storage_impl(tensor, new_raw_data)
-
-
-def quantize_param_shard(
- model_params, main_params, start_offsets, data_parallel_group, fsdp_shard_model_params=None
-):
- """Cast shard fp32 main params to fp8 model params."""
- assert HAVE_TE, "Transformer Engine is required for quantizing parameters."
- _quantize_param_shard_impl(
- model_params, main_params, start_offsets, data_parallel_group, fsdp_shard_model_params
- )
-
-
def _get_cuda_rng_state(
device: Union[int, str, torch.device] = "cuda", clone: bool = False, graph_safe: bool = False
) -> torch.Tensor:
@@ -789,6 +539,8 @@ def register_submesh(device_mesh, submesh, is_expert_parallel):
# Register EP submeshes
if self.expt_device_mesh is not None:
+ register_submesh(self.device_mesh, hsdp_submesh, True)
+ register_submesh(self.device_mesh, hsdp_tp_submesh, True)
register_submesh(self.expt_device_mesh, tp_submesh, True)
register_submesh(self.expt_device_mesh, fsdp_tp_submesh, True)
register_submesh(self.expt_device_mesh, fsdp_submesh, True)
diff --git a/megatron/core/extensions/transformer_engine.py b/megatron/core/extensions/transformer_engine.py
index 981cbacb809..d823e42b0bc 100644
--- a/megatron/core/extensions/transformer_engine.py
+++ b/megatron/core/extensions/transformer_engine.py
@@ -1,4 +1,4 @@
-# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved.
+# Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
import dataclasses
import enum
@@ -24,9 +24,6 @@
from megatron.core.parallel_state import (
get_amax_reduction_group,
get_context_parallel_group,
- get_expert_data_parallel_rank,
- get_expert_model_parallel_rank,
- get_expert_model_parallel_world_size,
get_hierarchical_context_parallel_groups,
get_tensor_model_parallel_group,
get_tensor_model_parallel_world_size,
@@ -523,6 +520,7 @@ def __init__(
extra_kwargs["delay_wgrad_compute"] = self.config.delay_wgrad_compute
else:
raise RuntimeError("Only TE with version >=2.3.0 supports delay_wgrad_compute now.")
+
if (
self.config.tp_comm_overlap
and tp_comm_buffer_name
@@ -594,9 +592,10 @@ def __init__(
extra_kwargs["rng_tracker_name"] = rng_tracker_name
te_parallel_mode = parallel_mode
+ tp_group_for_te = tp_group
if parallel_mode == "duplicated":
# Handle non-parallel case
- tp_group = None
+ tp_group_for_te = None
tp_size = 1
explicit_expert_comm = False
te_parallel_mode = None
@@ -611,7 +610,7 @@ def __init__(
input_size = divide(input_size, tp_size)
te_parallel_mode = None
tp_size = 1
- tp_group = None
+ tp_group_for_te = None
super().__init__(
in_features=input_size,
@@ -619,7 +618,7 @@ def __init__(
sequence_parallel=self.config.sequence_parallel,
fuse_wgrad_accumulation=self.config.gradient_accumulation_fusion,
# Pass None if not initialized for backward compatibility with the ckpt converter.
- tp_group=tp_group if torch.distributed.is_initialized() else None,
+ tp_group=tp_group_for_te if torch.distributed.is_initialized() else None,
tp_size=tp_size,
get_rng_state_tracker=(
get_cuda_rng_tracker if get_cuda_rng_tracker().is_initialized() else None
@@ -1269,6 +1268,7 @@ def __init__(
self.kept_packed_seq_params = set(
field.name for field in dataclasses.fields(PackedSeqParams)
)
+
if get_te_version() < PkgVersion("1.3.0"):
# TE 1.3.0 introduces precomputing max_seqlen to remove unnecessary kernels and D2H
# copies (#555)
@@ -1320,6 +1320,26 @@ def forward(
num_splits: Optional[int] = None,
):
"""Forward."""
+ if packed_seq_params is not None:
+ # If Dynamic CP group is provided, update TE DPA CP group
+ if packed_seq_params.cp_group is not None:
+ self.cp_group = packed_seq_params.cp_group
+ super().set_context_parallel_group(
+ self.cp_group,
+ torch.distributed.get_process_group_ranks(self.cp_group),
+ TEDotProductAttention.cp_stream,
+ self.cp_comm_type,
+ )
+ # If cp_group is None but local_cp_size is provided,
+ # Indicates to turn off CP dynamically
+ elif packed_seq_params.local_cp_size is not None:
+ assert (
+ packed_seq_params.local_cp_size == 1
+ ), "local_cp_size must be == 1 if provided without cp_group"
+ super().set_context_parallel_group(None, None, None, self.cp_comm_type)
+ self.kept_packed_seq_params.discard("cp_group")
+ self.kept_packed_seq_params.discard("local_cp_size")
+
# Default to constructor-provided num_splits unless explicitly overridden
if num_splits is None:
num_splits = self.num_splits
@@ -1436,7 +1456,7 @@ def __init__(
skip_bias_add: bool,
is_expert: bool = False,
tp_comm_buffer_name: Optional[str] = None,
- tp_group: Optional[torch.distributed.ProcessGroup] = None,
+ pg_collection: Optional[ProcessGroupCollection] = None,
):
self.config = config
@@ -1467,9 +1487,14 @@ def __init__(
# The comms between TP and EP group is explicitly handled by MoE token dispatcher.
# So we disable comms by making TE agnostic of model parallel.
- tp_group = get_tensor_model_parallel_group_if_none(tp_group, is_expert=is_expert)
+ if pg_collection is None:
+ pg_collection = ProcessGroupCollection.use_mpu_process_groups()
+ self._pg_collection = pg_collection
+ assert is_expert, "TEGroupedLinear only supports expert parallelism"
+ tp_group = pg_collection.expt_tp
self._tp_group = tp_group
tp_size = get_pg_size(tp_group)
+ tp_group_for_te = tp_group
self.explicit_expert_comm = is_expert and (tp_size > 1 or self.expert_parallel)
@@ -1480,7 +1505,7 @@ def __init__(
input_size = divide(input_size, tp_size)
parallel_mode = None
tp_size = 1
- tp_group = None
+ tp_group_for_te = None
super().__init__(
num_gemms=num_gemms,
@@ -1488,7 +1513,7 @@ def __init__(
out_features=output_size,
sequence_parallel=self.config.sequence_parallel,
fuse_wgrad_accumulation=self.config.gradient_accumulation_fusion,
- tp_group=tp_group if torch.distributed.is_initialized() else None,
+ tp_group=tp_group_for_te if torch.distributed.is_initialized() else None,
tp_size=tp_size,
get_rng_state_tracker=(
get_cuda_rng_tracker if get_cuda_rng_tracker().is_initialized() else None
@@ -1697,8 +1722,8 @@ def _sharded_state_dict_grouped(
singleton_local_shards = (metadata or {}).get('singleton_local_shards', False)
sharded_state_dict = {}
full_state_dict = self.state_dict(prefix="", keep_vars=True)
- num_global_experts = get_expert_model_parallel_world_size() * self.num_gemms
- local_expert_indices_offset = get_expert_model_parallel_rank() * self.num_gemms
+ num_global_experts = get_pg_size(self._pg_collection.ep) * self.num_gemms
+ local_expert_indices_offset = get_pg_rank(self._pg_collection.ep) * self.num_gemms
ep_axis = len(sharded_offsets)
extra_states = self._split_extra_state(full_state_dict["_extra_state"])
for gemm_idx in range(self.num_gemms):
@@ -1747,7 +1772,7 @@ def _sharded_state_dict_grouped(
if getattr(sh_ten, "is_data_parallel_fully_shard", False):
edp_replica_id = 0
else:
- edp_replica_id = get_expert_data_parallel_rank()
+ edp_replica_id = get_pg_rank(self._pg_collection.expt_dp)
sh_ten.replica_id = (*replica_id[:2], edp_replica_id)
return sharded_state_dict
@@ -1777,7 +1802,7 @@ def __init__(
skip_bias_add: bool,
is_expert: bool,
tp_comm_buffer_name: Optional[str] = None,
- tp_group: Optional[torch.distributed.ProcessGroup] = None,
+ pg_collection: Optional[ProcessGroupCollection] = None,
):
super().__init__(
num_gemms=num_gemms,
@@ -1790,7 +1815,7 @@ def __init__(
skip_bias_add=skip_bias_add,
is_expert=is_expert,
tp_comm_buffer_name=tp_comm_buffer_name,
- tp_group=tp_group,
+ pg_collection=pg_collection,
)
def sharded_state_dict(self, prefix="", sharded_offsets=(), metadata=None):
@@ -1823,7 +1848,7 @@ def __init__(
skip_bias_add: bool,
is_expert: bool,
tp_comm_buffer_name: Optional[str] = None,
- tp_group: Optional[torch.distributed.ProcessGroup] = None,
+ pg_collection: Optional[ProcessGroupCollection] = None,
):
super().__init__(
num_gemms=num_gemms,
@@ -1836,7 +1861,7 @@ def __init__(
skip_bias_add=skip_bias_add,
is_expert=is_expert,
tp_comm_buffer_name=tp_comm_buffer_name,
- tp_group=tp_group,
+ pg_collection=pg_collection,
)
def sharded_state_dict(self, prefix="", sharded_offsets=(), metadata=None):
@@ -2114,7 +2139,7 @@ def forward_post_hook(module, *_) -> None:
"TEFusedMLP module does not support submodules with post-backward hooks"
)
- def forward(self, hidden_states: torch.Tensor) -> Tuple[Tensor, Optional[Tensor]]:
+ def forward(self, hidden_states: torch.Tensor, **kwargs) -> Tuple[Tensor, Optional[Tensor]]:
"""Forward."""
# Construct fused impl if needed
@@ -2469,3 +2494,18 @@ def set_save_original_input(module):
"set_save_original_input is only needed on transformer-engine modules that save "
"quantized tensors by default. It needs transformer-engine>=2.6.0dev0."
)
+
+
+try:
+ # pylint: disable=unused-import
+ from transformer_engine.pytorch import cpu_offload_v1 as cpu_offload
+except ImportError:
+ try:
+ from transformer_engine.pytorch import cpu_offload
+ except ImportError:
+ cpu_offload = None
+try:
+ # pylint: disable=unused-import
+ from transformer_engine.pytorch.float8_tensor import Float8Tensor
+except ImportError:
+ Float8Tensor = None
diff --git a/megatron/core/extensions/transformer_engine_spec_provider.py b/megatron/core/extensions/transformer_engine_spec_provider.py
index a071959bfc9..6f8947078b9 100644
--- a/megatron/core/extensions/transformer_engine_spec_provider.py
+++ b/megatron/core/extensions/transformer_engine_spec_provider.py
@@ -17,6 +17,7 @@
from megatron.core.fusions.fused_layer_norm import FusedLayerNorm
from megatron.core.models.backends import BackendSpecProvider
from megatron.core.tensor_parallel.layers import ColumnParallelLinear, RowParallelLinear
+from megatron.core.transformer.dot_product_attention import DotProductAttention
from megatron.core.transformer.mlp import MLPSubmodules
from megatron.core.transformer.moe.experts import GroupedMLP, SequentialMLP, TEGroupedMLP
from megatron.core.utils import get_te_version, is_te_min_version
@@ -25,6 +26,10 @@
class TESpecProvider(BackendSpecProvider):
"""A protocol for providing the submodules used in Spec building."""
+ def __init__(self, fallback_to_eager_attn: bool = False):
+ super().__init__()
+ self.fallback_to_eager_attn = fallback_to_eager_attn
+
def linear(self) -> type:
"""Which linear module TE backend uses"""
return TELinear
@@ -56,6 +61,8 @@ def layer_norm(self, rms_norm: bool = False, for_qk: bool = False) -> type:
def core_attention(self) -> type:
"""Which module to use for attention"""
+ if self.fallback_to_eager_attn:
+ return DotProductAttention
return TEDotProductAttention
def grouped_mlp_modules(
diff --git a/megatron/core/fp4_utils.py b/megatron/core/fp4_utils.py
index 4f9e7e5d026..95368d7c2b7 100644
--- a/megatron/core/fp4_utils.py
+++ b/megatron/core/fp4_utils.py
@@ -61,13 +61,23 @@ def get_fp4_align_size(fp4_recipe: Fp4Recipe) -> int:
Note that since we are also random hadamard transform for NVFP4 training, we want
fused group nvfp4 quantize plus hadamard transform. Hadamard transform will leverage
tensor core instructions for better performance, while group quantize kernels also
- prefer a more aligned size in token dimension M. Therefore, we apply align size 64
- here for better performance in MOE.
+ prefer a more aligned size in token dimension M. The efficiently leverage grouped
+ kernels, padding needs to be 64 multiple, but 128 multiple will bring even faster.
+
+ When it comes to MOE cuda graph support, the number of tokens for each expert should
+ be a buffer on device memory, which means that we don't know the token dimension for
+ each expertin host, therefore we cannot calculate the zero padded scaling factors shape
+ on host to comply with the NVFP4 GEMM scaling factor layout. However, if we have already
+ zero padded the tokens to 128 multiple, then there is no need for such padding, so that
+ host doesn't need to copy the token distribution from device to host (which will break
+ the CUDA graph).
Paper link: https://arxiv.org/pdf/2509.25149
+ Scaling factor layout: https://docs.nvidia.com/cuda/cublas/#d-block-scaling-factors-layout
+ TE NVFP4 Grouped Quantization: https://github.com/NVIDIA/TransformerEngine/pull/2411
"""
# pylint: disable=unused-argument
- return 64
+ return 128
def dequantize_fp4_tensor(fp4_tensor: torch.Tensor) -> torch.Tensor:
@@ -86,7 +96,9 @@ def get_fp4_recipe(config: TransformerConfig):
if is_te_min_version("2.7.0.dev0"):
if config.fp4_recipe == Fp4Recipe.nvfp4:
try:
- fp4_recipe = transformer_engine.common.recipe.NVFP4BlockScaling()
+ fp4_recipe = transformer_engine.common.recipe.NVFP4BlockScaling(
+ fp8_dpa=config.fp8_dot_product_attention
+ )
except AttributeError:
raise ValueError(
"""NVFP4BlockScaling recipe is not available in this version of
diff --git a/megatron/core/fusions/fused_linear_cross_entropy.py b/megatron/core/fusions/fused_linear_cross_entropy.py
new file mode 100644
index 00000000000..b533fef7aa3
--- /dev/null
+++ b/megatron/core/fusions/fused_linear_cross_entropy.py
@@ -0,0 +1,242 @@
+# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved.
+
+"""
+Linear Cross Entropy API
+Fuse cross entropy with linear layer.
+"""
+
+import typing
+from functools import lru_cache
+
+import torch
+
+
+class Platform:
+ """
+ Singleton class for targeted GPU platform.
+ """
+
+ _instance: typing.Optional["Platform"] = None
+
+ def __new__(cls) -> "Platform":
+ if cls._instance is None:
+ cls._instance = super().__new__(cls)
+ return cls._instance
+
+ def __init__(self) -> None:
+ if getattr(self, "_initialized", False):
+ return
+
+ assert torch.cuda.is_available(), "CUDA is not available"
+ device = torch.cuda.current_device()
+ cc = torch.cuda.get_device_capability(device)
+
+ if cc[0] == 10:
+ from .linear_cross_entropy.blackwell import entry as gpu_entry
+
+ self.forward_func: typing.Callable[..., typing.Any] = gpu_entry.forward
+ self.backward_func: typing.Callable[..., typing.Any] = gpu_entry.backward
+ else:
+ raise ValueError(f"Unsupported architecture: {cc[0]}")
+
+ self._initialized = True
+
+
+@lru_cache(maxsize=1)
+def _get_platform() -> Platform:
+ """
+ Helper function to lazy initialize the platform.
+ """
+ return Platform()
+
+
+class LinearCrossEntropy(torch.autograd.Function):
+ """
+ This class implements a custom autograd function for linear and cross entropy,
+ whose equivalent logic in PyTorch is:
+ ```python
+ def torch_entropy(hidden, weight, labels):
+ logits = torch.matmul(hidden, weight)
+ logprobs = torch.nn.functional.cross_entropy(logits, labels)
+ return logprobs
+ ```
+ """
+
+ @staticmethod
+ def forward(
+ ctx,
+ hidden: torch.Tensor,
+ weight: torch.Tensor,
+ labels: torch.Tensor,
+ tp_group: typing.Optional[torch.distributed.ProcessGroup] = None,
+ reduction: typing.Literal["none", "sum", "mean"] = "mean",
+ ignore_index: int = -100,
+ sequence_parallel: bool = False,
+ ) -> torch.Tensor:
+ """
+ The forward pass of the Linear Cross Entropy.
+ If tp_group is not None, the weight tensor to each TP rank should be
+ (global_vocab_size // world_size, dim).
+ Note that each of the ranks should get equal shards along the vocab_size dimension.
+
+ Args:
+ @param hidden: the input tensor with shape (num_tokens, dim)
+ @param weight: the lm_head weight tensor with shape (local_vocab_size, dim)
+ @param labels: the labels tensor with shape (num_tokens,)
+ @param tp_group: the distributed process group for TP.
+ @param reduction: Default to "mean", and can be one of "none", "sum", "mean".
+ @param ignore_index: The index to ignore. Default to -100.
+ @param sequence_parallel: Whether to use sequence parallel. Default to False.
+ Returns:
+ @return: logprobs with shape
+ - either (num_tokens,) when reduction is "none"
+ - or (1,) when reduction is "mean" or "sum"
+
+ tp_group is None ----------------------------------> DP
+ B
+ A C
+ tp_group is not None & sequence_parallel is False -> TP
+ B0 B1
+ A C0 C1
+ tp_group is not None & sequence_parallel is True --> SP
+ B0 B1
+ A0 C0 XX
+ A1 XX C1
+
+ When tp_group is not None, the weight tensor will be split along the vocab_size
+ dimension, which means each rank will get equal shards along the global_vocab_size
+ dimension. Specifically, the weight tensor to each rank will be (local_vocab_size, dim).
+ And there is an assumption that each rank will get the same local_vocab_size.
+
+ When sequence_parallel is True, the hidden tensor will be split along the
+ sequence length dimension, which means each rank will get equal shards along
+ the sequence length dimension. Specifically, the hidden tensor to each rank
+ will be (local_num_tokens, dim). And there is an assumption that each rank
+ will get the same local_num_tokens.
+
+ In TP forward pass, the hidden tensor and label tensor shall be identical
+ among all TP ranks, and it's user's responsibility to ensure the hidden tensor
+ is identical among all TP ranks. Then this operation will produce identical
+ logprobs among all TP ranks.
+
+ In TP backward pass, the gradient of the logprobs shall be identical among all
+ TP ranks, and it's user's responsibility to ensure the gradient of the logprobs
+ is identical among all TP ranks. Then this operation will produce distinct gradients
+ for the local weight tensor, and identical gradients for the hidden tensor.
+
+ ```python
+ # ------------ forward pass ------------ #
+ hidden = tp_group.broadcast(hidden, src=0) # handled by framework
+ labels = tp_group.broadcast(labels, src=0) # handled by framework
+ logprobs = linear_cross_entropy(...)
+ # each rank will get the same logprobs
+
+ # ------------ backward pass ------------ #
+ g_logprobs = tp_group.broadcast(g_logprobs, src=0) # handled by framework
+ d_hidden, d_weight = torch.autograd.grad(...)
+ # each rank will get the same d_hidden,
+ # and distinct d_weight for local weight shard
+ ```
+
+ In SP forward pass, the hidden tensor shall be split along the sequence length dimension,
+ and the label tensor shall be identical among all TP ranks.
+ Then this operation will produce identical logprobs among all TP ranks.
+
+ In SP backward pass, the gradient of the logprobs shall be identical among all TP ranks,
+ Then this operation will produce distinct gradients for the local hidden tensor
+ and local weight tensor.
+ ```python
+ # ------------ forward pass ------------ #
+ hidden = global_hidden[tp_rank] # handled by framework
+ labels = tp_group.broadcast(labels, src=0) # handled by framework
+ logprobs = linear_cross_entropy(...)
+ # each rank will get the same logprobs
+
+ # ------------ backward pass ------------ #
+ g_logprobs = tp_group.broadcast(g_logprobs, src=0) # handled by framework
+ d_hidden, d_weight = torch.autograd.grad(...)
+ # each rank will get distinct local d_hidden and d_weight
+ ```
+ """
+ with torch.cuda.nvtx.range("LinearCrossEntropy-forward"):
+ (
+ logprobs,
+ _maximum,
+ _acc,
+ _num_valid_tokens,
+ tp_rank,
+ tp_world_size,
+ global_hidden,
+ ) = _get_platform().forward_func(
+ hidden, weight, labels, tp_group, reduction, ignore_index, sequence_parallel
+ )
+ ctx.save_for_backward(global_hidden, weight, labels, _maximum, _acc, _num_valid_tokens)
+ ctx.tp_group = tp_group
+ ctx.ignore_index = ignore_index
+ ctx.reduction = reduction
+ ctx.tp_rank = tp_rank
+ ctx.tp_world_size = tp_world_size
+ ctx.sequence_parallel = sequence_parallel
+
+ return logprobs
+
+ @staticmethod
+ def backward(
+ ctx, dlogprobs: torch.Tensor
+ ) -> typing.Tuple[torch.Tensor, torch.Tensor, None, None, None, None, None]:
+ """
+ The backward pass of the Linear Cross Entropy.
+ Args:
+ dlogprobs (torch.Tensor): The gradient of the cross entropy, with shape
+ - either (num_tokens,) when reduction is "none"
+ - or (1,) when reduction is "mean" or "sum"
+ Returns:
+ dhidden (torch.Tensor): The gradient of the hidden.
+ dweight (torch.Tensor): The gradient of the weight.
+ """
+ with torch.cuda.nvtx.range("LinearCrossEntropy-backward"):
+ (global_hidden, weight, labels, _maximum, _accu, _num_valid_tokens) = ctx.saved_tensors
+
+ tp_group = ctx.tp_group
+ ignore_index = ctx.ignore_index
+ reduction = ctx.reduction
+ tp_rank = ctx.tp_rank
+ tp_world_size = ctx.tp_world_size
+ sequence_parallel = ctx.sequence_parallel
+
+ d_hidden, d_weight = _get_platform().backward_func(
+ dlogprobs,
+ global_hidden,
+ weight,
+ labels,
+ _maximum,
+ _accu,
+ _num_valid_tokens,
+ reduction,
+ ignore_index,
+ tp_group,
+ tp_rank,
+ tp_world_size,
+ sequence_parallel,
+ )
+
+ return d_hidden, d_weight, None, None, None, None, None
+
+
+def linear_cross_entropy(
+ hidden: torch.Tensor,
+ weight: torch.Tensor,
+ labels: torch.Tensor,
+ tp_group: typing.Optional[torch.distributed.ProcessGroup] = None,
+ reduction: typing.Literal["none", "sum", "mean"] = "mean",
+ ignore_index: int = -100,
+ sequence_parallel: bool = False,
+) -> torch.Tensor:
+ """
+ helper function for linear cross entropy.
+ """
+ _impl = LinearCrossEntropy.apply
+ return _impl(hidden, weight, labels, tp_group, reduction, ignore_index, sequence_parallel)
+
+
+__all__ = ["linear_cross_entropy", "LinearCrossEntropy"]
diff --git a/megatron/core/fusions/linear_cross_entropy/__init__.py b/megatron/core/fusions/linear_cross_entropy/__init__.py
new file mode 100644
index 00000000000..b9a9591fa69
--- /dev/null
+++ b/megatron/core/fusions/linear_cross_entropy/__init__.py
@@ -0,0 +1 @@
+# Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
diff --git a/megatron/core/fusions/linear_cross_entropy/blackwell/__init__.py b/megatron/core/fusions/linear_cross_entropy/blackwell/__init__.py
new file mode 100644
index 00000000000..b9a9591fa69
--- /dev/null
+++ b/megatron/core/fusions/linear_cross_entropy/blackwell/__init__.py
@@ -0,0 +1 @@
+# Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
diff --git a/megatron/core/fusions/linear_cross_entropy/blackwell/bwd_partial_dlogits.py b/megatron/core/fusions/linear_cross_entropy/blackwell/bwd_partial_dlogits.py
new file mode 100644
index 00000000000..3178e8c6909
--- /dev/null
+++ b/megatron/core/fusions/linear_cross_entropy/blackwell/bwd_partial_dlogits.py
@@ -0,0 +1,667 @@
+# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved.
+
+import logging
+from typing import Optional, Tuple, Type
+
+try:
+ import cuda.bindings.driver as cuda # type: ignore
+ import cutlass
+ import cutlass.cute as cute
+ import cutlass.pipeline as pipeline # type: ignore
+ import cutlass.utils as utils # type: ignore
+ import cutlass.utils.blackwell_helpers as sm100_utils # type: ignore
+ from cutlass.cute.nvgpu import cpasync, tcgen05
+
+ SM100_TMEM_CAPACITY_COLUMNS: int = 512
+
+ def make_thread_cooperative_group(size: int, alignment: Optional[int] = None):
+ """
+ Create a thread cooperative group.
+ """
+ return pipeline.CooperativeGroup(
+ pipeline.Agent.Thread, size, alignment=alignment if alignment is not None else size
+ )
+
+ class BwdPartialDlogits:
+ """
+ This class implements the backward kernel for partial d_logits.
+ """
+
+ def __init__(
+ self,
+ reduction: int,
+ acc_dtype: Type[cutlass.Numeric] = cutlass.Float32,
+ use_2cta_instrs: bool = False,
+ mma_tiler_mn: Tuple[int, int] = (128, 256),
+ vocab_per_split: int = 512,
+ ):
+ self.REDUCTION: cutlass.Constexpr[cutlass.Int32] = cutlass.const_expr(reduction)
+ self.acc_dtype = acc_dtype
+ self.use_2cta_instrs = use_2cta_instrs
+ self.mma_tiler = (*mma_tiler_mn, 1)
+ self.vocab_per_split = vocab_per_split
+
+ self.cta_group = tcgen05.CtaGroup.TWO if self.use_2cta_instrs else tcgen05.CtaGroup.ONE
+ self.cluster_shape_mn = (2, 1) if self.use_2cta_instrs else (1, 1)
+
+ self.smem_capacity = utils.get_smem_capacity_in_bytes("sm_100")
+
+ self.threads_per_warp: int = 32
+
+ self.epi_warp_ids = (0, 1, 2, 3)
+ self.load_warp_ids = 4
+ self.mma_warp_ids = 5
+ self.empty_warp_ids = (6, 7)
+
+ self.threads_per_cta: int = self.threads_per_warp * len(
+ (*self.epi_warp_ids, self.load_warp_ids, self.mma_warp_ids, *self.empty_warp_ids)
+ )
+ self.cta_sync_barrier = pipeline.NamedBarrier(
+ barrier_id=1, num_threads=self.threads_per_cta
+ )
+
+ self.buffer_align_bytes: int = 1024
+ self.num_regs_other: int = 32
+ self.num_regs_epi: int = 192
+
+ def _compute_grid(
+ self,
+ problem_mnk: Tuple[int, int, int],
+ cluster_shape_mn: Tuple[int, int],
+ cta_tiler: Tuple[int, int, int],
+ ) -> Tuple[int, int, int]:
+ cluster_shape_mnk = (*cluster_shape_mn, 1)
+
+ grid = cute.round_up(
+ (
+ cute.ceil_div(problem_mnk[0], cta_tiler[0]),
+ cute.ceil_div(self.vocab_per_split, cta_tiler[1]),
+ 1,
+ ),
+ cluster_shape_mnk,
+ )
+ return grid
+
+ def _compute_stages(
+ self,
+ tiled_mma: cute.TiledMma,
+ mma_tiler: Tuple[int, int, int],
+ a_dtype: Type[cutlass.Numeric],
+ b_dtype: Type[cutlass.Numeric],
+ ):
+ num_acc_stage = 1
+ num_ab_stage = 4
+ num_epi_stage_per_tile = 4
+ return num_acc_stage, num_ab_stage, num_epi_stage_per_tile
+
+ def _setup_attributes(
+ self,
+ tiled_mma: cute.TiledMma,
+ a_dtype: Type[cutlass.Numeric],
+ b_dtype: Type[cutlass.Numeric],
+ ):
+ self.cluster_shape_mnk = (*self.cluster_shape_mn, 1)
+ self.cluster_layout_vmnk = cute.tiled_divide(
+ cute.make_layout(self.cluster_shape_mnk), (tiled_mma.thr_id.shape,)
+ )
+
+ mma_inst_shape_k = cute.size(tiled_mma.shape_mnk, mode=[2])
+ # it requires k-mode to be 128B aligned
+ mma_inst_tile_k: int = 4
+ self.mma_tiler = (
+ self.mma_tiler[0],
+ self.mma_tiler[1],
+ mma_inst_shape_k * mma_inst_tile_k,
+ )
+
+ self.num_acc_stage, self.num_ab_stage, self.num_epi_stage_per_tile = (
+ self._compute_stages(tiled_mma, self.mma_tiler, a_dtype, b_dtype)
+ )
+ self.tmem_alloc_cols = self.num_acc_stage * self.mma_tiler[1]
+ assert self.tmem_alloc_cols <= SM100_TMEM_CAPACITY_COLUMNS
+
+ self.cta_tile_shape_mnk = (
+ self.mma_tiler[0] // cute.size(tiled_mma.thr_id.shape),
+ self.mma_tiler[1],
+ self.mma_tiler[2],
+ )
+
+ @cute.kernel
+ def kernel(
+ self,
+ split_idx: cutlass.Int32,
+ tiled_mma: cute.TiledMma,
+ tma_atom_a: cute.CopyAtom,
+ mA: cute.Tensor,
+ tma_atom_b: cute.CopyAtom,
+ mB: cute.Tensor,
+ mLabels: cute.Tensor,
+ mDlogprobs: cute.Tensor,
+ mMaximum: cute.Tensor,
+ mAccu: cute.Tensor,
+ mDlogits_partial: cute.Tensor,
+ scalarNumValidTokens: cute.Pointer,
+ ignore_index: cutlass.Int64,
+ a_smem_layout_staged: cute.ComposedLayout,
+ b_smem_layout_staged: cute.ComposedLayout,
+ cluster_layout_vmnk: cute.Layout,
+ problem_mnk: Tuple[int, int, int],
+ rank: cutlass.Int32,
+ ) -> None:
+ """
+ The backward kernel for partial d_logits.
+ """
+ warp_idx = cute.arch.make_warp_uniform(cute.arch.warp_idx())
+ tidx, _, _ = cute.arch.thread_idx()
+ bidx, bidy, _ = cute.arch.block_idx()
+ # FIXME: block swizzling applied here
+ pidm, pidn = bidx, bidy
+
+ # FIXME: if 2 CTAs, modify here
+ cta_rank_in_cluster = 0
+ block_in_cluster_coord_vmnk = cluster_layout_vmnk.get_flat_coord(cta_rank_in_cluster)
+
+ # prefetch tma descriptors
+ if warp_idx == self.load_warp_ids:
+ cute.nvgpu.cpasync.prefetch_descriptor(tma_atom_a)
+ cute.nvgpu.cpasync.prefetch_descriptor(tma_atom_b)
+
+ smem = utils.SmemAllocator()
+ storage = smem.allocate(self.shared_storage)
+
+ ab_pipeline = pipeline.PipelineTmaUmma.create(
+ num_stages=self.num_ab_stage,
+ producer_group=make_thread_cooperative_group(len([self.load_warp_ids])),
+ consumer_group=make_thread_cooperative_group(len([self.mma_warp_ids])),
+ tx_count=self.tma_copy_ab_bytes,
+ barrier_storage=storage.load_ab_mbar_ptr.data_ptr(),
+ )
+ ab_producer_state = pipeline.make_pipeline_state(
+ pipeline.PipelineUserType.Producer, self.num_ab_stage
+ )
+ ab_consumer_state = pipeline.make_pipeline_state(
+ pipeline.PipelineUserType.Consumer, self.num_ab_stage
+ )
+
+ mma_pipeline = pipeline.PipelineUmmaAsync.create(
+ num_stages=self.num_acc_stage,
+ producer_group=make_thread_cooperative_group(len([self.mma_warp_ids])),
+ consumer_group=make_thread_cooperative_group(
+ self.threads_per_warp * len(self.epi_warp_ids)
+ ),
+ barrier_storage=storage.mma_mbar_ptr.data_ptr(),
+ )
+ mma_producer_state = pipeline.make_pipeline_state(
+ pipeline.PipelineUserType.Producer, self.num_acc_stage
+ )
+ mma_consumer_state = pipeline.make_pipeline_state(
+ pipeline.PipelineUserType.Consumer, self.num_acc_stage
+ )
+
+ tmem_dealloc_mbar_ptr = storage.tmem_dealloc_mbar_ptr.data_ptr()
+ if warp_idx == self.empty_warp_ids[0]:
+ with cute.arch.elect_one():
+ cute.arch.mbarrier_init(
+ tmem_dealloc_mbar_ptr, self.threads_per_warp * len(self.epi_warp_ids)
+ )
+ cute.arch.mbarrier_init_fence()
+
+ # -------- tensor partition ------------ #
+ # swizzle o [(tileM, tileK), loopM, loopK, stage]
+ sA = storage.sA.get_tensor(
+ a_smem_layout_staged.outer, swizzle=a_smem_layout_staged.inner
+ )
+ # swizzle o [(tileN, tileK), loopN, loopK, stage]
+ sB = storage.sB.get_tensor(
+ b_smem_layout_staged.outer, swizzle=b_smem_layout_staged.inner
+ )
+
+ # FIXME: if 2 CTAs, modify here
+ thr_mma = tiled_mma.get_slice(0)
+ # [MMA, loopM, loopK, stage]
+ tCsA = thr_mma.make_fragment_A(sA)
+ # [MMA, loopN, loopK, stage]
+ tCsB = thr_mma.make_fragment_B(sB)
+
+ # [tileM, tileK, loopK]
+ gA = cute.local_tile(
+ mA, (self.cta_tile_shape_mnk[0], self.cta_tile_shape_mnk[2]), (pidm, None)
+ )
+ # [vocab_per_split, dim]
+ mB_n = cute.local_tile(
+ mB, (self.vocab_per_split, cute.size(mB.layout.shape, mode=[1])), (split_idx, 0)
+ )
+ # [tileN, tileK, loopK]
+ gB = cute.local_tile(
+ mB_n, (self.cta_tile_shape_mnk[1], self.cta_tile_shape_mnk[2]), (pidn, None)
+ )
+
+ a_cta_layout = cute.make_layout(cute.slice_(cluster_layout_vmnk, (0, 0, None, 0)).shape)
+ # just to make sure SMEM and GMEM tensor has the same size in the first rank
+ tCgA = thr_mma.partition_A(gA)
+ tCgB = thr_mma.partition_B(gB)
+ # [CPY, stage] & [CPY, loopK]
+ tTMAsA, tTMAgA = cpasync.tma_partition(
+ tma_atom_a,
+ block_in_cluster_coord_vmnk[2], # cta_coord,
+ a_cta_layout,
+ cute.group_modes(sA, 0, 3),
+ cute.group_modes(tCgA, 0, 3),
+ )
+ b_cta_layout = cute.make_layout(cute.slice_(cluster_layout_vmnk, (0, None, 0, 0)).shape)
+ # [CPY, stage] & [CPY, loopK]
+ tTMAsB, tTMAgB = cpasync.tma_partition(
+ tma_atom_b,
+ block_in_cluster_coord_vmnk[1], # cta_coord
+ b_cta_layout,
+ cute.group_modes(sB, 0, 3),
+ cute.group_modes(tCgB, 0, 3),
+ )
+
+ # ------ Allocate TMEM ------ #
+ tmem_holding_buf = storage.tmem_holding_buf
+ if warp_idx == self.empty_warp_ids[0]:
+ cute.arch.alloc_tmem(
+ self.tmem_alloc_cols, tmem_holding_buf, is_two_cta=self.use_2cta_instrs
+ )
+ self.cta_sync_barrier.arrive_and_wait()
+ tmem_ptr = cute.arch.retrieve_tmem_ptr(
+ self.acc_dtype, alignment=16, ptr_to_buffer_holding_addr=tmem_holding_buf
+ )
+
+ tmem_shape = (128, self.tmem_alloc_cols)
+ acc_shape = thr_mma.partition_shape_C(tmem_shape)
+ tCtC_fake = thr_mma.make_fragment_C(acc_shape)
+ # [(tileM, tileN), loopM, loopN]
+ tCtC = cute.make_tensor(tmem_ptr, tCtC_fake.layout)
+
+ # ------ Empty ------ #
+ if warp_idx in self.empty_warp_ids:
+ cute.arch.warpgroup_reg_dealloc(self.num_regs_other)
+
+ # ------ Load ------ #
+ if warp_idx == self.load_warp_ids:
+ cute.arch.warpgroup_reg_dealloc(self.num_regs_other)
+
+ for k in cutlass.range(cute.size(gA, mode=[2])):
+ ab_pipeline.producer_acquire(ab_producer_state)
+ cute.copy(
+ tma_atom_a,
+ tTMAgA[(None, k)],
+ tTMAsA[(None, ab_producer_state.index)],
+ tma_bar_ptr=ab_pipeline.producer_get_barrier(ab_producer_state),
+ )
+ cute.copy(
+ tma_atom_b,
+ tTMAgB[(None, k)],
+ tTMAsB[(None, ab_producer_state.index)],
+ tma_bar_ptr=ab_pipeline.producer_get_barrier(ab_producer_state),
+ )
+ ab_pipeline.producer_commit(ab_producer_state)
+ ab_producer_state.advance()
+
+ # ------ MMA ------ #
+ if warp_idx == self.mma_warp_ids:
+ cute.arch.warpgroup_reg_dealloc(self.num_regs_other)
+
+ tiled_mma.set(tcgen05.Field.ACCUMULATE, False)
+ mma_pipeline.producer_acquire(mma_producer_state)
+
+ for k in cutlass.range(cute.size(gA, mode=[2])):
+ ab_pipeline.consumer_wait(ab_consumer_state)
+
+ for kblock_idx in cutlass.range(cute.size(tCsA, mode=[2]), unroll_full=True):
+ cute.gemm(
+ tiled_mma,
+ cute.append_ones(tCtC[(None, None, mma_producer_state.index)]),
+ tCsA[(None, None, kblock_idx, ab_consumer_state.index)],
+ tCsB[(None, None, kblock_idx, ab_consumer_state.index)],
+ cute.append_ones(tCtC[(None, None, mma_producer_state.index)]),
+ )
+ tiled_mma.set(tcgen05.Field.ACCUMULATE, True)
+
+ ab_pipeline.consumer_release(ab_consumer_state)
+ ab_consumer_state.advance()
+
+ mma_pipeline.producer_commit(mma_producer_state)
+ mma_producer_state.advance()
+
+ # ------ EPI ------ #
+ if warp_idx in self.epi_warp_ids:
+ cute.arch.warpgroup_reg_alloc(self.num_regs_epi)
+
+ copy_atom_t2r = sm100_utils.get_tmem_load_op(
+ self.cta_tile_shape_mnk,
+ utils.LayoutEnum.ROW_MAJOR,
+ self.acc_dtype,
+ self.acc_dtype,
+ (self.epi_tile[0], self.epi_tile[1] // self.num_epi_stage_per_tile),
+ self.use_2cta_instrs,
+ )
+ # [tileM, subTileN, loopM, CntSubTileN, loopN]
+ tAcc_epi = cute.flat_divide(
+ tCtC[((None, None), 0, None)],
+ (self.epi_tile[0], self.epi_tile[1] // self.num_epi_stage_per_tile),
+ )
+ tiled_copy_t2r = tcgen05.make_tmem_copy(
+ copy_atom_t2r, tAcc_epi[(None, None, 0, 0, 0)]
+ )
+ thr_copy_t2r = tiled_copy_t2r.get_slice(tidx)
+ tTMEM_load_tAcc = thr_copy_t2r.partition_S(tAcc_epi)
+ tTMEM_load_tAcc = cute.group_modes(
+ tTMEM_load_tAcc, 3, cute.rank(tTMEM_load_tAcc) - 1
+ )
+
+ # predicates
+ cAcc = cute.make_identity_tensor(self.mma_tiler[:2])
+ tCcAcc = thr_mma.partition_C(cAcc)
+ tCcAcc_epi = cute.flat_divide(
+ tCcAcc[((None, None), 0, None)],
+ (self.epi_tile[0], self.epi_tile[1] // self.num_epi_stage_per_tile),
+ )
+ tTMEM_load_cAcc = thr_copy_t2r.partition_D(tCcAcc_epi)
+ tTMEM_load_cAcc_shape = cute.select(tTMEM_load_cAcc.shape, mode=[0, 1, 2])
+ tTMEM_load_rAcc = cute.make_fragment(tTMEM_load_cAcc_shape, self.acc_dtype)
+
+ copy_atom_g2r_int64 = cute.make_copy_atom(
+ cute.nvgpu.CopyUniversalOp(), mLabels.element_type
+ )
+ copy_atom_g2r_fp32 = cute.make_copy_atom(
+ cute.nvgpu.CopyUniversalOp(), mDlogprobs.element_type
+ )
+ epilogue_thread_layout = cute.make_layout((128, 1), stride=(1, 1))
+ tiled_copy_g2r_int64 = cute.make_tiled_copy_tv(
+ copy_atom_g2r_int64, epilogue_thread_layout, cute.make_layout((1, 1))
+ )
+ tiled_copy_g2r_fp32 = cute.make_tiled_copy_tv(
+ copy_atom_g2r_fp32, epilogue_thread_layout, cute.make_layout((1, 1))
+ )
+ thr_copy_g2r_int64 = tiled_copy_g2r_int64.get_slice(tidx)
+ thr_copy_g2r_fp32 = tiled_copy_g2r_fp32.get_slice(tidx)
+
+ # [tileM]
+ gLabels = cute.local_tile(mLabels, (self.epi_tile[0],), (pidm,))
+ gMaximum = cute.local_tile(mMaximum, (self.epi_tile[0],), (pidm,))
+ gAccu = cute.local_tile(mAccu, (self.epi_tile[0],), (pidm,))
+
+ # slice along M direction
+ tMCAcc = thr_copy_g2r_int64.partition_S(cAcc)[(None, None, 0)]
+ # [(1, 1), 1]
+ tMCAcc_mask = cute.make_fragment(tMCAcc.shape, cutlass.Boolean)
+ # to align shape with gMax and gAccu
+ tMCAcc_mask = cute.append_ones(tMCAcc_mask)
+ tMCAcc_mask[0] = cute.elem_less(
+ pidm * self.epi_tile[0] + tidx, cute.size(mA, mode=[0])
+ )
+ # [(1, 1), 1, 1]
+ tMgLabels = thr_copy_g2r_int64.partition_S(cute.append_ones(gLabels))
+ tMrLabels = cute.make_fragment(tMgLabels.shape, tMgLabels.element_type)
+ cute.copy(tiled_copy_g2r_int64, tMgLabels, tMrLabels, pred=tMCAcc_mask)
+ tMgMaximum = thr_copy_g2r_fp32.partition_S(cute.append_ones(gMaximum))
+ tMrMaximum = cute.make_fragment(tMgMaximum.layout, tMgMaximum.element_type)
+ cute.copy(tiled_copy_g2r_fp32, tMgMaximum, tMrMaximum, pred=tMCAcc_mask)
+ tMgAccu = thr_copy_g2r_fp32.partition_S(cute.append_ones(gAccu))
+ tMrAccu = cute.make_fragment(tMgAccu.layout, tMgAccu.element_type)
+ cute.copy(tiled_copy_g2r_fp32, tMgAccu, tMrAccu, pred=tMCAcc_mask)
+
+ tMrDlogprobs = cute.make_fragment(tMgAccu.layout, mDlogprobs.element_type)
+ if cutlass.const_expr(self.REDUCTION == 2):
+ # mean reduction
+ num_valid_tokens = cute.make_tensor(scalarNumValidTokens, layout=(1,))
+ tMrDlogprobs[0] = mDlogprobs[0] / num_valid_tokens[0].to(cutlass.Float32)
+ elif cutlass.const_expr(self.REDUCTION == 1):
+ # sum reduction
+ tMrDlogprobs[0] = mDlogprobs[0]
+ else:
+ # no reduction
+ gDlogprobs = cute.local_tile(mDlogprobs, (self.epi_tile[0],), (pidm,))
+ tMgDlogprobs = thr_copy_g2r_fp32.partition_S(cute.append_ones(gDlogprobs))
+ cute.copy(tiled_copy_g2r_fp32, tMgDlogprobs, tMrDlogprobs, pred=tMCAcc_mask)
+
+ tMrAccu[0] = cute.arch.rcp_approx(tMrAccu[0])
+ tMrDlogprobs[0] *= tMrLabels[0] != ignore_index
+ tMr_d_acc_exp_logits = tMrDlogprobs[0] * tMrAccu[0]
+
+ # ------ Partial output ------ #
+ # [tileM, tileN]
+ gDlogits_partial = cute.local_tile(
+ mDlogits_partial, (self.epi_tile[0], self.epi_tile[1]), (pidm, pidn)
+ )
+ # blackwell supports STG.256
+ copy_atom_r2g = cute.make_copy_atom(
+ cute.nvgpu.CopyUniversalOp(),
+ gDlogits_partial.element_type,
+ num_bits_per_copy=256,
+ )
+ tiled_copy_r2g = cute.make_tiled_copy_tv(
+ copy_atom_r2g, epilogue_thread_layout, copy_atom_r2g.layout_dst_tv
+ )
+ thr_copy_r2g = tiled_copy_r2g.get_slice(tidx)
+
+ # [CPY, loopM, loopN]
+ tR2GCAcc = thr_copy_r2g.partition_S(cAcc)
+ tR2GCAcc_pred = cute.make_fragment(tR2GCAcc.shape, cutlass.Boolean)
+ for elem in cutlass.range(cute.size(tR2GCAcc_pred, mode=[0])):
+ for row in cutlass.range(cute.size(tR2GCAcc_pred, mode=[1])):
+ for col in cutlass.range(cute.size(tR2GCAcc_pred, mode=[2])):
+ tR2GCAcc_pred[elem, row, col] = cute.elem_less(
+ pidm * self.epi_tile[0] + tR2GCAcc[elem, row, col][0],
+ problem_mnk[0],
+ ) and cute.elem_less(
+ split_idx * self.vocab_per_split
+ + pidn * self.epi_tile[1]
+ + tR2GCAcc[elem, row, col][1],
+ problem_mnk[1],
+ )
+
+ tR2GgDlogits = thr_copy_r2g.partition_D(gDlogits_partial)
+
+ # for type conversion
+ dLogits_half = cute.make_fragment(tTMEM_load_rAcc.shape, tR2GgDlogits.element_type)
+ dLogits_half = cute.tiled_divide(
+ dLogits_half, (cute.size(tR2GgDlogits, mode=[0]), 1)
+ )
+ dLogits_half = cute.group_modes(dLogits_half, 2, cute.rank(dLogits_half))
+
+ mma_pipeline.consumer_wait(mma_consumer_state)
+
+ block_vocab_left_idx: cutlass.Int64 = (
+ split_idx * self.vocab_per_split + pidn * self.epi_tile[1]
+ )
+ block_vocab_right_idx: cutlass.Int64 = min(
+ split_idx * self.vocab_per_split + (pidn + 1) * self.epi_tile[1],
+ min((split_idx + 1) * self.vocab_per_split, problem_mnk[1]),
+ )
+ num_n_subtiles: cutlass.Int64 = cute.ceil_div(
+ (block_vocab_right_idx - block_vocab_left_idx),
+ cute.size(tTMEM_load_rAcc, mode=[0]),
+ )
+ for n_subtile in cutlass.range(num_n_subtiles):
+ cute.copy(
+ tiled_copy_t2r,
+ tTMEM_load_tAcc[(None, None, None, n_subtile, mma_consumer_state.index)],
+ tTMEM_load_rAcc,
+ )
+
+ for idx in cutlass.range(
+ cute.size(tTMEM_load_rAcc, mode=[0]), unroll_full=True
+ ):
+ # exp_logits
+ tTMEM_load_rAcc[idx] = cute.exp(tTMEM_load_rAcc[idx] - tMrMaximum[0])
+
+ position: cutlass.Int64 = (
+ rank * problem_mnk[1]
+ + split_idx * self.vocab_per_split
+ + pidn * self.epi_tile[1]
+ + n_subtile * cute.size(tTMEM_load_rAcc, mode=[0])
+ + idx
+ )
+ mask: cutlass.Boolean = (
+ position == tMrLabels[0] and tMrLabels[0] != ignore_index
+ )
+ # d_logits
+ tTMEM_load_rAcc[idx] *= tMr_d_acc_exp_logits
+ tTMEM_load_rAcc[idx] += mask * -tMrDlogprobs[0]
+ dLogits_half[idx] = tTMEM_load_rAcc[idx].to(dLogits_half.element_type)
+
+ for idx in cutlass.range(cute.size(dLogits_half, mode=[1]), unroll_full=True):
+ copy_id = n_subtile * cute.size(dLogits_half, mode=[1]) + idx
+ cute.copy(
+ tiled_copy_r2g,
+ dLogits_half[(None, idx, None)],
+ tR2GgDlogits[(None, None, copy_id)],
+ pred=tR2GCAcc_pred[((0, None), None, copy_id)],
+ )
+
+ mma_pipeline.consumer_release(mma_consumer_state)
+ mma_consumer_state.advance()
+
+ # ------ Deallocate TMEM ------ #
+ self.cta_sync_barrier.arrive_and_wait()
+ if warp_idx == self.empty_warp_ids[0]:
+ cute.arch.relinquish_tmem_alloc_permit()
+ cute.arch.dealloc_tmem(
+ tmem_ptr, self.tmem_alloc_cols, is_two_cta=self.use_2cta_instrs
+ )
+
+ @cute.jit
+ def __call__(
+ self,
+ split_idx: cutlass.Int32,
+ hidden: cute.Tensor,
+ weight: cute.Tensor,
+ labels: cute.Tensor,
+ dlogprobs: cute.Tensor,
+ maximum: cute.Tensor,
+ accu: cute.Tensor,
+ dlogits_partial: cute.Tensor,
+ scalarNumValidTokens: cute.Pointer,
+ ignore_index: cutlass.Int64,
+ rank: cutlass.Int32,
+ stream: cuda.CUstream,
+ ) -> None:
+ a_dtype: Type[cutlass.Numeric] = hidden.element_type
+ b_dtype: Type[cutlass.Numeric] = weight.element_type
+
+ if cutlass.const_expr(hidden.element_type != weight.element_type):
+ raise RuntimeError(
+ f"data type don't match: {hidden.element_type} v.s. {weight.element_type}"
+ )
+ if cutlass.const_expr(hidden.element_type not in [cutlass.Float16, cutlass.BFloat16]):
+ raise RuntimeError("hidden can only be FP16 or BF16")
+ if cutlass.const_expr(hidden.layout.shape[1] != weight.layout.shape[1]):
+ raise RuntimeError("K dimension doesn't match")
+
+ problem_mnk = (hidden.layout.shape[0], weight.layout.shape[0], hidden.layout.shape[1])
+ if cutlass.const_expr((problem_mnk[2] * a_dtype.width // 8) % 16 != 0):
+ raise RuntimeError(f"K dimension is not 16B aligned: {problem_mnk[2]}")
+ if cutlass.const_expr((problem_mnk[2] * b_dtype.width // 8) % 128 != 0):
+ raise RuntimeError(f"N dimension is not 128B aligned: {problem_mnk[1]}")
+
+ grid = self._compute_grid(
+ problem_mnk=problem_mnk,
+ cluster_shape_mn=self.cluster_shape_mn,
+ cta_tiler=self.mma_tiler,
+ )
+
+ a_major_mode = utils.LayoutEnum.from_tensor(hidden).mma_major_mode()
+ b_major_mode = utils.LayoutEnum.from_tensor(weight).mma_major_mode()
+
+ tiled_mma = sm100_utils.make_trivial_tiled_mma(
+ a_dtype,
+ a_major_mode,
+ b_major_mode,
+ self.acc_dtype,
+ self.cta_group,
+ self.mma_tiler[:2],
+ )
+ self._setup_attributes(tiled_mma, a_dtype, b_dtype)
+
+ self.epi_tile = self.cta_tile_shape_mnk[:2]
+
+ # Swizzle o [(tileM, tileK), loopM, loopK, stage]
+ a_smem_layout_staged = sm100_utils.make_smem_layout_a(
+ tiled_mma, self.mma_tiler, a_dtype, self.num_ab_stage
+ )
+ # Swizzle o [(tileN, tileK), loopN, loopK, stage]
+ b_smem_layout_staged = sm100_utils.make_smem_layout_b(
+ tiled_mma, self.mma_tiler, b_dtype, self.num_ab_stage
+ )
+ tma_load_op = cpasync.CopyBulkTensorTileG2SOp(self.cta_group)
+ tma_store_op = cpasync.CopyBulkTensorTileS2GOp()
+
+ # Swizzle o [(tileM, tileK), loopM, loopK]
+ a_smem_layout = cute.select(a_smem_layout_staged, mode=[0, 1, 2])
+ tma_atom_a, tma_tensor_a = cute.nvgpu.make_tiled_tma_atom_A(
+ tma_load_op,
+ hidden,
+ a_smem_layout,
+ self.mma_tiler,
+ tiled_mma,
+ self.cluster_layout_vmnk.shape,
+ )
+ # Swizzle o [(tileN, tileK), loopN, loopK]
+ b_smem_layout = cute.select(b_smem_layout_staged, mode=[0, 1, 2])
+ tma_atom_b, tma_tensor_b = cute.nvgpu.make_tiled_tma_atom_B(
+ tma_load_op,
+ weight,
+ b_smem_layout,
+ self.mma_tiler,
+ tiled_mma,
+ self.cluster_layout_vmnk.shape,
+ )
+ a_copy_size = cute.size_in_bytes(a_dtype, a_smem_layout)
+ b_copy_size = cute.size_in_bytes(b_dtype, b_smem_layout)
+ self.tma_copy_ab_bytes = a_copy_size + b_copy_size
+
+ @cute.struct
+ class SharedStorage:
+ """
+ The shared storage for the backward kernel.
+ """
+
+ load_ab_mbar_ptr: cute.struct.MemRange[cutlass.Int64, self.num_ab_stage * 2]
+ mma_mbar_ptr: cute.struct.MemRange[cutlass.Int64, self.num_acc_stage * 2]
+
+ tmem_dealloc_mbar_ptr: cute.struct.MemRange[cutlass.Int64, 1]
+ tmem_holding_buf: cutlass.Int32
+
+ sA: cute.struct.Align[
+ cute.struct.MemRange[a_dtype, cute.cosize(a_smem_layout_staged)],
+ self.buffer_align_bytes,
+ ]
+ sB: cute.struct.Align[
+ cute.struct.MemRange[b_dtype, cute.cosize(b_smem_layout_staged)],
+ self.buffer_align_bytes,
+ ]
+
+ self.shared_storage = SharedStorage
+
+ self.kernel(
+ split_idx,
+ tiled_mma,
+ tma_atom_a,
+ tma_tensor_a,
+ tma_atom_b,
+ tma_tensor_b,
+ labels,
+ dlogprobs,
+ maximum,
+ accu,
+ dlogits_partial,
+ scalarNumValidTokens,
+ ignore_index,
+ a_smem_layout_staged,
+ b_smem_layout_staged,
+ self.cluster_layout_vmnk,
+ problem_mnk,
+ rank,
+ ).launch(
+ grid=grid,
+ block=[self.threads_per_cta, 1, 1],
+ cluster=self.cluster_shape_mnk,
+ stream=stream,
+ )
+
+except ImportError:
+ logging.warning("Cutlass or CUDA bindings not found. BwdPartialDlogits will not be available.")
diff --git a/megatron/core/fusions/linear_cross_entropy/blackwell/entry.py b/megatron/core/fusions/linear_cross_entropy/blackwell/entry.py
new file mode 100644
index 00000000000..dc369a7c558
--- /dev/null
+++ b/megatron/core/fusions/linear_cross_entropy/blackwell/entry.py
@@ -0,0 +1,475 @@
+# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved.
+
+import logging
+import os
+import typing
+from dataclasses import dataclass, field
+from functools import lru_cache
+
+try:
+ import cuda.bindings.driver as cuda # type: ignore
+ import cutlass
+ import cutlass.cute as cute
+ import torch
+ import torch.distributed as dist
+ import triton # type: ignore
+ from cutlass.cute.runtime import from_dlpack
+
+ import megatron.core.fusions.linear_cross_entropy.utils as utils
+ from megatron.core.fusions.linear_cross_entropy.blackwell import (
+ bwd_partial_dlogits as bwd_partial_dlogits,
+ )
+ from megatron.core.fusions.linear_cross_entropy.blackwell import fwd_mainloop as fwd_mainloop
+ from megatron.core.fusions.linear_cross_entropy.blackwell import triton as triton_kernels
+
+ @dataclass
+ class FwdConfig:
+ """
+ The configuration for the forward pass.
+ """
+
+ _dedicated_stream: torch.cuda.Stream = field(default_factory=torch.cuda.Stream)
+ _dedicated_events: typing.List[torch.cuda.Event] = field(default_factory=list)
+ _initialized: bool = field(default=False)
+ _fwd_mainloop_kernels: typing.Dict[str, cute.kernel] = field(default_factory=dict)
+ _vocab_per_split: int = field(
+ default=int(os.environ.get("LCE_FWD_VOCAB_SPLIT_SIZE", 512 * 6))
+ )
+
+ @dataclass
+ class BwdConfig:
+ """
+ The configuration for the backward pass.
+ """
+
+ _bwd_kernel: typing.Dict[str, cute.kernel] = field(default_factory=dict)
+ _vocab_per_split: int = field(
+ default=int(os.environ.get("LCE_BWD_VOCAB_SPLIT_SIZE", 512 * 6))
+ )
+ _backward_method: utils.BackwardMethodEnum = field(
+ default=utils.BackwardMethodEnum.kDlogitsSplitN
+ )
+
+ @lru_cache(maxsize=1)
+ def _get_fwd_config() -> FwdConfig:
+ """
+ Helper function to lazy initialize the forward configuration.
+ """
+ return FwdConfig()
+
+ @lru_cache(maxsize=1)
+ def _get_bwd_config() -> BwdConfig:
+ """
+ Helper function to lazy initialize the backward configuration.
+ """
+ return BwdConfig()
+
+ def forward(
+ hidden: torch.Tensor,
+ weight: torch.Tensor,
+ labels: torch.Tensor,
+ tp_group: typing.Optional[torch.distributed.ProcessGroup] = None,
+ reduction: typing.Literal["none", "sum", "mean"] = "mean",
+ ignore_index: int = -100,
+ sequence_parallel: bool = False,
+ ) -> typing.Tuple[
+ torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, int, int, torch.Tensor
+ ]:
+ """
+ forward host function
+ """
+ tp_rank = 0 if tp_group is None else torch.distributed.get_rank(tp_group)
+ tp_world_size = 1 if tp_group is None else torch.distributed.get_world_size(tp_group)
+ in_tp_mode = (tp_group is not None) and (tp_world_size > 1)
+
+ assert hidden.is_cuda and weight.is_cuda and labels.is_cuda
+ assert weight.device == hidden.device and labels.device == hidden.device
+
+ # hidden could be [batch, seqlen, dim] or [seqlen, batch, dim] or [tokens, dim]
+ assert hidden.dim() == 2 or hidden.dim() == 3
+ # weight must be [vocab_size, dim]
+ assert weight.dim() == 2
+ # labels could be [batch, seqlen] or [seqlen, batch] or [tokens]
+ assert (hidden.dim() == 2 and labels.dim() == 1) or (
+ hidden.dim() == 3 and labels.dim() == 2
+ )
+ assert hidden.is_contiguous() and weight.is_contiguous() and labels.is_contiguous()
+
+ hidden_view = hidden.view(-1, hidden.shape[-1])
+ labels_view = labels.view(-1)
+
+ assert (
+ sequence_parallel and hidden_view.shape[0] * tp_world_size == labels_view.shape[0]
+ ) or (not sequence_parallel and hidden_view.shape[0] == labels_view.shape[0])
+ assert hidden_view.shape[1] == weight.shape[1]
+
+ global_hidden = hidden
+ if in_tp_mode and sequence_parallel:
+ partial_hidden_shape = hidden.shape
+ global_hidden_shape = (
+ partial_hidden_shape[0] * tp_world_size,
+ *partial_hidden_shape[1:],
+ )
+ global_hidden = torch.empty(
+ global_hidden_shape, dtype=hidden.dtype, device=hidden.device
+ )
+ dist.all_gather_into_tensor(global_hidden, hidden, group=tp_group)
+ assert global_hidden.is_contiguous()
+ hidden_view = global_hidden.view(-1, global_hidden.shape[-1])
+
+ num_tokens, dim = hidden_view.shape
+ vocab_size, _ = weight.shape
+
+ if not _get_fwd_config()._initialized:
+ _get_fwd_config()._dedicated_stream = torch.cuda.Stream(hidden.device)
+ _get_fwd_config()._dedicated_events = [torch.cuda.Event() for _ in range(2)]
+ _get_fwd_config()._initialized = True
+
+ REDUCTION = utils.str_to_reduction_enum(reduction)
+ # declare logprobs
+ if REDUCTION == utils.EntropyReductionEnum.kNone:
+ logprobs = torch.empty((num_tokens,), device=hidden.device, dtype=torch.float32)
+ if in_tp_mode:
+ logprobs.zero_()
+ else:
+ logprobs = torch.zeros((), device=hidden.device, dtype=torch.float32)
+ # declare auxiliary tensors
+ maximum = torch.empty((num_tokens,), device=hidden.device, dtype=torch.float32)
+ accumulate = torch.empty_like(maximum, dtype=torch.float32)
+ num_valid_tokens = torch.empty((), device=hidden.device, dtype=torch.int64)
+ assert (
+ maximum.is_contiguous()
+ and accumulate.is_contiguous()
+ and num_valid_tokens.is_contiguous()
+ )
+ # declare intermediate tensors
+ # NOTE: this is a parameter for tuning
+ num_splits = (
+ vocab_size + _get_fwd_config()._vocab_per_split - 1
+ ) // _get_fwd_config()._vocab_per_split
+ _max = torch.empty((num_tokens, num_splits), device=hidden.device, dtype=torch.float32)
+ _accu = torch.empty((num_tokens, num_splits), device=hidden.device, dtype=torch.float32)
+ if REDUCTION == utils.EntropyReductionEnum.kNone:
+ _logprobs = logprobs
+ else:
+ _logprobs = torch.empty((num_tokens,), device=hidden.device, dtype=torch.float32)
+ if in_tp_mode:
+ _logprobs.zero_()
+ assert _max.is_contiguous() and _accu.is_contiguous() and _logprobs.is_contiguous()
+
+ triton_kernels.get_num_valid_tokens[(1,)](
+ num_tokens, ignore_index, labels_view, labels_view.stride(0), num_valid_tokens
+ )
+
+ # need to compile the kernel for the first time
+ hidden_packed = from_dlpack(
+ hidden_view.detach(), assumed_align=16
+ ).mark_compact_shape_dynamic(mode=0)
+ weight_packed = from_dlpack(weight.detach(), assumed_align=16)
+ labels_packed = from_dlpack(
+ labels_view.detach(), assumed_align=8
+ ).mark_compact_shape_dynamic(mode=0)
+ logprobs_packed = from_dlpack(_logprobs, assumed_align=16).mark_compact_shape_dynamic(
+ mode=0
+ )
+ _max_packed = from_dlpack(_max, assumed_align=8).mark_compact_shape_dynamic(
+ mode=0, stride_order=(0, 1)
+ )
+ _accu_packed = from_dlpack(_accu, assumed_align=8).mark_compact_shape_dynamic(
+ mode=0, stride_order=(0, 1)
+ )
+ cuda_stream = cuda.CUstream(torch.cuda.current_stream().cuda_stream)
+
+ # VocabSize and Dim are fixed for a given model,
+ # only the number of tokens can vary
+ key = f"vocab_size:{vocab_size}+dim:{dim}+dtype:{hidden_view.dtype}"
+ if _get_fwd_config()._fwd_mainloop_kernels.get(key) is None:
+ fwd_mainloop_kernel = fwd_mainloop.FwdMainLoop(
+ vocab_per_split=_get_fwd_config()._vocab_per_split
+ )
+ fwd_mainloop_compiled_kernel = cute.compile(
+ fwd_mainloop_kernel,
+ hidden_packed,
+ weight_packed,
+ labels_packed,
+ logprobs_packed,
+ _max_packed,
+ _accu_packed,
+ ignore_index,
+ tp_rank,
+ cuda_stream,
+ )
+ _get_fwd_config()._fwd_mainloop_kernels[key] = fwd_mainloop_compiled_kernel
+ else:
+ fwd_mainloop_compiled_kernel = _get_fwd_config()._fwd_mainloop_kernels[key]
+ fwd_mainloop_compiled_kernel(
+ hidden_packed,
+ weight_packed,
+ labels_packed,
+ logprobs_packed,
+ _max_packed,
+ _accu_packed,
+ ignore_index,
+ tp_rank,
+ cuda_stream,
+ )
+
+ if not in_tp_mode:
+
+ def grid(meta):
+ return (triton.cdiv(num_tokens, meta["BLOCK_SIZE_M"]),)
+
+ triton_kernels.forward_dp_epilogue[grid](
+ num_tokens,
+ num_splits,
+ ignore_index,
+ labels_view,
+ labels_view.stride(0),
+ num_valid_tokens,
+ _max,
+ _max.stride(0),
+ _max.stride(1),
+ _accu,
+ _accu.stride(0),
+ _accu.stride(1),
+ maximum,
+ maximum.stride(0),
+ accumulate,
+ maximum.stride(0),
+ _logprobs,
+ _logprobs.stride(0),
+ logprobs,
+ triton.language.constexpr(REDUCTION.value),
+ )
+ else:
+ _max_backup = _max.clone()
+ dist.all_reduce(_max, op=dist.ReduceOp.MAX, group=tp_group)
+
+ torch.cuda.current_stream().record_event(_get_fwd_config()._dedicated_events[0])
+ with torch.cuda.stream(_get_fwd_config()._dedicated_stream):
+ _get_fwd_config()._dedicated_stream.wait_event(
+ _get_fwd_config()._dedicated_events[0]
+ )
+ dist.all_reduce(_logprobs, op=dist.ReduceOp.SUM, group=tp_group)
+ _get_fwd_config()._dedicated_stream.record_event(
+ _get_fwd_config()._dedicated_events[1]
+ )
+
+ def grid(meta):
+ return (triton.cdiv(num_tokens, meta["BLOCK_SIZE_M"]),)
+
+ triton_kernels.forward_tp_epilogue[grid](
+ num_tokens,
+ num_splits,
+ _max,
+ _max.stride(0),
+ _max.stride(1),
+ _max_backup,
+ _max_backup.stride(0),
+ _max_backup.stride(1),
+ _accu,
+ _accu.stride(0),
+ _accu.stride(1),
+ maximum,
+ maximum.stride(0),
+ accumulate,
+ maximum.stride(0),
+ )
+ # reduce accumulate
+ dist.all_reduce(accumulate, op=dist.ReduceOp.SUM, group=tp_group)
+
+ # update logprobs
+ torch.cuda.current_stream().wait_event(_get_fwd_config()._dedicated_events[1])
+ triton_kernels.forward_tp_epilogue_update_logprobs[grid](
+ num_tokens,
+ ignore_index,
+ num_valid_tokens,
+ labels_view,
+ labels_view.stride(0),
+ _logprobs,
+ _logprobs.stride(0),
+ maximum,
+ maximum.stride(0),
+ accumulate,
+ accumulate.stride(0),
+ logprobs,
+ REDUCTION.value,
+ )
+
+ return (
+ logprobs,
+ maximum,
+ accumulate,
+ num_valid_tokens,
+ tp_rank,
+ tp_world_size,
+ global_hidden,
+ )
+
+ def backward(
+ dlogprobs: torch.Tensor,
+ global_hidden: torch.Tensor,
+ weight: torch.Tensor,
+ labels: torch.Tensor,
+ maximum: torch.Tensor,
+ accu: torch.Tensor,
+ num_valid_tokens: torch.Tensor,
+ reduction: typing.Literal["none", "sum", "mean"] = "mean",
+ ignore_index: int = -100,
+ tp_group: typing.Optional[dist.ProcessGroup] = None,
+ tp_rank: int = 0,
+ tp_world_size: int = 1,
+ sequence_parallel: bool = False,
+ ) -> typing.Tuple[torch.Tensor, torch.Tensor]:
+ """
+ backward host function
+ """
+ in_tp_mode = (tp_group is not None) and (tp_world_size > 1)
+
+ hidden_view = global_hidden.view(-1, global_hidden.shape[-1])
+ labels_view = labels.view(-1)
+
+ num_tokens, dim = hidden_view.shape
+ vocab_size, _ = weight.shape
+
+ REDUCTION = utils.str_to_reduction_enum(reduction)
+ dlogprobs_view = dlogprobs.view(-1)
+ assert (
+ REDUCTION == utils.EntropyReductionEnum.kNone and dlogprobs.shape == (num_tokens,)
+ ) or (REDUCTION != utils.EntropyReductionEnum.kNone and dlogprobs.dim() == 0)
+ assert dlogprobs.is_contiguous() and dlogprobs.is_cuda
+
+ assert (
+ num_valid_tokens.dim() == 0
+ and num_valid_tokens.is_cuda
+ and num_valid_tokens.dtype == torch.int64
+ )
+
+ d_hidden = torch.empty_like(global_hidden)
+ d_weight = torch.empty_like(weight)
+ assert d_hidden.is_contiguous() and d_weight.is_contiguous()
+
+ # FIXME: implement different backward methods
+ _backward_method = _get_bwd_config()._backward_method
+ if _backward_method == utils.BackwardMethodEnum.kDlogitsSplitN:
+ vocab_per_split = _get_bwd_config()._vocab_per_split
+ num_splits = (vocab_size + vocab_per_split - 1) // vocab_per_split
+
+ _d_logits = torch.empty(
+ (num_tokens, vocab_per_split),
+ device=global_hidden.device,
+ dtype=global_hidden.dtype,
+ )
+
+ hidden_packed = from_dlpack(
+ hidden_view.detach(), assumed_align=16
+ ).mark_compact_shape_dynamic(mode=0)
+ weight_packed = from_dlpack(weight.detach(), assumed_align=16)
+ labels_packed = from_dlpack(
+ labels_view.detach(), assumed_align=8
+ ).mark_compact_shape_dynamic(mode=0)
+ dlogprobs_packed = from_dlpack(
+ dlogprobs_view.detach(), assumed_align=8
+ ).mark_compact_shape_dynamic(mode=0)
+ maximum_packed = from_dlpack(
+ maximum.detach(), assumed_align=8
+ ).mark_compact_shape_dynamic(mode=0)
+ accu_packed = from_dlpack(accu.detach(), assumed_align=8).mark_compact_shape_dynamic(
+ mode=0
+ )
+ dlogits_packed = from_dlpack(_d_logits, assumed_align=32).mark_compact_shape_dynamic(
+ mode=0
+ )
+ scalarNumValidTokens_packed = cute.runtime.make_ptr(
+ cutlass.Int64, num_valid_tokens.data_ptr(), cute.AddressSpace.gmem, assumed_align=8
+ )
+
+ stream = cuda.CUstream(torch.cuda.current_stream().cuda_stream)
+
+ key = (
+ f"vocab_size:{vocab_size}+dim:{dim}+reduction:{REDUCTION}+dtype:{hidden_view.dtype}"
+ )
+ if _get_bwd_config()._bwd_kernel.get(key) is None:
+ bwd_kernel = bwd_partial_dlogits.BwdPartialDlogits(
+ reduction=REDUCTION.value, vocab_per_split=vocab_per_split
+ )
+ bwd_kernel_compiled = cute.compile(
+ bwd_kernel,
+ 0, # split_idx
+ hidden_packed,
+ weight_packed,
+ labels_packed,
+ dlogprobs_packed,
+ maximum_packed,
+ accu_packed,
+ dlogits_packed,
+ scalarNumValidTokens_packed,
+ ignore_index,
+ tp_rank,
+ stream,
+ )
+ _get_bwd_config()._bwd_kernel[key] = bwd_kernel_compiled
+ else:
+ bwd_kernel_compiled = _get_bwd_config()._bwd_kernel.get(key)
+
+ for split_idx in range(num_splits):
+ bwd_kernel_compiled(
+ split_idx,
+ hidden_packed,
+ weight_packed,
+ labels_packed,
+ dlogprobs_packed,
+ maximum_packed,
+ accu_packed,
+ dlogits_packed,
+ scalarNumValidTokens_packed,
+ ignore_index,
+ tp_rank,
+ stream,
+ )
+ # remove padding areas
+ # cublas can handle non-contiguous tensors
+ # therefore, we do not need to contiguous the tensor
+ vocab_right_bound = (
+ min((split_idx + 1) * vocab_per_split, vocab_size) - split_idx * vocab_per_split
+ )
+ valid_d_logits = _d_logits[:, :vocab_right_bound]
+
+ torch.addmm(
+ input=d_hidden.view(-1, dim),
+ mat1=valid_d_logits,
+ mat2=weight[split_idx * vocab_per_split : (split_idx + 1) * vocab_per_split, :],
+ beta=(split_idx != 0),
+ alpha=1.0,
+ out=d_hidden.view(-1, dim),
+ )
+ torch.matmul(
+ valid_d_logits.T,
+ hidden_view,
+ out=d_weight[
+ split_idx * vocab_per_split : (split_idx + 1) * vocab_per_split, :
+ ],
+ )
+ else:
+ raise NotImplementedError(f"Unsupported backward method: {_backward_method}")
+
+ if in_tp_mode:
+ dist.all_reduce(d_hidden, op=dist.ReduceOp.SUM, group=tp_group)
+ if sequence_parallel:
+ partial_hidden_shape = (
+ global_hidden.shape[0] // tp_world_size,
+ *global_hidden.shape[1:],
+ )
+ partial_num_tokens = num_tokens // tp_world_size
+ d_hidden = d_hidden.view(-1, d_hidden.shape[-1])[
+ tp_rank * partial_num_tokens : (tp_rank + 1) * partial_num_tokens, :
+ ]
+ d_hidden = d_hidden.view(partial_hidden_shape).clone()
+
+ return d_hidden, d_weight
+
+except ImportError:
+ logging.warning(
+ "Cutlass or CUDA bindings not found. LinearCrossEntropy Blackwell entry "
+ "points will not be available."
+ )
diff --git a/megatron/core/fusions/linear_cross_entropy/blackwell/fwd_mainloop.py b/megatron/core/fusions/linear_cross_entropy/blackwell/fwd_mainloop.py
new file mode 100644
index 00000000000..93f5b9523e7
--- /dev/null
+++ b/megatron/core/fusions/linear_cross_entropy/blackwell/fwd_mainloop.py
@@ -0,0 +1,693 @@
+# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved.
+
+"""
+Implementations of the fusion lm_head(Linear) + Cross-Entropy kernel
+"""
+
+import logging
+from typing import Tuple, Type
+
+try:
+ import cuda.bindings.driver as cuda # type: ignore
+ import cutlass
+ import cutlass.cute as cute
+ import cutlass.pipeline as pipeline # type: ignore
+ import cutlass.utils as utils # type: ignore
+ import cutlass.utils.blackwell_helpers as sm100_utils # type: ignore
+ from cutlass.cute.nvgpu import cpasync, tcgen05
+
+ SM100_TMEM_CAPACITY_COLUMNS: int = 512
+
+ def make_thread_cooperative_group(size: int):
+ """
+ Create a thread cooperative group.
+ """
+ return pipeline.CooperativeGroup(pipeline.Agent.Thread, size, alignment=size)
+
+ class FwdMainLoop:
+ """
+ This class implements the mainloop for forward process.
+
+ Traits stored as attributes.
+
+ :param acc_dtype:
+ """
+
+ def __init__(
+ self,
+ acc_dtype: Type[cutlass.Numeric] = cutlass.Float32,
+ use_2cta_instrs: bool = False,
+ mma_tiler_mn: Tuple[int, int] = (128, 256),
+ vocab_per_split: int = 512,
+ ):
+ """
+ Configuration including:
+ - MMA instruction settings
+ - Cluster Shape
+ """
+ self.acc_dtype: Type[cutlass.Numeric] = acc_dtype
+ self.use_2cta_instrs = use_2cta_instrs
+ # This is the shape covered by tiledMMA, not just single MMA instruction
+ self.mma_tiler = (*mma_tiler_mn, 1)
+ self.cta_tiler = (self.mma_tiler[0], vocab_per_split, self.mma_tiler[2])
+ self.vocab_per_split = vocab_per_split
+
+ self.cta_group = tcgen05.CtaGroup.TWO if self.use_2cta_instrs else tcgen05.CtaGroup.ONE
+ self.cluster_shape_mn = (2, 1) if self.use_2cta_instrs else (1, 1)
+
+ self.occupancy = 1
+ # query SMEM capacity
+ self.smem_capacity = utils.get_smem_capacity_in_bytes("sm_100")
+
+ # the maximum columns per MMA is 256, and there is only one GEMM, so we can fully
+ # assign TMEM for that GEMM of different tiles.
+ # so 512 = 2 * 256
+
+ self.threads_per_warp: int = 32
+ # 1 warp for loading, 1 warp for issuing MMA, 1 WG for storing
+ self.epi_warp_ids = (0, 1, 2, 3)
+ self.load_warp_ids = 4
+ self.mma_warp_ids = 5
+ self.empty_warp_ids = (6, 7)
+
+ self.threads_per_cta: int = self.threads_per_warp * len(
+ (*self.epi_warp_ids, self.load_warp_ids, self.mma_warp_ids, *self.empty_warp_ids)
+ )
+
+ self.cta_sync_barrier = pipeline.NamedBarrier(
+ barrier_id=1, num_threads=self.threads_per_cta
+ )
+ self.tmem_alloc_barrier = pipeline.NamedBarrier(
+ barrier_id=2, num_threads=self.threads_per_cta
+ )
+
+ self.buffer_align_bytes: int = 1024
+ self.num_regs_other: int = 32
+ self.num_regs_epi: int = 192
+
+ def _compute_stages(
+ self,
+ tiled_mma: cute.TiledMma,
+ mma_tiler: Tuple[int, int, int],
+ a_dtype: Type[cutlass.Numeric],
+ b_dtype: Type[cutlass.Numeric],
+ ):
+ a_smem_layout_stage_one = sm100_utils.make_smem_layout_a(
+ tiled_mma, mma_tiler, a_dtype, 1 # only single stage
+ )
+ b_smem_layout_stage_one = sm100_utils.make_smem_layout_b(
+ tiled_mma, mma_tiler, b_dtype, 1
+ )
+ a_bytes_per_stage = cute.size_in_bytes(a_dtype, a_smem_layout_stage_one)
+ b_bytes_per_stage = cute.size_in_bytes(b_dtype, b_smem_layout_stage_one)
+ num_acc_stage = 2
+ num_a_stage = 4
+ num_b_stage = 4
+ num_epi_stage_per_tile = 4
+
+ return num_acc_stage, num_a_stage, num_b_stage, num_epi_stage_per_tile
+
+ def _setup_attributes(
+ self,
+ tiled_mma: cute.TiledMma,
+ a_dtype: Type[cutlass.Numeric],
+ b_dtype: Type[cutlass.Numeric],
+ ):
+ self.cluster_shape_mnk = (*self.cluster_shape_mn, 1)
+ self.cluster_layout_vmnk = cute.tiled_divide(
+ cute.make_layout(self.cluster_shape_mnk), (tiled_mma.thr_id.shape,)
+ )
+
+ # this is fixed for dense MMA, k=16
+ mma_inst_shape_k = cute.size(tiled_mma.shape_mnk, mode=[2])
+ # 16*4 = 64; 64 * sizeof(FP16) = 128Bytes
+ mma_inst_tile_k: int = 4
+ self.mma_tiler = (
+ self.mma_tiler[0],
+ self.mma_tiler[1],
+ mma_inst_shape_k * mma_inst_tile_k,
+ )
+
+ self.num_acc_stage, self.num_a_stage, self.num_b_stage, self.num_epi_stage_per_tile = (
+ self._compute_stages(tiled_mma, self.mma_tiler, a_dtype, b_dtype)
+ )
+ self.tmem_alloc_cols = self.num_acc_stage * self.mma_tiler[1]
+ assert self.tmem_alloc_cols <= SM100_TMEM_CAPACITY_COLUMNS
+
+ self.cta_tile_shape_mnk = (
+ self.mma_tiler[0] // cute.size(tiled_mma.thr_id.shape),
+ self.mma_tiler[1],
+ self.mma_tiler[2],
+ )
+
+ @cute.kernel
+ def kernel(
+ self,
+ tiled_mma: cute.TiledMma,
+ tma_atom_a: cute.CopyAtom,
+ mA: cute.Tensor,
+ tma_atom_b: cute.CopyAtom,
+ mB: cute.Tensor,
+ mLabels: cute.Tensor,
+ mMax: cute.Tensor,
+ mAccu: cute.Tensor,
+ mLogprobs: cute.Tensor,
+ a_smem_layout_staged: cute.ComposedLayout,
+ b_smem_layout_staged: cute.ComposedLayout,
+ cluster_layout_vmnk: cute.Layout,
+ problem_mnk: Tuple[int, int, int],
+ ignore_index: cutlass.Int64,
+ rank: cutlass.Int32,
+ ):
+ """
+ The forward kernel for the mainloop.
+ """
+ warp_idx = cute.arch.make_warp_uniform(cute.arch.warp_idx())
+ tidx, _, _ = cute.arch.thread_idx()
+ bidx, bidy, _ = cute.arch.block_idx()
+ # FIXME: block swizzling applied here
+ pidm, pidn = bidx, bidy
+
+ # prefetch tma descriptors
+ if warp_idx == self.load_warp_ids:
+ cute.nvgpu.cpasync.prefetch_descriptor(tma_atom_a)
+ cute.nvgpu.cpasync.prefetch_descriptor(tma_atom_b)
+
+ # declare SMEM
+ smem = utils.SmemAllocator()
+ storage = smem.allocate(self.shared_storage)
+
+ ab_pipeline = pipeline.PipelineTmaUmma.create(
+ num_stages=self.num_a_stage,
+ producer_group=make_thread_cooperative_group(len([self.load_warp_ids])),
+ consumer_group=make_thread_cooperative_group(len([self.mma_warp_ids])),
+ tx_count=self.tma_copy_a_bytes + self.tma_copy_b_bytes,
+ barrier_storage=storage.load_ab_mbar_ptr.data_ptr(),
+ )
+ ab_producer_state = pipeline.make_pipeline_state(
+ pipeline.PipelineUserType.Producer, self.num_a_stage
+ )
+ ab_consumer_state = pipeline.make_pipeline_state(
+ pipeline.PipelineUserType.Consumer, self.num_a_stage
+ )
+
+ mma_pipeline = pipeline.PipelineUmmaAsync.create(
+ num_stages=self.num_acc_stage,
+ producer_group=make_thread_cooperative_group(len([self.mma_warp_ids])),
+ consumer_group=make_thread_cooperative_group(
+ self.threads_per_warp * len(self.epi_warp_ids)
+ ),
+ barrier_storage=storage.mma_mbar_ptr.data_ptr(),
+ )
+ mma_producer_state = pipeline.make_pipeline_state(
+ pipeline.PipelineUserType.Producer, self.num_acc_stage
+ )
+ mma_consumer_state = pipeline.make_pipeline_state(
+ pipeline.PipelineUserType.Consumer, self.num_acc_stage
+ )
+
+ tmem_dealloc_mbar_ptr = storage.tmem_dealloc_mbar_ptr.data_ptr()
+ if warp_idx == self.empty_warp_ids[0]:
+ with cute.arch.elect_one():
+ cute.arch.mbarrier_init(
+ tmem_dealloc_mbar_ptr, self.threads_per_warp * len(self.epi_warp_ids)
+ )
+ cute.arch.mbarrier_init_fence()
+
+ # -------- SMEM partition ------------ #
+ # swizzle o [(tileM, tileK), loopM, loopK, Stage]
+ sA = storage.sA.get_tensor(
+ a_smem_layout_staged.outer, swizzle=a_smem_layout_staged.inner
+ )
+ # swizzle o [(tileN, tileK), loopN, loopK, stage]
+ sB = storage.sB.get_tensor(
+ b_smem_layout_staged.outer, swizzle=b_smem_layout_staged.inner
+ )
+
+ # FIXME: if 2 CTAs, modify here
+ thr_mma = tiled_mma.get_slice(0)
+ # [MMA, loopM, loopK, stage]
+ tCsA = thr_mma.make_fragment_A(sA)
+ # [MMA, loopN, loopK, stage]
+ tCsB = thr_mma.make_fragment_B(sB)
+
+ # ---------- GMEM partition ----------- #
+ # [tileM, tileK, loopK]
+ gA = cute.local_tile(mA, (self.mma_tiler[0], self.mma_tiler[2]), (pidm, None))
+
+ # [vocab_size_per_split, dim]
+ mB_n = cute.local_tile(
+ mB, (self.vocab_per_split, cute.size(mB.layout.shape, mode=[1])), (pidn, 0)
+ )
+
+ # [tileN, tileK, loopN, loopK]
+ gB = cute.local_tile(mB_n, (self.mma_tiler[1], self.mma_tiler[2]), (None, None))
+
+ # [MMA, tileCntM, tileCntK, loopK]
+ tCgA = thr_mma.partition_A(gA)
+ # [MMA, tileCntN, tileCntK, loopN, loopK]
+ tCgB = thr_mma.partition_B(gB)
+
+ a_cta_layout = cute.make_layout(cute.slice_(cluster_layout_vmnk, (0, 0, None, 0)).shape)
+ # FIXME: if 2 CTAs, modify here
+ cta_rank_in_cluster = 0
+ block_in_cluster_coord_vmnk = cluster_layout_vmnk.get_flat_coord(cta_rank_in_cluster)
+ tTMAsA, tTMAgA = cpasync.tma_partition(
+ tma_atom_a,
+ block_in_cluster_coord_vmnk[2], # cta_coord,
+ a_cta_layout,
+ cute.group_modes(sA, 0, 3), # SMEM tensor
+ cute.group_modes(tCgA, 0, 3), # GMEM tensor
+ )
+ b_cta_layout = cute.make_layout(cute.slice_(cluster_layout_vmnk, (0, None, 0, 0)).shape)
+ tTMAsB, tTMAgB = cpasync.tma_partition(
+ tma_atom_b,
+ block_in_cluster_coord_vmnk[1], # cta_coord
+ b_cta_layout,
+ cute.group_modes(sB, 0, 3),
+ cute.group_modes(tCgB, 0, 3),
+ )
+
+ # Allocate TMEM
+ tmem_holding_buf = storage.tmem_holding_buf
+ if warp_idx == self.empty_warp_ids[0]:
+ cute.arch.alloc_tmem(
+ self.tmem_alloc_cols, tmem_holding_buf, is_two_cta=self.use_2cta_instrs
+ )
+ self.cta_sync_barrier.arrive_and_wait()
+ tmem_ptr = cute.arch.retrieve_tmem_ptr(
+ self.acc_dtype, alignment=16, ptr_to_buffer_holding_addr=tmem_holding_buf
+ )
+
+ # [(tileM, tileN), loopM, loopN]
+ tmem_shape = (128, self.tmem_alloc_cols)
+ acc_shape = thr_mma.partition_shape_C(tmem_shape)
+ tCtC_fake = thr_mma.make_fragment_C(acc_shape)
+ tCtC = cute.make_tensor(tmem_ptr, tCtC_fake.layout)
+
+ block_vocab_left_idx: cutlass.Int64 = pidn * self.vocab_per_split
+ block_vocab_right_idx: cutlass.Int64 = min(
+ (pidn + 1) * self.vocab_per_split, problem_mnk[1]
+ )
+ num_n_tiles: cutlass.Int64 = cute.ceil_div(
+ (block_vocab_right_idx - block_vocab_left_idx), self.mma_tiler[1]
+ )
+
+ # ///////
+ # empty
+ # ///////
+ if warp_idx in self.empty_warp_ids:
+ cute.arch.warpgroup_reg_dealloc(self.num_regs_other)
+
+ # ///////
+ # load
+ # ///////
+ if warp_idx == self.load_warp_ids:
+ cute.arch.warpgroup_reg_dealloc(self.num_regs_other)
+
+ for n in cutlass.range(num_n_tiles):
+ for k in cutlass.range(cute.size(gA, mode=[2])):
+ ab_pipeline.producer_acquire(ab_producer_state)
+ cute.copy(
+ tma_atom_a,
+ tTMAgA[(None, k)],
+ tTMAsA[(None, ab_producer_state.index)],
+ tma_bar_ptr=ab_pipeline.producer_get_barrier(ab_producer_state),
+ )
+ cute.copy(
+ tma_atom_b,
+ tTMAgB[(None, n, k)],
+ tTMAsB[(None, ab_producer_state.index)],
+ tma_bar_ptr=ab_pipeline.producer_get_barrier(ab_producer_state),
+ )
+ ab_pipeline.producer_commit(ab_producer_state)
+ ab_producer_state.advance()
+
+ # ///////
+ # mma
+ # ///////
+ if warp_idx == self.mma_warp_ids:
+ cute.arch.warpgroup_reg_dealloc(self.num_regs_other)
+
+ for n in cutlass.range(num_n_tiles):
+ # disable accumulate for the first tile
+ tiled_mma.set(tcgen05.Field.ACCUMULATE, False)
+ mma_pipeline.producer_acquire(mma_producer_state)
+
+ for k in cutlass.range(cute.size(gA, mode=[2])):
+ ab_pipeline.consumer_wait(ab_consumer_state)
+
+ for kblock_idx in cutlass.range(
+ cute.size(tCsA, mode=[2]), unroll_full=True
+ ):
+ cute.gemm(
+ tiled_mma,
+ cute.append_ones(tCtC[(None, None, mma_producer_state.index)]),
+ tCsA[(None, None, kblock_idx, ab_consumer_state.index)],
+ tCsB[(None, None, kblock_idx, ab_consumer_state.index)],
+ cute.append_ones(tCtC[(None, None, mma_producer_state.index)]),
+ )
+ # enable accumulate for the next tile
+ tiled_mma.set(tcgen05.Field.ACCUMULATE, True)
+
+ ab_pipeline.consumer_release(ab_consumer_state)
+ ab_consumer_state.advance()
+
+ mma_pipeline.producer_commit(mma_producer_state)
+ mma_producer_state.advance()
+
+ # //////////
+ # epilogue
+ # //////////
+ if warp_idx in self.epi_warp_ids:
+ cute.arch.warpgroup_reg_alloc(self.num_regs_epi)
+
+ # epilog TMEM copy and partition
+ copy_atom_t2r = sm100_utils.get_tmem_load_op(
+ self.cta_tile_shape_mnk,
+ utils.LayoutEnum.ROW_MAJOR, # This is hard-coded
+ self.acc_dtype,
+ self.acc_dtype,
+ (self.epi_tile[0], self.epi_tile[1] // self.num_epi_stage_per_tile),
+ self.use_2cta_instrs,
+ )
+ # [tileM, subTileN, loopM, CntSubTileN, loopN]
+ tAcc_epi = cute.flat_divide(
+ tCtC[((None, None), 0, None)],
+ (self.epi_tile[0], self.epi_tile[1] // self.num_epi_stage_per_tile),
+ )
+ tiled_copy_t2r = tcgen05.make_tmem_copy(
+ copy_atom_t2r, tAcc_epi[(None, None, 0, 0, 0)]
+ )
+ thr_copy_t2r = tiled_copy_t2r.get_slice(tidx)
+ tTMEM_load_tAcc = thr_copy_t2r.partition_S(tAcc_epi)
+ # [(pattern), loopM, loopN, CntTileM, CntTileN]
+ tTMEM_load_tAcc = cute.group_modes(
+ tTMEM_load_tAcc, 3, cute.rank(tTMEM_load_tAcc) - 1
+ )
+
+ cAcc = cute.make_identity_tensor(self.mma_tiler[:2])
+ tCcAcc = thr_mma.partition_C(cAcc)
+ # [tileM, subTileN, loopM, CntSubTileN, CntTileN]
+ tCcAcc_epi = cute.flat_divide(
+ tCcAcc[((None, None), 0, None)],
+ (self.epi_tile[0], self.epi_tile[1] // self.num_epi_stage_per_tile),
+ )
+ tTMEM_load_cAcc = thr_copy_t2r.partition_D(tCcAcc_epi)
+ tTMEM_load_cAcc_shape = cute.select(tTMEM_load_cAcc.shape, mode=[0, 1, 2])
+
+ # epilogue layouts
+ epilogue_thread_layout = cute.make_layout((128, 1))
+ copy_atom_g2r = cute.make_copy_atom(
+ cute.nvgpu.CopyUniversalOp(), mLabels.element_type
+ )
+ tiled_copy_g2r = cute.make_tiled_copy(
+ copy_atom_g2r, epilogue_thread_layout, (128, 1)
+ )
+ thr_copy_g2r = tiled_copy_g2r.get_slice(tidx)
+
+ copy_atom_r2g = cute.make_copy_atom(cute.nvgpu.CopyUniversalOp(), cutlass.Float32)
+ tiled_copy_r2g = cute.make_tiled_copy(
+ copy_atom_r2g, epilogue_thread_layout, (128, 1)
+ )
+ thr_copy_r2g = tiled_copy_r2g.get_slice(tidx)
+
+ # auxiliary tensors
+ # [tileM]
+ gLabels = cute.local_tile(mLabels, (self.epi_tile[0],), (pidm,))
+
+ tLabelsCAcc = thr_copy_g2r.partition_S(cAcc)[(None, None, 0)]
+ tLabelsCAcc_mask = cute.make_fragment(tLabelsCAcc.shape, cutlass.Boolean)
+ # [(1, 1), 1]
+ tLabelsCAcc_mask[0] = cute.elem_less(pidm * self.epi_tile[0] + tidx, problem_mnk[0])
+ # to align shape with gMax and gAccu
+ tLabelsCAcc_mask = cute.append_ones(tLabelsCAcc_mask)
+
+ # [(1, 1), 1, 1]
+ tLabelsgLabels = thr_copy_g2r.partition_S(cute.append_ones(gLabels))
+ tLabelsrLabels = cute.make_fragment(
+ tLabelsgLabels.shape, tLabelsgLabels.element_type
+ )
+ cute.copy(tiled_copy_g2r, tLabelsgLabels, tLabelsrLabels, pred=tLabelsCAcc_mask)
+ valid_mask: cutlass.Boolean = (
+ tLabelsrLabels[0] != ignore_index
+ ) and tLabelsCAcc_mask[0]
+
+ # [tileM, 1]
+ gMax = cute.local_tile(mMax, (self.epi_tile[0], 1), (pidm, pidn))
+ # [(CPYM, CPYN), loopM, loopN]
+ tR2GgMax = thr_copy_r2g.partition_D(gMax)
+ tR2GrMax = cute.make_fragment(tR2GgMax.shape, tR2GgMax.element_type)
+ tR2GrMax.fill(-1e30)
+
+ # [tileM, 1]
+ gAccu = cute.local_tile(mAccu, (self.epi_tile[0], 1), (pidm, pidn))
+ # [(CPYM, CPYN), loopM, loopN]
+ tR2GgAccu = thr_copy_r2g.partition_D(gAccu)
+ tR2GrAccu = cute.make_fragment(tR2GgAccu.shape, tR2GgAccu.element_type)
+ tR2GrAccu.fill(0.0)
+
+ # [tileM, 1]
+ gLogprobs = cute.append_ones(
+ cute.local_tile(mLogprobs, (self.epi_tile[0],), (pidm,))
+ )
+ # [(CPYM, CPYN), loopM, loopN]
+ tR2GgLogprobs = thr_copy_r2g.partition_D(gLogprobs)
+ tR2GrLogprobs = cute.make_fragment(tR2GgLogprobs.shape, tR2GgLogprobs.element_type)
+ tR2GrLogprobs.fill(0.0)
+
+ # [(tileN // num_epi_stage_per_tile, 1), 1, 1]
+ tTMEM_load_rAcc = cute.make_fragment(tTMEM_load_cAcc_shape, self.acc_dtype)
+
+ for n in cutlass.range(num_n_tiles):
+ mma_pipeline.consumer_wait(mma_consumer_state)
+
+ left: cutlass.Int64 = block_vocab_left_idx + n * self.epi_tile[1]
+ right: cutlass.Int64 = min(
+ (n + 1) * self.epi_tile[1] + block_vocab_left_idx, block_vocab_right_idx
+ )
+ num_n_subtiles: cutlass.Int64 = cute.ceil_div(
+ (right - left), cute.size(tTMEM_load_rAcc, mode=[0])
+ )
+ for n_subtile in cutlass.range(num_n_subtiles):
+ cute.copy(
+ tiled_copy_t2r,
+ tTMEM_load_tAcc[
+ (None, None, None, n_subtile, mma_consumer_state.index)
+ ],
+ tTMEM_load_rAcc,
+ )
+
+ for idx in cutlass.range(
+ cute.size(tTMEM_load_rAcc, mode=[0]), unroll_full=True
+ ):
+ local_position: cutlass.Int64 = (
+ n * self.epi_tile[1]
+ + n_subtile * cute.size(tTMEM_load_rAcc, mode=[0])
+ + idx
+ )
+ if (block_vocab_left_idx + local_position) < block_vocab_right_idx:
+ _max_old = tR2GrMax[0]
+ tR2GrMax[0] = cute.arch.fmax(tR2GrMax[0], tTMEM_load_rAcc[idx])
+ exp_logits = cute.exp(tTMEM_load_rAcc[idx] - tR2GrMax[0])
+ coeff = cute.exp(_max_old - tR2GrMax[0])
+ tR2GrAccu[0] = coeff * tR2GrAccu[0] + exp_logits
+
+ position: cutlass.Int64 = (
+ rank * problem_mnk[1]
+ + pidn * self.vocab_per_split
+ + local_position
+ )
+ mask: cutlass.Boolean = valid_mask and (
+ position == tLabelsrLabels[0]
+ )
+ tR2GrLogprobs[0] += mask * tTMEM_load_rAcc[idx]
+
+ mma_pipeline.consumer_release(mma_consumer_state)
+ mma_consumer_state.advance()
+
+ cute.copy(tiled_copy_r2g, tR2GrMax, tR2GgMax, pred=tLabelsCAcc_mask)
+ cute.copy(tiled_copy_r2g, tR2GrAccu, tR2GgAccu, pred=tLabelsCAcc_mask)
+
+ vocab_left_idx: cutlass.Int64 = rank * problem_mnk[1] + pidn * self.vocab_per_split
+ vocab_right_idx: cutlass.Int64 = rank * problem_mnk[1] + min(
+ (pidn + 1) * self.vocab_per_split, problem_mnk[1]
+ )
+ valid: cutlass.Boolean = (
+ tLabelsrLabels[0] >= vocab_left_idx and tLabelsrLabels[0] < vocab_right_idx
+ )
+ tLabelsCAcc_mask[0] &= valid
+
+ cute.copy(tiled_copy_r2g, tR2GrLogprobs, tR2GgLogprobs, pred=tLabelsCAcc_mask)
+
+ # Dealloc TMEM
+ self.cta_sync_barrier.arrive_and_wait()
+ if warp_idx == self.empty_warp_ids[0]:
+ cute.arch.relinquish_tmem_alloc_permit()
+ cute.arch.dealloc_tmem(
+ tmem_ptr, self.tmem_alloc_cols, is_two_cta=self.use_2cta_instrs
+ )
+
+ @staticmethod
+ def _compute_grid(
+ problem_mnk: Tuple[int, int, int],
+ cluster_shape_mn: Tuple[int, int],
+ cta_tiler: Tuple[int, int, int],
+ num_splits: int,
+ ) -> Tuple[int, int, int]:
+
+ cluster_shape = (*cluster_shape_mn, 1)
+
+ grid = cute.round_up(
+ (cute.ceil_div(problem_mnk[0], cta_tiler[0]), num_splits, 1), cluster_shape
+ )
+ return grid
+
+ @cute.jit
+ def __call__(
+ self,
+ hidden: cute.Tensor,
+ weight: cute.Tensor,
+ labels: cute.Tensor,
+ _logprobs: cute.Tensor,
+ _max: cute.Tensor,
+ _accu: cute.Tensor,
+ ignore_index: cutlass.Int64,
+ rank: cutlass.Int32,
+ stream: cuda.CUstream,
+ ) -> None:
+ a_dtype: Type[cutlass.Numeric] = hidden.element_type
+ b_dtype: Type[cutlass.Numeric] = weight.element_type
+
+ if cutlass.const_expr(hidden.element_type != weight.element_type):
+ raise RuntimeError(
+ f"data type don't match: {hidden.element_type} v.s. {weight.element_type}"
+ )
+ if cutlass.const_expr(hidden.element_type not in [cutlass.Float16, cutlass.BFloat16]):
+ raise RuntimeError("hidden can only be FP16 or BF16")
+ if cutlass.const_expr(hidden.layout.shape[1] != weight.layout.shape[1]):
+ raise RuntimeError("K dimension doesn't match")
+
+ problem_mnk = (hidden.layout.shape[0], weight.layout.shape[0], hidden.layout.shape[1])
+ if cutlass.const_expr((problem_mnk[2] * a_dtype.width // 8) % 16 != 0):
+ raise RuntimeError(f"K dimension is not 16B aligned: {problem_mnk[2]}")
+
+ num_splits = cute.ceil_div(problem_mnk[1], self.vocab_per_split)
+
+ grid = self._compute_grid(
+ problem_mnk=problem_mnk,
+ cluster_shape_mn=self.cluster_shape_mn,
+ cta_tiler=self.cta_tiler,
+ num_splits=num_splits,
+ )
+ a_major_mode = utils.LayoutEnum.from_tensor(hidden).mma_major_mode()
+ b_major_mode = utils.LayoutEnum.from_tensor(weight).mma_major_mode()
+
+ tiled_mma = sm100_utils.make_trivial_tiled_mma(
+ a_dtype,
+ a_major_mode,
+ b_major_mode,
+ self.acc_dtype,
+ self.cta_group,
+ self.mma_tiler[:2],
+ )
+
+ self._setup_attributes(tiled_mma, a_dtype, b_dtype)
+ if cutlass.const_expr((problem_mnk[2] * a_dtype.width // 8) % 128 != 0):
+ raise RuntimeError(f"K dimension is not 128B aligned: {problem_mnk[2]}")
+
+ self.epi_tile = self.mma_tiler[:2]
+
+ # Swizzle o [(tileM, tileK), loopM, loopK, stage]
+ a_smem_layout_staged = sm100_utils.make_smem_layout_a(
+ tiled_mma, self.mma_tiler, a_dtype, self.num_a_stage
+ )
+ # Swizzle o [(tileN, tileK), loopN, loopK, stage]
+ b_smem_layout_staged = sm100_utils.make_smem_layout_b(
+ tiled_mma, self.mma_tiler, b_dtype, self.num_b_stage
+ )
+
+ # TMA loading
+ tma_load_op = cpasync.CopyBulkTensorTileG2SOp(self.cta_group)
+ tma_store_op = cpasync.CopyBulkTensorTileS2GOp()
+
+ # Swizzle o [(tileM, tileK), loopM, loopK]
+ a_smem_layout = cute.select(a_smem_layout_staged, mode=[0, 1, 2])
+ # create tma copy atom for hidden,
+ # and the cooresponding tma descriptor tensor
+ tma_atom_a, tma_desc_a = cute.nvgpu.make_tiled_tma_atom_A(
+ tma_load_op,
+ hidden, # gmem_tensor
+ a_smem_layout, # SMEM layout
+ self.mma_tiler, # MMA tiler
+ tiled_mma, # TiledMMA
+ self.cluster_layout_vmnk.shape, # cluster_shape_vmnk
+ )
+ # Swizzle o [(tileN, tileK), loopN, loopK]
+ b_smem_layout = cute.select(b_smem_layout_staged, mode=[0, 1, 2])
+ tma_atom_b, tma_desc_b = cute.nvgpu.make_tiled_tma_atom_B(
+ tma_load_op,
+ weight, # gmem_tensor
+ b_smem_layout, # SMEM layout
+ self.mma_tiler, # MMA tiler
+ tiled_mma, # TiledMMA
+ self.cluster_layout_vmnk.shape, # cluster_shape_vmnk
+ )
+ a_copy_size = cute.size_in_bytes(a_dtype, a_smem_layout)
+ b_copy_size = cute.size_in_bytes(b_dtype, b_smem_layout)
+ self.tma_copy_a_bytes = a_copy_size
+ self.tma_copy_b_bytes = b_copy_size
+
+ assert self.num_a_stage == self.num_b_stage
+
+ @cute.struct
+ class SharedStorage:
+ """
+ The shared storage for the forward kernel.
+ """
+
+ # pipeline barriers, 2 = producer + consumer
+ load_ab_mbar_ptr: cute.struct.MemRange[cutlass.Int64, self.num_a_stage * 2]
+ mma_mbar_ptr: cute.struct.MemRange[cutlass.Int64, self.num_acc_stage * 2]
+ tmem_dealloc_mbar_ptr: cute.struct.MemRange[cutlass.Int64, 1]
+ # tmem holding buffer
+ tmem_holding_buf: cutlass.Int32
+ # SMEM tensors
+ sA: cute.struct.Align[
+ cute.struct.MemRange[a_dtype, cute.cosize(a_smem_layout_staged)],
+ self.buffer_align_bytes,
+ ]
+ sB: cute.struct.Align[
+ cute.struct.MemRange[b_dtype, cute.cosize(b_smem_layout_staged)],
+ self.buffer_align_bytes,
+ ]
+
+ self.shared_storage = SharedStorage
+
+ # launch kernel
+ self.kernel(
+ tiled_mma,
+ tma_atom_a,
+ tma_desc_a,
+ tma_atom_b,
+ tma_desc_b,
+ labels,
+ _max,
+ _accu,
+ _logprobs,
+ a_smem_layout_staged,
+ b_smem_layout_staged,
+ self.cluster_layout_vmnk,
+ problem_mnk,
+ ignore_index,
+ rank,
+ ).launch(
+ grid=grid,
+ block=[self.threads_per_cta, 1, 1],
+ cluster=self.cluster_shape_mnk,
+ stream=stream,
+ )
+ return None
+
+except ImportError:
+ logging.warning("Cutlass or CUDA Python bindings not found. FwdMainLoop will not be available.")
diff --git a/megatron/core/fusions/linear_cross_entropy/blackwell/triton.py b/megatron/core/fusions/linear_cross_entropy/blackwell/triton.py
new file mode 100644
index 00000000000..e025cc046f4
--- /dev/null
+++ b/megatron/core/fusions/linear_cross_entropy/blackwell/triton.py
@@ -0,0 +1,248 @@
+# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved.
+
+import triton # type: ignore
+import triton.language as tl # type: ignore
+
+# NOTE: tl.pointer_type() is not available in Triton 3.3.0
+
+
+@triton.autotune(
+ configs=[
+ triton.Config({"BLOCK_SIZE_M": 1024}, num_stages=3, num_warps=32),
+ triton.Config({"BLOCK_SIZE_M": 2048}, num_stages=3, num_warps=32),
+ ],
+ key=["num_tokens"],
+)
+@triton.jit
+def get_num_valid_tokens(
+ num_tokens: tl.int64,
+ ignore_index: tl.int64,
+ labels_ptr, #: tl.pointer_type(tl.int64),
+ stride_labels: tl.int64,
+ num_valid_tokens_ptr, #: tl.pointer_type(tl.int64),
+ BLOCK_SIZE_M: tl.constexpr,
+):
+ """
+ Calculate the number of valid tokens in the labels tensor.
+ """
+ num_pid_m: tl.int64 = tl.cdiv(num_tokens, BLOCK_SIZE_M)
+
+ num_valid_tokens: tl.int64 = tl.zeros((), dtype=tl.int64)
+ for m in range(0, num_pid_m):
+ offs_am = m * BLOCK_SIZE_M + tl.arange(0, BLOCK_SIZE_M)
+
+ labels = tl.load(
+ labels_ptr + offs_am * stride_labels, mask=offs_am < num_tokens, other=ignore_index
+ )
+
+ valid_labels_mask = labels != ignore_index
+ num_valid_tokens += (tl.sum(valid_labels_mask.to(tl.int32), axis=0)).to(tl.int64)
+ tl.store(num_valid_tokens_ptr, num_valid_tokens)
+
+
+@triton.autotune(
+ configs=[triton.Config({"BLOCK_SIZE_M": 16, "BLOCK_SIZE_N": 64})],
+ key=["num_tokens", "num_splits"],
+)
+@triton.jit
+def forward_dp_epilogue(
+ num_tokens: tl.int64,
+ num_splits: tl.int64, # TODO: maybe this could be a constexpr
+ ignore_index: tl.int64,
+ labels_ptr, #: tl.pointer_type(tl.int64),
+ stride_labels: tl.int64,
+ num_valid_tokens_ptr, #: tl.pointer_type(tl.int64),
+ max_ptr, #: tl.pointer_type(tl.float32),
+ stride_max_m: tl.int64,
+ stride_max_n: tl.int64,
+ accu_ptr, #: tl.pointer_type(tl.float32),
+ stride_accu_m: tl.int64,
+ stride_accu_n: tl.int64,
+ global_max_ptr, #: tl.pointer_type(tl.float32),
+ stride_global_max: tl.int64,
+ global_accu_ptr, #: tl.pointer_type(tl.float32),
+ stride_global_accu: tl.int64,
+ global_logprobs_ptr, #: tl.pointer_type(tl.float32),
+ stride_global_logprobs: tl.int64,
+ global_logprobs_scalar_ptr, #: tl.pointer_type(tl.float32),
+ REDUCTION: tl.constexpr,
+ BLOCK_SIZE_M: tl.constexpr,
+ BLOCK_SIZE_N: tl.constexpr,
+):
+ """
+ forward epilogue in dp
+ """
+ pid_m = tl.program_id(axis=0)
+
+ offs_m = pid_m * BLOCK_SIZE_M + tl.arange(0, BLOCK_SIZE_M)
+ global_max = tl.zeros((BLOCK_SIZE_M,), dtype=tl.float32)
+ global_accu = tl.zeros((BLOCK_SIZE_M,), dtype=tl.float32)
+
+ for pid_n in range(0, tl.cdiv(num_splits, BLOCK_SIZE_N)):
+ offs_n = pid_n * BLOCK_SIZE_N + tl.arange(0, BLOCK_SIZE_N)
+
+ _max = tl.load(
+ max_ptr + offs_m[:, None] * stride_max_m + offs_n[None, :] * stride_max_n,
+ mask=(offs_m[:, None] < num_tokens) & (offs_n[None, :] < num_splits),
+ other=0.0,
+ )
+ _accu = tl.load(
+ accu_ptr + offs_m[:, None] * stride_accu_m + offs_n[None, :] * stride_accu_n,
+ mask=(offs_m[:, None] < num_tokens) & (offs_n[None, :] < num_splits),
+ other=0.0,
+ )
+
+ # local reduction
+ _max_old = global_max
+ _local_max = tl.max(_max, axis=1, return_indices=False)
+ global_max = tl.maximum(global_max, _local_max)
+
+ _scale = tl.exp(_max - global_max[:, None])
+ _coeff = tl.exp(_max_old - global_max)
+ global_accu = _coeff * global_accu + tl.sum(_scale * _accu, axis=1)
+
+ # store maximum
+ tl.store(global_max_ptr + offs_m * stride_global_max, global_max, mask=offs_m < num_tokens)
+ # store accumulate
+ tl.store(global_accu_ptr + offs_m * stride_global_accu, global_accu, mask=offs_m < num_tokens)
+ # update logprobs
+ labels = tl.load(
+ labels_ptr + offs_m * stride_labels, mask=offs_m < num_tokens, other=ignore_index
+ )
+ global_logprobs_ptrs = global_logprobs_ptr + offs_m * stride_global_logprobs
+ global_logprobs = tl.load(global_logprobs_ptrs, mask=offs_m < num_tokens)
+ global_logprobs = global_max + tl.log(global_accu) - global_logprobs
+ label_mask = labels != ignore_index
+ global_logprobs = tl.where(label_mask, global_logprobs, 0.0)
+
+ if REDUCTION == 0: # no-reduction
+ tl.store(global_logprobs_ptrs, global_logprobs, mask=offs_m < num_tokens)
+ elif REDUCTION == 1: # sum
+ global_logprobs_scalar = tl.sum(global_logprobs, axis=0)
+ tl.atomic_add(global_logprobs_scalar_ptr, global_logprobs_scalar)
+ elif REDUCTION == 2: # mean
+ num_valid_tokens = tl.load(num_valid_tokens_ptr)
+ global_logprobs_scalar = tl.fdiv(
+ tl.sum(global_logprobs, axis=0), num_valid_tokens.to(tl.float32)
+ )
+ tl.atomic_add(global_logprobs_scalar_ptr, global_logprobs_scalar)
+
+
+@triton.autotune(
+ configs=[triton.Config({"BLOCK_SIZE_M": 16, "BLOCK_SIZE_N": 64})],
+ key=["num_tokens", "num_splits"],
+)
+@triton.jit
+def forward_tp_epilogue(
+ num_tokens: tl.int64,
+ num_splits: tl.int64,
+ reduced_max_ptr, #: tl.pointer_type(tl.float32),
+ stride_reduced_max_m: tl.int64,
+ stride_reduced_max_n: tl.int64,
+ original_max_ptr, #: tl.pointer_type(tl.float32),
+ stride_original_max_m: tl.int64,
+ stride_original_max_n: tl.int64,
+ accu_ptr, #: tl.pointer_type(tl.float32),
+ stride_accu_m: tl.int64,
+ stride_accu_n: tl.int64,
+ global_max_ptr, #: tl.pointer_type(tl.float32),
+ stride_global_max: tl.int64,
+ global_accu_ptr, #: tl.pointer_type(tl.float32),
+ stride_global_accu: tl.int64,
+ BLOCK_SIZE_M: tl.constexpr,
+ BLOCK_SIZE_N: tl.constexpr,
+):
+ """
+ forward epilogue in tp
+ """
+ pid_m = tl.program_id(axis=0)
+
+ offs_m = pid_m * BLOCK_SIZE_M + tl.arange(0, BLOCK_SIZE_M)
+
+ global_max = tl.zeros((BLOCK_SIZE_M,), dtype=tl.float32)
+ global_accu = tl.zeros((BLOCK_SIZE_M,), dtype=tl.float32)
+
+ for pid_n in range(0, tl.cdiv(num_splits, BLOCK_SIZE_N)):
+ offs_n = pid_n * BLOCK_SIZE_N + tl.arange(0, BLOCK_SIZE_N)
+
+ _reduced_max = tl.load(
+ reduced_max_ptr
+ + offs_m[:, None] * stride_reduced_max_m
+ + offs_n[None, :] * stride_reduced_max_n,
+ mask=(offs_m[:, None] < num_tokens) & (offs_n[None, :] < num_splits),
+ other=0.0,
+ )
+ _original_max = tl.load(
+ original_max_ptr
+ + offs_m[:, None] * stride_original_max_m
+ + offs_n[None, :] * stride_original_max_n,
+ mask=(offs_m[:, None] < num_tokens) & (offs_n[None, :] < num_splits),
+ other=0.0,
+ )
+ _accu = tl.load(
+ accu_ptr + offs_m[:, None] * stride_accu_m + offs_n[None, :] * stride_accu_n,
+ mask=(offs_m[:, None] < num_tokens) & (offs_n[None, :] < num_splits),
+ other=0.0,
+ )
+
+ # local reduction
+ _max_old = global_max
+ _local_max = tl.max(_reduced_max, axis=1)
+ global_max = tl.maximum(global_max, _local_max)
+
+ # update accumulate
+ _coeff = tl.exp(_max_old - global_max)
+ _scale = tl.exp(_original_max - global_max[:, None])
+ global_accu = _coeff * global_accu + tl.sum(_scale * _accu, axis=1)
+
+ # store
+ tl.store(global_max_ptr + offs_m * stride_global_max, global_max, mask=offs_m < num_tokens)
+ tl.store(global_accu_ptr + offs_m * stride_global_accu, global_accu, mask=offs_m < num_tokens)
+
+
+@triton.autotune(configs=[triton.Config({"BLOCK_SIZE_M": 16})], key=["num_tokens"])
+@triton.jit
+def forward_tp_epilogue_update_logprobs(
+ num_tokens: tl.int64,
+ ignore_index: tl.int64,
+ num_valid_tokens_ptr, #: tl.pointer_type(tl.int64),
+ labels_ptr, #: tl.pointer_type(tl.int64),
+ stride_labels: tl.int64,
+ logprobs_ptr, #: tl.pointer_type(tl.float32),
+ stride_logprobs: tl.int64,
+ maximum_ptr, #: tl.pointer_type(tl.float32),
+ stride_maximum: tl.int64,
+ accumulate_ptr, #: tl.pointer_type(tl.float32),
+ stride_accumulate: tl.int64,
+ logprobs_scalar_ptr, #: tl.pointer_type(tl.float32),
+ REDUCTION: tl.constexpr,
+ BLOCK_SIZE_M: tl.constexpr,
+):
+ """
+ update logprobs in tp
+ """
+ pid_m = tl.program_id(axis=0)
+
+ offs_m = pid_m * BLOCK_SIZE_M + tl.arange(0, BLOCK_SIZE_M)
+
+ logprobs = tl.load(logprobs_ptr + offs_m * stride_logprobs, mask=offs_m < num_tokens)
+ maximum = tl.load(maximum_ptr + offs_m * stride_maximum, mask=offs_m < num_tokens)
+ accumulate = tl.load(accumulate_ptr + offs_m * stride_accumulate, mask=offs_m < num_tokens)
+
+ labels = tl.load(
+ labels_ptr + offs_m * stride_labels, mask=offs_m < num_tokens, other=ignore_index
+ )
+ label_mask = labels != ignore_index
+
+ logprobs = maximum + tl.log(accumulate) - logprobs
+ logprobs = tl.where(label_mask, logprobs, 0.0)
+
+ if REDUCTION == 0: # no-reduction
+ tl.store(logprobs_ptr + offs_m * stride_logprobs, logprobs, mask=offs_m < num_tokens)
+ elif REDUCTION == 1: # sum
+ logprobs_scalar = tl.sum(logprobs, axis=0)
+ tl.atomic_add(logprobs_scalar_ptr, logprobs_scalar)
+ elif REDUCTION == 2: # mean
+ num_valid_tokens = tl.load(num_valid_tokens_ptr)
+ logprobs_scalar = tl.fdiv(tl.sum(logprobs, axis=0), num_valid_tokens.to(tl.float32))
+ tl.atomic_add(logprobs_scalar_ptr, logprobs_scalar)
diff --git a/megatron/core/fusions/linear_cross_entropy/utils.py b/megatron/core/fusions/linear_cross_entropy/utils.py
new file mode 100644
index 00000000000..d077d64ab17
--- /dev/null
+++ b/megatron/core/fusions/linear_cross_entropy/utils.py
@@ -0,0 +1,43 @@
+# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved.
+
+import typing
+from enum import Enum
+
+
+class EntropyReductionEnum(Enum):
+ """
+ Enum for the reduction method of cross entropy.
+ """
+
+ kNone = 0
+ kSum = 1
+ kMean = 2
+
+
+def str_to_reduction_enum(reduction: typing.Literal["none", "sum", "mean"]) -> EntropyReductionEnum:
+ """
+ str -> EntropyReductionEnum
+ """
+ _enum = EntropyReductionEnum.kNone
+ if reduction == "none":
+ _enum = EntropyReductionEnum.kNone
+ elif reduction == "sum":
+ _enum = EntropyReductionEnum.kSum
+ elif reduction == "mean":
+ _enum = EntropyReductionEnum.kMean
+ else:
+ raise ValueError(f"Invalid reduction: {reduction}")
+ return _enum
+
+
+class BackwardMethodEnum(Enum):
+ """
+ Enum for the backward method of linear cross entropy.
+ """
+
+ # two separate kernels for d_hidden and d_weight, respectively
+ kTwoKernels = 0
+ # calculate partial d_logits along its N dimension
+ kDlogitsSplitN = 1
+ # fuse d_hidden and d_weight into a single kernel
+ kFused = 2
diff --git a/megatron/core/hyper_comm_grid.py b/megatron/core/hyper_comm_grid.py
index dce2aa16a7f..379bca69f74 100644
--- a/megatron/core/hyper_comm_grid.py
+++ b/megatron/core/hyper_comm_grid.py
@@ -160,7 +160,6 @@ def create_pg(self, dims: Union[str, list[str]], **kwargs: Any) -> dist.ProcessG
logging.info(f"Generated process group for {unique_group_key} with enumeration {rank_enum}")
self._pgs[unique_group_key] = pg
-
return pg
def get_pg(self, dims: Union[str, list[str]]) -> dist.ProcessGroup:
diff --git a/megatron/core/inference/text_generation_controllers/text_generation_controller.py b/megatron/core/inference/text_generation_controllers/text_generation_controller.py
index d908b18b4c8..15b19835121 100644
--- a/megatron/core/inference/text_generation_controllers/text_generation_controller.py
+++ b/megatron/core/inference/text_generation_controllers/text_generation_controller.py
@@ -26,6 +26,7 @@
)
from megatron.core.inference.sampling_params import SamplingParams
from megatron.core.inference.utils import get_attention_mask, set_decode_expert_padding
+from megatron.core.transformer.enums import CudaGraphScope
from megatron.core.transformer.moe.moe_layer import BaseMoELayer
from megatron.core.transformer.utils import set_model_to_sequence_parallel
from megatron.core.utils import get_asyncio_loop, get_model_config, unwrap_model
@@ -987,7 +988,7 @@ def generate_all_output_tokens_static_batch(
# Check whether CUDA graphs are enabled
enable_cuda_graph = (
model_config.cuda_graph_impl == "local"
- and model_config.cuda_graph_scope != "full_iteration"
+ and CudaGraphScope.full_iteration not in model_config.cuda_graph_scope
)
# Pad batch tokens if necessary
diff --git a/megatron/core/jit.py b/megatron/core/jit.py
index b1aa3e0b611..b67810f2e34 100644
--- a/megatron/core/jit.py
+++ b/megatron/core/jit.py
@@ -7,12 +7,27 @@
jit_fuser = torch.jit.script
# nvFuser is deprecated in PyTorch JIT starting from 2.2
-try:
- if is_torch_min_version("2.2.0a0"):
- jit_fuser = torch.compile
-except ImportError:
- def noop_decorator(func):
- return func
+def noop_decorator(func):
+ '''No-op decorator'''
+ return func
+
+def enable_jit_fuser():
+ '''Enable the JIT fuser'''
+ global jit_fuser
+ try:
+ if is_torch_min_version("2.2.0a0"):
+ jit_fuser = torch.compile
+ except ImportError:
+
+ jit_fuser = noop_decorator
+
+
+def disable_jit_fuser():
+ '''Disable the JIT fuser'''
+ global jit_fuser
jit_fuser = noop_decorator
+
+
+enable_jit_fuser()
diff --git a/megatron/core/model_parallel_config.py b/megatron/core/model_parallel_config.py
index 62d3d940481..4452bdf360b 100644
--- a/megatron/core/model_parallel_config.py
+++ b/megatron/core/model_parallel_config.py
@@ -1,4 +1,4 @@
-# Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved.
+# Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
import warnings
from dataclasses import dataclass
@@ -6,8 +6,11 @@
import torch
+from megatron.core.utils import experimental_api
+
@dataclass
+@experimental_api
class ModelParallelConfig:
"""Base configuration for Megatron Core
@@ -53,6 +56,22 @@ class ModelParallelConfig:
type.
"""
+ max_seqlen_per_dp_cp_rank: Optional[int] = None
+ """
+ Maximum sequence length per DPxCP rank. This is the maximum sequence length each rank
+ can handle without overflowing the memory. Typically, a good starting point is to set this
+ to maximum sequence length / context parallel size.
+ This is used to calculate the number and length of sub-samples assigned to
+ each rank when using hybrid_context_parallel.
+ """
+
+ hybrid_context_parallel: bool = False
+ """
+ If true, enables hybrid context parallel. This is used to balance the workload of
+ each CP rank when we use packed samples with variable sequence lengths.
+ Please set max_seqlen_per_dp_cp_rank when using hybrid_context_parallel.
+ """
+
expert_model_parallel_size: int = 1
"""Distributes Moe Experts across sub data parallel dimension."""
@@ -246,6 +265,19 @@ class ModelParallelConfig:
delay_wgrad_compute: bool = False
"""Delay the weight gradient computation to improve batch-level communication overlapping"""
+ ep_overlap_early_attn_memory_release: bool = False
+ """Enable early memory release of attention activations during EP overlap.
+ EP overlap can increase peak memory usage when the overlapped forward module allocates
+ more memory than what is freed by the backward module. This flag addresses this by
+ reordering the attention backward pass to occur earlier in the schedule.
+ Specifically:
+ - Without this flag: attn_bwd executes after moe_combine_fwd
+ - With this flag: attn_bwd executes before mlp_fwd
+ The earlier execution releases attention activations sooner, reducing peak memory.
+ Note: This may impact performance as moe_combine_fwd and moe_dispatch_bwd become
+ exposed (not overlapped with other computation).
+ """
+
###################
# Pipeline Parallel
###################
@@ -316,6 +348,10 @@ class ModelParallelConfig:
rank 1 | 0 1 2 0 1 2 3 4 3 4
"""
+ mtp_standalone: bool = False
+ """This will be set automatically according to the pipeline layout,
+ and will be set to True if MTP is in a separate vpp stage."""
+
###################
# CPU Offloading
###################
diff --git a/megatron/core/models/common/embeddings/rotary_pos_embedding.py b/megatron/core/models/common/embeddings/rotary_pos_embedding.py
index 0d7d5e626d0..5d7b69cd34e 100644
--- a/megatron/core/models/common/embeddings/rotary_pos_embedding.py
+++ b/megatron/core/models/common/embeddings/rotary_pos_embedding.py
@@ -25,7 +25,7 @@
apply_rotary_pos_emb,
get_pos_emb_on_this_cp_rank,
)
-from megatron.core.utils import deprecate_inference_params
+from megatron.core.utils import deprecate_inference_params, internal_api
logger = logging.getLogger(__name__)
@@ -148,13 +148,12 @@ def get_cos_sin(self, max_seq_len: int, offset: int = 0) -> (Tensor, Tensor):
return cos, sin
@lru_cache(maxsize=32)
- def forward(self, max_seq_len: int, offset: int = 0, packed_seq: bool = False) -> Tensor:
- """Forward pass of RoPE embedding.
+ def get_emb(self, max_seq_len: int, offset: int = 0) -> Tensor:
+ """Forward pass of RoPE embedding before CP sharding.
Args:
max_seq_len (int): Maximum size of sequence
offset (int, optional): RoPE offset. Defaults to 0.
- packed_seq (bool, optional): Whether to use packed sequence. Defaults to False.
Returns:
Tensor: Embeddings after applying RoPE.
@@ -174,10 +173,35 @@ def forward(self, max_seq_len: int, offset: int = 0, packed_seq: bool = False) -
)
# emb [seq_length, .., dim]
emb = emb[:, None, None, :]
- if self.cp_group is not None and self.cp_group.size() > 1 and not packed_seq:
- # slice rotary_pos_emb along sequence dimension and select the parition of the current
- # CP rank
- emb = get_pos_emb_on_this_cp_rank(emb, 0, self.cp_group)
+ return emb
+
+ @internal_api
+ def forward(
+ self, max_seq_len: int, offset: int = 0, packed_seq_params: Optional[PackedSeqParams] = None
+ ) -> Tensor:
+ """Forward pass of RoPE embedding.
+
+ Args:
+ max_seq_len (int): Maximum size of sequence
+ offset (int, optional): RoPE offset. Defaults to 0.
+ packed_seq_params (PackedSeqParams, optional): Packed sequence params. Defaults to None.
+
+ Returns:
+ Tensor: Embeddings after applying RoPE.
+ """
+ emb = self.get_emb(max_seq_len, offset)
+ packed_seq = packed_seq_params is not None and packed_seq_params.qkv_format == 'thd'
+ if packed_seq_params is not None and packed_seq_params.local_cp_size is not None:
+ # Set CP group to dynamic CP group for CP slicing
+ cp_group = packed_seq_params.cp_group
+ else:
+ cp_group = self.cp_group
+
+ if cp_group is not None and cp_group.size() > 1 and not packed_seq:
+ # slice rotary_pos_emb along sequence dimension
+ # and select the parition of the current CP rank
+ emb = get_pos_emb_on_this_cp_rank(emb, 0, cp_group)
+
return emb
def _load_from_state_dict(self, state_dict, prefix, *args, **kwargs):
@@ -279,13 +303,19 @@ def __init__(
else parallel_state.get_context_parallel_group(check_initialized=False)
)
- def forward(self, position_ids: torch.Tensor, mrope_section: List[int]) -> Tensor:
+ def forward(
+ self,
+ position_ids: torch.Tensor,
+ mrope_section: List[int],
+ packed_seq_params: Optional[PackedSeqParams] = None,
+ ) -> Tensor:
"""Forward pass of multimodal RoPE embedding.
Args:
position_ids (torch.Tensor): A postion_id tensor with shape [3, batchsize, seqlens]
mrope_section (list[int]): Multimodal rope section is for channel dimension of temporal,
height and width in rope calculation.
+ packed_seq_params (PackedSeqParams, optional): Packed sequence params. Defaults to None.
Returns:
Tensor: Embeddings after applying RoPE.
@@ -318,8 +348,17 @@ def forward(self, position_ids: torch.Tensor, mrope_section: List[int]) -> Tenso
# shape (seq_length, bs, 1, 2 * dim)
emb = emb[..., None, :].transpose(0, 1).contiguous()
- if self.cp_group is not None and self.cp_group.size() > 1:
+ if packed_seq_params is not None and packed_seq_params.local_cp_size is not None:
+ if packed_seq_params.local_cp_size > 1:
+ # Set CP group to dynamic CP group for CP slicing
+ cp_group = packed_seq_params.cp_group
+ else:
+ # Set CP group to None to avoid CP slicing
+ cp_group = None
+ else:
+ cp_group = self.cp_group
+ if cp_group is not None and cp_group.size() > 1:
# slice rotary_pos_emb along sequence dimension and select the parition of the current
# CP rank
- emb = get_pos_emb_on_this_cp_rank(emb, 0, self.cp_group)
+ emb = get_pos_emb_on_this_cp_rank(emb, 0, cp_group)
return emb
diff --git a/megatron/core/models/common/embeddings/yarn_rotary_pos_embedding.py b/megatron/core/models/common/embeddings/yarn_rotary_pos_embedding.py
index 79bcd144f30..7b224ec56c0 100644
--- a/megatron/core/models/common/embeddings/yarn_rotary_pos_embedding.py
+++ b/megatron/core/models/common/embeddings/yarn_rotary_pos_embedding.py
@@ -13,6 +13,7 @@
from megatron.core.models.common.embeddings.rope_utils import get_pos_emb_on_this_cp_rank
from megatron.core.models.common.embeddings.rotary_pos_embedding import RotaryEmbedding
from megatron.core.transformer import TransformerConfig
+from megatron.core.utils import internal_api
logger = logging.getLogger(__name__)
@@ -98,18 +99,17 @@ def __init__(
self.original_max_position_embeddings, offset=0, dtype=torch.get_default_dtype()
)
- # clear the lru_cache for the forward method. If not cleared, the cache of forward
+ # clear the lru_cache for the get_emb method. If not cleared, the cache of get_emb
# method causes a memory leak in NeMo-RL.
- self.forward.cache_clear()
+ self.get_emb.cache_clear()
@lru_cache(maxsize=32)
- def forward(self, max_seq_len: int, offset: int = 0, packed_seq: bool = False) -> Tensor:
+ def get_emb(self, max_seq_len: int, offset: int = 0) -> Tensor:
"""Forward pass of Yarn Rotary Embedding.
Args:
max_seq_len (int): Maximum size of sequence
offset (int, optional): RoPE offset. Defaults to 0.
- packed_seq (bool, optional): Whether to use packed sequence. Defaults to False.
Returns:
Tensor: Embeddings after applying Yarn RoPE.
@@ -155,19 +155,44 @@ def forward(self, max_seq_len: int, offset: int = 0, packed_seq: bool = False) -
emb = torch.cat((freqs, freqs), dim=-1)
# emb [seq_length, .., dim]
emb = emb[:, None, None, :]
- if self.cp_group is not None and self.cp_group.size() > 1 and not packed_seq:
+ return emb, _mscale
+
+ @internal_api
+ def forward(
+ self, max_seq_len: int, offset: int = 0, packed_seq_params: Optional[PackedSeqParams] = None
+ ) -> Tensor:
+ """Forward pass of Yarn Rotary Embedding.
+
+ Args:
+ max_seq_len (int): Maximum size of sequence
+ offset (int, optional): RoPE offset. Defaults to 0.
+ packed_seq_params (PackedSeqParams, optional): Packed sequence params. Defaults to None.
+
+ Returns:
+ Tensor: Embeddings after applying Yarn RoPE.
+ """
+ emb, _mscale = self.get_emb(max_seq_len, offset)
+ packed_seq = packed_seq_params is not None and packed_seq_params.qkv_format == 'thd'
+ if packed_seq_params is not None and packed_seq_params.local_cp_size is not None:
+ # Set CP group to dynamic CP group for CP slicing
+ cp_group = packed_seq_params.cp_group
+ else:
+ cp_group = self.cp_group
+ if cp_group is not None and cp_group.size() > 1 and not packed_seq:
# slice rotary_pos_emb along sequence dimension
# and select the parition of the current CP rank
- emb = get_pos_emb_on_this_cp_rank(emb, 0, self.cp_group)
+ emb = get_pos_emb_on_this_cp_rank(emb, 0, cp_group)
return emb, _mscale
- def _set_cos_sin_cache(self, seq_len, offset, dtype, packed_seq=False):
+ def _set_cos_sin_cache(self, seq_len, offset, dtype, packed_seq_params=None):
self.max_seq_len_cached = seq_len
self.offset_cached = offset
self.dtype_cached = dtype
- self.packed_seq_cached = packed_seq
+ self.packed_seq_cached = (
+ packed_seq_params is not None and packed_seq_params.qkv_format == 'thd'
+ )
- emb, _mscale = self.forward(seq_len, offset, packed_seq)
+ emb, _mscale = self.forward(seq_len, offset, packed_seq_params)
self.register_buffer(
"cos_cached", (emb.cos() * _mscale).to(dtype).contiguous(), persistent=False
)
@@ -176,16 +201,17 @@ def _set_cos_sin_cache(self, seq_len, offset, dtype, packed_seq=False):
)
def get_cached_cos_sin(
- self, seq_len, offset=0, dtype=torch.get_default_dtype(), packed_seq=False
+ self, seq_len, offset=0, dtype=torch.get_default_dtype(), packed_seq_params=None
):
"""Get cached cos and sin values."""
+ packed_seq = packed_seq_params is not None and packed_seq_params.qkv_format == 'thd'
if (
seq_len > self.max_seq_len_cached
or offset != self.offset_cached
or dtype != self.dtype_cached
or packed_seq != self.packed_seq_cached
):
- self._set_cos_sin_cache(seq_len, offset, dtype, packed_seq)
+ self._set_cos_sin_cache(seq_len, offset, dtype, packed_seq_params)
return (self.cos_cached[:seq_len, ...], self.sin_cached[:seq_len, ...])
diff --git a/megatron/core/models/common/language_module/language_module.py b/megatron/core/models/common/language_module/language_module.py
index 19eea55dec3..13d74aa5271 100644
--- a/megatron/core/models/common/language_module/language_module.py
+++ b/megatron/core/models/common/language_module/language_module.py
@@ -1,7 +1,7 @@
-# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved.
+# Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
import logging
import os
-from typing import Optional, Tuple
+from typing import Any, Dict, Literal, Optional, Tuple
import torch
from torch import Tensor
@@ -14,6 +14,7 @@
except:
te_parallel_cross_entropy = None
from megatron.core.fusions.fused_cross_entropy import fused_vocab_parallel_cross_entropy
+from megatron.core.fusions.fused_linear_cross_entropy import linear_cross_entropy
from megatron.core.pipeline_parallel.utils import (
is_pp_first_stage,
is_pp_last_stage,
@@ -21,7 +22,7 @@
is_vp_last_stage,
)
from megatron.core.process_groups_config import ProcessGroupCollection
-from megatron.core.transformer.enums import AttnBackend
+from megatron.core.transformer.enums import AttnBackend, CudaGraphScope
from megatron.core.transformer.module import MegatronModule
from megatron.core.transformer.transformer_config import TransformerConfig
from megatron.core.transformer.utils import ensure_metadata_has_dp_cp_group
@@ -68,6 +69,8 @@ def _is_in_embd_group(self):
if torch.distributed.get_rank() in torch.distributed.get_process_group_ranks(
self.embd_group
):
+ if getattr(self, 'mtp_process', False):
+ return True
if (
torch.distributed.get_rank()
== torch.distributed.get_process_group_ranks(self.embd_group)[0]
@@ -123,6 +126,68 @@ def check_and_set_env_variable(
check_and_set_env_variable("NVTE_FUSED_ATTN", 1, AttnBackend.auto)
check_and_set_env_variable("NVTE_UNFUSED_ATTN", 1, AttnBackend.auto)
+ def compute_output_layer_and_language_model_loss(
+ self,
+ hidden: Tensor,
+ labels: Optional[Tensor],
+ weight: Tensor = None,
+ sequence_parallel_enabled: bool = False,
+ column_parallel_linear: torch.nn.Module = None,
+ col_linear_kwargs: Dict[str, Any] = {},
+ reduction: Literal["none", "sum", "mean"] = "none",
+ ignore_index: int = -100,
+ ) -> Tensor:
+ """Computes the language model logits and loss (Cross entropy across vocabulary)
+
+ Args:
+ hidden (Tensor): The hidden states from the transformer model
+ labels (Optional[Tensor]): The labels of dimension [batch size, seq length]
+ weight (Tensor): The weight tensor of shape [vocab size, hidden size].
+ Required if using fused linear cross entropy.
+ column_parallel_linear (torch.nn.Module): The column parallel linear
+ layer to use for computing logits when not using fused linear cross entropy.
+ col_linear_kwargs (Dict[str, Any]): Additional kwargs for column parallel linear layer
+ reduction (Optional[str]): The reduction method. Defaults to "none", and can be
+ one of "none", "sum", "mean".
+ ignore_index (Optional[int]): The index to ignore in the loss calculation.
+ Defaults to -100.
+
+ Returns:
+ Tensor: Loss tensor of dimensions [batch size, sequence_length].
+ """
+ if (
+ self.config.cross_entropy_loss_fusion
+ and self.config.cross_entropy_fusion_impl == 'linear'
+ ):
+ assert (
+ weight is not None
+ ), "weight cannot be None when using fused linear cross entropy."
+ assert (
+ labels is not None
+ ), "labels cannot be None when using fused linear cross entropy."
+ # [b s] => [s b]
+ labels = labels.transpose(0, 1).contiguous()
+ loss = linear_cross_entropy(
+ hidden,
+ weight,
+ labels,
+ tp_group=self.pg_collection.tp,
+ sequence_parallel=sequence_parallel_enabled,
+ reduction=reduction,
+ ignore_index=ignore_index,
+ )
+
+ # [s b] => [b, s]
+ loss = loss.view_as(labels).transpose(0, 1).contiguous()
+ return loss
+ else:
+ assert (
+ column_parallel_linear is not None
+ ), "column_parallel_linear cannot be None when not using fused linear cross entropy."
+ logits, _ = column_parallel_linear(hidden, **col_linear_kwargs)
+
+ return self.compute_language_model_loss(labels, logits)
+
def compute_language_model_loss(self, labels: Tensor, logits: Tensor) -> Tensor:
"""Computes the language model loss (Cross entropy across vocabulary)
@@ -142,7 +207,7 @@ def compute_language_model_loss(self, labels: Tensor, logits: Tensor) -> Tensor:
# Use is_cg_capturable=True for full iteration CUDA graphs to avoid torch.equal checks
is_cg_capturable = (
hasattr(self.config, 'cuda_graph_scope')
- and self.config.cuda_graph_scope == 'full_iteration'
+ and CudaGraphScope.full_iteration in self.config.cuda_graph_scope
)
if is_cg_capturable and not is_te_min_version("2.7.0"):
from megatron.core.utils import get_te_version
@@ -207,7 +272,10 @@ def setup_embeddings_and_output_layer(self) -> None:
):
self.shared_embedding_or_output_weight().shared_embedding = True
- if (self.post_process or getattr(self, 'mtp_process', False)) and not self.pre_process:
+ if (
+ (self.post_process and self.share_embeddings_and_output_weights)
+ or getattr(self, 'mtp_process', False)
+ ) and not self.pre_process:
assert not (
is_vp_first_stage(self.vp_stage, self.vp_size) and is_pp_first_stage(self.pp_group)
)
@@ -307,7 +375,7 @@ def tie_embeddings_and_output_weights_state_dict(
sharded_state_dict: ShardedStateDict,
output_layer_weight_key: str,
first_stage_word_emb_key: str,
- metadata: dict = {},
+ metadata: Optional[dict] = None,
) -> None:
"""Ties the embedding and output weights in a given sharded state dict.
@@ -317,9 +385,11 @@ def tie_embeddings_and_output_weights_state_dict(
This entry will be replaced with a tied version
first_stage_word_emb_key (str): this must be the same as the
ShardedTensor.key of the first stage word embeddings.
+ metadata (Optional[Dict]): metadata controlling sharded state dict creation.
Returns: None, acts in-place
"""
+ metadata = ensure_metadata_has_dp_cp_group(metadata)
if not self.post_process:
# No output layer
assert output_layer_weight_key not in sharded_state_dict, sharded_state_dict.keys()
diff --git a/megatron/core/models/common/model_chunk_schedule_plan.py b/megatron/core/models/common/model_chunk_schedule_plan.py
index d501c11a0a9..0c29423edab 100644
--- a/megatron/core/models/common/model_chunk_schedule_plan.py
+++ b/megatron/core/models/common/model_chunk_schedule_plan.py
@@ -14,7 +14,7 @@
get_comm_stream,
get_comp_stream,
)
-from megatron.core.transformer.multi_token_prediction import get_mtp_num_layers_to_build
+from megatron.core.transformer.enums import CudaGraphScope
class ModelChunkState:
@@ -35,23 +35,20 @@ class TransformerLayerSchedulePlan:
mtp post process nodes.
layer (TransformerLayerSchedulePlan)
- ├── attn (TransformerLayerNode): attention module
- ├── post_attn (TransformerLayerNode): layernorm -> router -> dispatch preprocess
+ ├── attn (TransformerLayerNode): attention -> router -> dispatch preprocess
├── moe_dispatch (TransformerLayerNode): dispatch All2All
├── mlp (TransformerLayerNode): mlp module
├── moe_combine (TransformerLayerNode): combine All2All
└── mtp_post_process (PostProcessNode): mtp post process
Note that MTP layer has the same operation and execution order with TransformerLayer regarding
- post_attn, moe_dispatch, mlp, moe_combine, but contains extra operations in attn and
- mtp_post_process:
+ moe_dispatch, mlp, moe_combine, but contains extra operations in attn and mtp_post_process:
* mtp.attn wraps around transformer_layer.attn with extra norm, proj and embedding operations.
* mtp.mtp_post_process contains output_layer, mtp loss operations, whereas
transformer_layer.mtp_post_process is empty.
"""
attn = None
- post_attn = None
moe_dispatch = None
mlp = None
moe_combine = None
@@ -75,6 +72,7 @@ def __init__(self, layer, event, chunk_state, comp_stream, comm_stream, extra_ar
"""
from megatron.core.models.gpt.fine_grained_callables import TransformerLayerState
+ self.config = layer.config
self.layer_state = TransformerLayerState()
self.chunk_state = chunk_state
self.layer = layer
@@ -85,10 +83,36 @@ def __init__(self, layer, event, chunk_state, comp_stream, comm_stream, extra_ar
# get callable nodes for transformer/mtp layer
self._build_callable_nodes(event, comp_stream, comm_stream, extra_args)
+ def release_state(self):
+ """Release reference, this helps avoid memory leak."""
+ if hasattr(self, 'attn') and self.attn is not None:
+ del self.attn
+ self.attn = None
+ if hasattr(self, 'post_attn') and self.post_attn is not None:
+ del self.post_attn
+ self.post_attn = None
+ if hasattr(self, 'moe_dispatch') and self.moe_dispatch is not None:
+ del self.moe_dispatch
+ self.moe_dispatch = None
+ if hasattr(self, 'mlp') and self.mlp is not None:
+ del self.mlp
+ self.mlp = None
+ if hasattr(self, 'moe_combine') and self.moe_combine is not None:
+ del self.moe_combine
+ self.moe_combine = None
+ if hasattr(self, 'mtp_post_process') and self.mtp_post_process is not None:
+ del self.mtp_post_process
+ self.mtp_post_process = None
+ if hasattr(self, 'layer_state') and self.layer_state is not None:
+ del self.layer_state
+ self.layer_state = None
+ if hasattr(self, 'layer'):
+ del self.layer
+
def _build_callable_nodes(self, event, comp_stream, comm_stream, extra_args):
"""
Builds the callable nodes for the transformer/mtp layer:
- attn, post_attn, mlp, moe_dispatch and moe_combine, and mtp_post_process.
+ attn, mlp, moe_dispatch and moe_combine, and mtp_post_process.
"""
from megatron.core.models.gpt.fine_grained_callables import (
TransformerLayerNode,
@@ -108,11 +132,7 @@ def _build_callable_nodes(self, event, comp_stream, comm_stream, extra_args):
else isinstance(self.layer.mlp, MoELayer)
)
- enable_deepep = (
- self.layer.config.moe_token_dispatcher_type == "flex"
- and self.layer.config.moe_flex_dispatcher_backend == "deepep"
- )
- extra_args["enable_deepep"] = enable_deepep
+ extra_args["config"] = self.layer.config
extra_args["is_moe"] = is_moe
extra_args["delay_wgrad_compute"] = self.layer.config.delay_wgrad_compute
extra_args["is_mtp"] = is_mtp
@@ -133,7 +153,6 @@ def create_node(stream, module, name):
(
attn_module,
- post_attn_module,
moe_dispatch_module,
mlp_module,
moe_combine_module,
@@ -145,11 +164,9 @@ def create_node(stream, module, name):
self.attn = create_node(comp_stream, attn_module, "attn")
self.mlp = create_node(comp_stream, mlp_module, "mlp")
if is_moe:
- self.post_attn = create_node(comp_stream, post_attn_module, "post_attn")
self.moe_dispatch = create_node(comm_stream, moe_dispatch_module, "moe_dispatch")
self.moe_combine = create_node(comm_stream, moe_combine_module, "moe_combine")
else:
- self.post_attn = NoopScheduleNode()
self.moe_dispatch = NoopScheduleNode()
self.moe_combine = NoopScheduleNode()
@@ -160,6 +177,11 @@ def create_node(stream, module, name):
else:
self.mtp_post_process = NoopScheduleNode()
+ # mlp and combine may receive dgrad from attn, which is managed by cuda graph.
+ if CudaGraphScope.attn in self.config.cuda_graph_scope:
+ self.mlp.manual_grads_release = False
+ self.moe_combine.manual_grads_release = False
+
def get_fp8_context(self):
"""
Get the fp8 context for the transformer layer.
@@ -182,8 +204,8 @@ def run(f_layer, b_layer, f_input=None, b_grad=None, is_last_layer_in_bwd=False)
to maximize parallelism and efficiency.
When f_layer and b_layer are not None, forward and backward pass are overlapped as follows:
- comm_stream: combine_bwd | dispatch_fwd->dispatch_bwd | combine_fwd
- comp_stream: attn_fwd->post_attn_fwd| mlp_bwd->mlp_bwd_dw->mlp_fwd| post_attn_bwd->attn_bwd
+ comm_stream: combine_bwd | dispatch_fwd->dispatch_bwd | combine_fwd
+ comp_stream: attn_fwd | mlp_bwd->mlp_bwd_dw->mlp_fwd| attn_bwd
For MTP, mtp_post_process_fwd is executed after the combine_fwd in the comp_stream,
and mtp_post_process_bwd is executed before the combine_bwd in the comp_stream.
@@ -206,7 +228,6 @@ def run(f_layer, b_layer, f_input=None, b_grad=None, is_last_layer_in_bwd=False)
if f_layer is not None:
with f_layer.get_fp8_context():
f_input = f_layer.attn.forward(f_input)
- f_input = f_layer.post_attn.forward(f_input)
if b_layer is not None:
b_grad = b_layer.mlp.backward(b_grad)
@@ -219,6 +240,9 @@ def run(f_layer, b_layer, f_input=None, b_grad=None, is_last_layer_in_bwd=False)
b_layer.mlp.backward_dw()
b_grad = b_layer.moe_dispatch.backward(b_grad)
+ if b_layer is not None and b_layer.config.ep_overlap_early_attn_memory_release:
+ b_grad = b_layer.attn.backward(b_grad)
+
if f_layer is not None:
with f_layer.get_fp8_context():
f_input = f_layer.mlp.forward(f_input)
@@ -228,8 +252,7 @@ def run(f_layer, b_layer, f_input=None, b_grad=None, is_last_layer_in_bwd=False)
f_input = f_layer.moe_combine.forward(f_input)
f_input = f_layer.mtp_post_process.forward(f_input)
- if b_layer is not None:
- b_grad = b_layer.post_attn.backward(b_grad)
+ if b_layer is not None and not b_layer.config.ep_overlap_early_attn_memory_release:
b_grad = b_layer.attn.backward(b_grad)
# Delay the last attn_dw in backward pass (attn_dw of the first layer)
@@ -267,6 +290,7 @@ def __init__(
extra_block_kwargs=None,
runtime_gather_output: Optional[bool] = None,
loss_mask: Optional[Tensor] = None,
+ padding_mask=None,
):
"""Initialize the schedule plan of all Transformer layers' sub-modules.
@@ -309,6 +333,7 @@ def __init__(
self._model_chunk_state.mtp_hidden_states = None
self._model_chunk_state.loss_mask = loss_mask
self._model_chunk_state.packed_seq_params = packed_seq_params
+ self._model_chunk_state.padding_mask = padding_mask
self._model_chunk_state.extra_block_kwargs = extra_block_kwargs
self._model_chunk_state.runtime_gather_output = runtime_gather_output
self._model_chunk_state.model = model
@@ -316,37 +341,44 @@ def __init__(
self._model_chunk_state.context_mask = None
self._model_chunk_state.attention_bias = None
- transformer_num_layers = model.decoder.num_layers_per_pipeline_rank
- mtp_num_layers = get_mtp_num_layers_to_build(model.config, vp_stage=self.vp_stage)
-
# build preprocess
self.pre_process = PreProcessNode(model, self._model_chunk_state, self._event, comp_stream)
- # build layer schedule plan for each layer
- for layer_idx in range(transformer_num_layers):
- layer = model.decoder._get_layer(layer_idx)
- layer_plan = TransformerLayerSchedulePlan(
- layer, self._event, self._model_chunk_state, comp_stream, comm_stream
+
+ # build layer schedule plan for each layer.
+ # The methods to obtain layers are different for MTP so we need the other build plan for
+ # MTP. Also, this can help annotate MTP layer so that it can know where MTP is.
+ self._build_layer_schedule_plan(model.decoder, comp_stream, comm_stream)
+ self._build_layer_schedule_plan(getattr(model, "mtp", None), comp_stream, comm_stream)
+
+ # build post process
+ if model.post_process:
+ self.post_process = PostProcessNode(
+ model, self._model_chunk_state, self._event, comp_stream
)
- self._transformer_layers.append(layer_plan)
- # build mtp layers
- for layer_idx in range(mtp_num_layers):
+ # preprocess may receive dgrad from attn, which is managed by cuda graph.
+ if CudaGraphScope.attn in model.config.cuda_graph_scope:
+ self.pre_process.manual_grads_release = False
+
+ def _build_layer_schedule_plan(self, module, comp_stream, comm_stream):
+ if module is None:
+ return
+ num_layers = len(module.layers)
+ for layer_idx in range(num_layers):
extra_args = {
"is_first_layer": layer_idx == 0,
- "is_last_layer": layer_idx == mtp_num_layers - 1,
+ "is_last_layer": layer_idx == num_layers - 1,
}
- layer = model.mtp.layers[layer_idx]
layer_plan = TransformerLayerSchedulePlan(
- layer, self.event, self.state, comp_stream, comm_stream, extra_args
+ module.layers[layer_idx],
+ self.event,
+ self.state,
+ comp_stream,
+ comm_stream,
+ extra_args,
)
self._transformer_layers.append(layer_plan)
- # build post process
- if model.post_process:
- self.post_process = PostProcessNode(
- model, self._model_chunk_state, self._event, comp_stream
- )
-
@property
def event(self):
"""Gets the CUDA event for synchronization."""
@@ -367,6 +399,10 @@ def get_layer(self, i):
assert i < self.num_layers()
return self._transformer_layers[i]
+ def pop_layer(self):
+ """Pops the transformer layer in FILO order."""
+ return self._transformer_layers.pop()
+
def num_layers(self):
"""Gets the number of transformer layers."""
return len(self._transformer_layers)
@@ -445,11 +481,12 @@ def run(
b_num_layers = b_schedule_plan.num_layers() if b_schedule_plan is not None else 0
overlapped_layers = min(f_num_layers, b_num_layers)
+ f_layer = b_layer = None
# combined forward and backward pass for overlapped layers
for i in range(overlapped_layers):
f_layer = f_schedule_plan.get_layer(i)
- b_layer = b_schedule_plan.get_layer(b_num_layers - 1 - i)
- torch.cuda.nvtx.range_push(f"layer_{i}f-layer_{b_num_layers - 1 - i}b")
+ b_layer = b_schedule_plan.pop_layer()
+ torch.cuda.nvtx.range_push(f"layer_{i}f-layer_{b_schedule_plan.num_layers()}b")
f_input, b_grad = TransformerLayerSchedulePlan.run(
f_layer,
b_layer,
@@ -457,15 +494,19 @@ def run(
b_grad=b_grad,
is_last_layer_in_bwd=(i == b_num_layers - 1),
)
+ if i < b_num_layers - 1:
+ b_layer.release_state()
torch.cuda.nvtx.range_pop()
# backward pass for the remaining layers
for i in range(overlapped_layers, b_num_layers):
- b_layer = b_schedule_plan.get_layer(b_num_layers - 1 - i)
- torch.cuda.nvtx.range_push(f"layer_{b_num_layers - 1 - i}b")
+ b_layer = b_schedule_plan.pop_layer()
+ torch.cuda.nvtx.range_push(f"layer_{b_schedule_plan.num_layers()}b")
_, b_grad = TransformerLayerSchedulePlan.run(
None, b_layer, b_grad=b_grad, is_last_layer_in_bwd=(i == b_num_layers - 1)
)
+ if i < b_num_layers - 1:
+ b_layer.release_state()
torch.cuda.nvtx.range_pop()
# forward pass for the remaining layers
@@ -491,7 +532,9 @@ def run(
# Delay the last attn_dw in backward pass (attn_dw of the first layer)
# for overlapping with the p2p comm
if b_num_layers > 0:
- b_schedule_plan.get_layer(0).attn.backward_dw()
+ assert b_layer is not None
+ b_layer.attn.backward_dw()
+ b_layer.release_state()
# post process forward
if f_schedule_plan is not None and f_schedule_plan.post_process is not None:
@@ -504,9 +547,7 @@ def run(
f_schedule_plan.wait_current_stream()
if b_schedule_plan:
b_schedule_plan.wait_current_stream()
-
- # Release reference as early as possible, this helps avoid memory leak.
- if b_schedule_plan is not None:
+ # Release reference as early as possible, this helps avoid memory leak.
b_schedule_plan.release_state()
return f_input
diff --git a/megatron/core/models/gpt/experimental_attention_variant_module_specs.py b/megatron/core/models/gpt/experimental_attention_variant_module_specs.py
new file mode 100644
index 00000000000..7649a0b2165
--- /dev/null
+++ b/megatron/core/models/gpt/experimental_attention_variant_module_specs.py
@@ -0,0 +1,487 @@
+# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved.
+
+from typing import List, Optional
+
+from megatron.core.fusions.fused_bias_dropout import get_bias_dropout_add
+from megatron.core.models.backends import BackendSpecProvider
+from megatron.core.ssm.gated_delta_net import GatedDeltaNet, GatedDeltaNetSubmodules
+from megatron.core.transformer.enums import AttnMaskType, LayerType
+from megatron.core.transformer.experimental_attention_variant.dsa import (
+ DSAIndexer,
+ DSAIndexerSubmodules,
+ DSAttention,
+ DSAttentionSubmodules,
+)
+from megatron.core.transformer.identity_op import IdentityOp
+from megatron.core.transformer.multi_latent_attention import (
+ MLASelfAttention,
+ MLASelfAttentionSubmodules,
+)
+from megatron.core.transformer.spec_utils import ModuleSpec
+from megatron.core.transformer.transformer_block import (
+ TransformerBlockSubmodules,
+ get_num_layers_to_build,
+)
+from megatron.core.transformer.transformer_config import TransformerConfig
+from megatron.core.transformer.transformer_layer import (
+ TransformerLayer,
+ TransformerLayerSubmodules,
+ get_transformer_layer_offset,
+)
+
+try:
+ import transformer_engine as te # type: ignore[import-untyped] # pylint: disable=unused-import
+
+ from megatron.core.extensions.transformer_engine_spec_provider import TESpecProvider
+
+ HAVE_TE = True
+except ImportError:
+ HAVE_TE = False
+
+try:
+ import nvidia_kitchen # type: ignore[import-not-found] # pylint: disable=unused-import
+
+ from megatron.core.extensions.kitchen import KitchenSpecProvider
+
+ HAVE_KITCHEN = True
+except ImportError:
+ HAVE_KITCHEN = False
+
+
+##########
+# Experimental Attention Variant Module Specs
+##########
+
+
+def get_gated_delta_net_module_spec(
+ config: TransformerConfig, backend: BackendSpecProvider = None
+) -> ModuleSpec:
+ """Build module spec for GatedDeltaNet attention."""
+
+ if backend is None:
+ backend = _get_backend_spec_provider(config=config)
+
+ rms_norm = config.normalization == "RMSNorm"
+ attention = ModuleSpec(
+ module=GatedDeltaNet,
+ submodules=GatedDeltaNetSubmodules(
+ in_proj=backend.column_parallel_layer_norm_linear(),
+ out_norm=backend.layer_norm(rms_norm=rms_norm, for_qk=False),
+ out_proj=backend.row_parallel_linear(),
+ ),
+ metainfo={"fuse_input_layernorm": True},
+ )
+ return attention
+
+
+def get_dsa_module_spec_for_backend(
+ config: TransformerConfig, backend: BackendSpecProvider = None
+) -> ModuleSpec:
+ """Helper function to get module spec for Sparse Attention."""
+ assert config.multi_latent_attention, "Currently only MLA supports sparse attention."
+ assert config.qk_l2_norm is False, "qk_l2_norm is not supported with MLA."
+
+ linear_q_up_proj = (
+ backend.column_parallel_layer_norm_linear()
+ if config.qk_layernorm
+ else backend.column_parallel_linear()
+ )
+ linear_kv_up_proj = (
+ backend.column_parallel_layer_norm_linear()
+ if config.qk_layernorm
+ else backend.column_parallel_linear()
+ )
+
+ # Because TransformerEngine does not support sparse attention yet, we use local
+ # implementation whether the backend is TransformerEngine or not.
+ core_attention = ModuleSpec(
+ module=DSAttention,
+ submodules=DSAttentionSubmodules(
+ indexer=ModuleSpec(
+ module=DSAIndexer,
+ submodules=DSAIndexerSubmodules(
+ linear_wq_b=backend.linear(),
+ linear_wk=backend.linear(),
+ k_norm=backend.layer_norm(rms_norm=False, for_qk=True),
+ linear_weights_proj=backend.linear(),
+ ),
+ )
+ ),
+ )
+
+ attention = ModuleSpec(
+ module=MLASelfAttention,
+ params={"attn_mask_type": AttnMaskType.causal},
+ submodules=MLASelfAttentionSubmodules(
+ linear_q_proj=backend.column_parallel_linear(),
+ linear_q_down_proj=backend.linear(),
+ linear_q_up_proj=linear_q_up_proj,
+ linear_kv_down_proj=backend.linear(),
+ linear_kv_up_proj=linear_kv_up_proj,
+ core_attention=core_attention,
+ linear_proj=backend.row_parallel_linear(),
+ q_layernorm=IdentityOp,
+ kv_layernorm=IdentityOp,
+ ),
+ metainfo={"fuse_input_layernorm": False},
+ )
+
+ return attention
+
+
+def get_experimental_attention_variant_module_spec(
+ config: TransformerConfig, backend: BackendSpecProvider = None
+) -> ModuleSpec:
+ """Helper function to get module spec for experimental attention variant"""
+
+ if backend is None:
+ backend = _get_backend_spec_provider(config=config)
+
+ if config.experimental_attention_variant == "gated_delta_net":
+ return get_gated_delta_net_module_spec(config=config, backend=backend)
+ else:
+ raise ValueError(
+ f"Invalid experimental attention variant: {config.experimental_attention_variant}"
+ )
+
+
+##########
+# Experimental GPT Decoder Block Spec
+##########
+
+
+def get_transformer_layer_with_experimental_attention_variant_spec(
+ config: TransformerConfig, backend: BackendSpecProvider = None
+) -> List[ModuleSpec]:
+ """Build transformer layer specs with experimental attention variants (e.g., linear attention).
+
+ This function is for constructing a heterogeneous transformer that supports mixing different
+ attention mechanisms (experimental vs standard) and MLP types (MoE vs dense) across layers.
+ **Note that, this API is a experimental API in the short term, and might be deprecated in the
+ future. In the long run, we will move to a new design that better support hybrid models.**
+
+ Key Design:
+ 1. Attention and MLP patterns: The attention pattern and MLP pattern are orthogonal
+ and determined independently. This allows flexible combinations (e.g., linear attention
+ with MoE, or standard attention with dense MLP).
+ - Attention pattern: derived from `config.linear_attention_freq` or
+ `config.experimental_attention_variant`.
+ - MLP pattern: derived from `config.moe_layer_freq`.
+
+ 2. Per-Layer Spec Construction: Iterates through layers, constructing transformer
+ layer specs based on attention and MLP patterns.
+
+ Args:
+ config: Transformer configuration containing model hyperparameters and feature flags.
+
+ Returns:
+ List[ModuleSpec] containing per-layer specs.
+
+ Note:
+ Currently only supports transformer_engine backend. Kitchen backend can be used as a
+ wrapper with TE fallback for unsupported operations.
+ """
+
+ if backend is None:
+ backend = _get_backend_spec_provider(config=config)
+
+ # Get attention patterns and specs
+ experimental_attention_pattern = [0] * config.num_layers
+ if is_linear_attention_variant(config.experimental_attention_variant):
+ experimental_attention_pattern = get_linear_attention_pattern(config=config)
+ elif config.experimental_attention_variant is not None:
+ experimental_attention_pattern = [1] * config.num_layers
+
+ if 1 in experimental_attention_pattern:
+ experimental_attention_spec = get_experimental_attention_variant_module_spec(
+ config=config, backend=backend
+ )
+ else:
+ experimental_attention_spec = None
+
+ if 0 in experimental_attention_pattern:
+ standard_attention_spec = _get_self_attention_module_spec(config=config, backend=backend)
+ else:
+ standard_attention_spec = None
+
+ # Get MLP patterns and specs
+ if config.num_moe_experts is not None:
+ moe_layer_pattern = get_moe_layer_pattern(config=config)
+ else:
+ moe_layer_pattern = [0] * config.num_layers
+
+ if 1 in moe_layer_pattern:
+ moe_layer_spec = _get_moe_module_spec(config=config, backend=backend)
+ else:
+ moe_layer_spec = None
+
+ if 0 in moe_layer_pattern:
+ dense_mlp_layer_spec = _get_dense_mlp_module_spec(config=config, backend=backend)
+ else:
+ dense_mlp_layer_spec = None
+
+ # Get GPT decoder block layer specs
+ rms_norm = config.normalization == "RMSNorm"
+ layer_specs = []
+ for layer_number in range(config.num_layers):
+ attention = (
+ experimental_attention_spec
+ if experimental_attention_pattern[layer_number] == 1
+ else standard_attention_spec
+ )
+ mlp = moe_layer_spec if moe_layer_pattern[layer_number] == 1 else dense_mlp_layer_spec
+ input_layernorm = (
+ IdentityOp
+ if attention.metainfo["fuse_input_layernorm"]
+ else backend.layer_norm(rms_norm=rms_norm, for_qk=False)
+ )
+ pre_mlp_layernorm = (
+ IdentityOp
+ if mlp.metainfo["fuse_pre_mlp_layernorm"]
+ else backend.layer_norm(rms_norm=rms_norm, for_qk=False)
+ )
+
+ layer_specs.append(
+ ModuleSpec(
+ module=TransformerLayer,
+ submodules=TransformerLayerSubmodules(
+ input_layernorm=input_layernorm,
+ self_attention=attention,
+ self_attn_bda=get_bias_dropout_add,
+ pre_mlp_layernorm=pre_mlp_layernorm,
+ mlp=mlp,
+ mlp_bda=get_bias_dropout_add,
+ ),
+ )
+ )
+
+ return layer_specs
+
+
+def get_transformer_block_with_experimental_attention_variant_spec(
+ config: TransformerConfig, vp_stage: Optional[int] = None, pp_rank: Optional[int] = None
+) -> TransformerBlockSubmodules:
+ """Build transformer block spec with experimental attention variants (e.g., linear attention).
+
+ This function constructs a heterogeneous transformer block that supports mixing different
+ attention mechanisms (experimental vs standard) and MLP types (MoE vs dense) across layers.
+ **Note that, this API is a experimental API in the short term, and might be deprecated in the
+ future. In the long run, we will move to a new design that better support hybrid models.**
+
+ Constructing transformer layer specs by
+ `get_transformer_layer_with_experimental_attention_variant_spec` and then slicing the
+ layer specs to only include the layers that are built in this pipeline stage.
+
+ Args:
+ config: Transformer configuration containing model hyperparameters and feature flags.
+ vp_stage: Virtual pipeline stage index for interleaved pipeline parallelism.
+ pp_rank: Pipeline model parallel rank.
+
+ Returns:
+ TransformerBlockSubmodules containing per-layer specs and final layer norm.
+
+ Note:
+ Currently only supports transformer_engine backend. Kitchen backend can be used as a
+ wrapper with TE fallback for unsupported operations.
+ """
+
+ backend = _get_backend_spec_provider(config=config)
+
+ layer_specs = get_transformer_layer_with_experimental_attention_variant_spec(
+ config=config, backend=backend
+ )
+
+ # Slice the layer specs to only include the layers that are built in this pipeline stage.
+ if config.pipeline_model_parallel_layout is not None:
+ local_layer_ids = config.pipeline_model_parallel_layout.get_layer_id_list(
+ layer_type=LayerType.decoder, vp_stage=vp_stage, pp_rank=pp_rank
+ )
+ else:
+ offset = get_transformer_layer_offset(config, vp_stage=vp_stage, pp_rank=pp_rank)
+ num_layers_to_build = get_num_layers_to_build(config, vp_stage=vp_stage, pp_rank=pp_rank)
+ local_layer_ids = range(offset, offset + num_layers_to_build)
+
+ layer_specs = [layer_specs[layer_id] for layer_id in local_layer_ids]
+
+ # Get GPT decoder block spec
+ rms_norm = config.normalization == "RMSNorm"
+ gpt_decoder_block_spec = TransformerBlockSubmodules(
+ layer_specs=layer_specs, layer_norm=backend.layer_norm(rms_norm=rms_norm, for_qk=False)
+ )
+
+ return gpt_decoder_block_spec
+
+
+##########
+# Utilities
+##########
+
+
+def is_linear_attention_variant(experimental_attention_variant: Optional[str]) -> bool:
+ """Check if the experimental attention variant is a linear attention variant."""
+ linear_attention_variants = ["gated_delta_net"]
+ return experimental_attention_variant in linear_attention_variants
+
+
+def get_moe_layer_pattern(config: TransformerConfig) -> List[int]:
+ """Parse config.moe_layer_freq to get per-layer MoE pattern (1=MoE, 0=dense).
+
+ - int N: one MoE layer every N layers (e.g., N=2 -> [1,0,1,0,...])
+ - list: use directly as the pattern."""
+
+ if isinstance(config.moe_layer_freq, int):
+ # [1,0,0,...,0,1,0,0,...,0,...]
+ moe_layer_pattern = [
+ 1 if (i % config.moe_layer_freq == 0) else 0 for i in range(config.num_layers)
+ ]
+ elif isinstance(config.moe_layer_freq, list):
+ moe_layer_pattern = config.moe_layer_freq
+ assert len(moe_layer_pattern) == config.num_layers, (
+ f"Invalid length of moe_layer_pattern: {len(moe_layer_pattern)}, "
+ f"expected {config.num_layers}, "
+ f"current moe layer pattern: {config.moe_layer_freq}"
+ )
+ else:
+ raise ValueError(
+ f"Invalid moe_layer_freq: {type(config.moe_layer_freq)}, {config.moe_layer_freq}"
+ )
+ return moe_layer_pattern
+
+
+def get_linear_attention_pattern(config: TransformerConfig) -> List[int]:
+ """Parse config.linear_attention_freq to get per-layer attention pattern (1=LA, 0=SDPA).
+
+ - int N: one SDPA layer every N layers (e.g., N=4 -> [1,1,1,0,1,1,1,0,...])
+ - list: use directly as the pattern."""
+
+ if isinstance(config.linear_attention_freq, int):
+ linear_attention_pattern = [
+ # [1,1,...,1,0,1,1,...,1,0,...]
+ 0 if ((i + 1) % config.linear_attention_freq == 0) else 1
+ for i in range(config.num_layers)
+ ]
+ elif isinstance(config.linear_attention_freq, list):
+ linear_attention_pattern = config.linear_attention_freq
+ assert len(linear_attention_pattern) == config.num_layers, (
+ f"Invalid length of linear_attention_pattern: {len(linear_attention_pattern)}, "
+ f"expected {config.num_layers}, "
+ f"current linear attention pattern: {config.linear_attention_freq}"
+ )
+ elif config.linear_attention_freq is None:
+ if not is_linear_attention_variant(config.experimental_attention_variant):
+ linear_attention_pattern = [0] * config.num_layers
+ else:
+ # This should be caught by config validation, but raise here as a safety check
+ raise ValueError(
+ f"Linear attention type {config.experimental_attention_variant} is specified "
+ "but linear_attention_freq is None. "
+ "Please set linear_attention_freq to specify the LA/SDPA layer pattern."
+ )
+ else:
+ raise ValueError(
+ f"Invalid linear_attention_freq: {type(config.linear_attention_freq)},"
+ f" {config.linear_attention_freq}"
+ )
+ return linear_attention_pattern
+
+
+def _get_backend_spec_provider(config: TransformerConfig) -> BackendSpecProvider:
+ """Get backend spec provider for experimental attention variant."""
+
+ assert config.transformer_impl == "transformer_engine", (
+ "Experimental GPT decoder block spec only supports "
+ "transformer engine implementation for now."
+ )
+ backend: BackendSpecProvider = (
+ KitchenSpecProvider(
+ fallback=TESpecProvider(fallback_to_eager_attn=config.fallback_to_eager_attn),
+ use_kitchen_attention=config.use_kitchen_attention,
+ kitchen_attention_backend=config.kitchen_attention_backend,
+ )
+ if config.use_kitchen
+ else TESpecProvider()
+ )
+ return backend
+
+
+##########
+# Spec functions for non-experimental self attention and MLP layer.
+##########
+
+
+def _get_self_attention_module_spec(
+ config: TransformerConfig, backend: BackendSpecProvider = None
+) -> ModuleSpec:
+ """Get non-experimental self-attention module spec.
+ For hybrid models that mix experimental and non-experimental attention architectures.
+
+ Warning: This function may be deprecated in the future."""
+
+ if backend is None:
+ backend = _get_backend_spec_provider(config=config)
+
+ from megatron.core.models.gpt.gpt_layer_specs import get_gpt_layer_with_transformer_engine_spec
+
+ layer_spec = get_gpt_layer_with_transformer_engine_spec(
+ num_experts=config.num_moe_experts,
+ moe_grouped_gemm=config.moe_grouped_gemm,
+ qk_layernorm=config.qk_layernorm,
+ multi_latent_attention=config.multi_latent_attention,
+ moe_use_legacy_grouped_gemm=config.moe_use_legacy_grouped_gemm,
+ qk_l2_norm=config.qk_l2_norm,
+ use_kitchen=config.use_kitchen,
+ use_te_activation_func=config.use_te_activation_func,
+ fallback_to_eager_attn=config.fallback_to_eager_attn,
+ use_kitchen_attention=config.use_kitchen_attention,
+ kitchen_attention_backend=config.kitchen_attention_backend,
+ )
+ attn_spec = layer_spec.submodules.self_attention
+ if config.multi_latent_attention:
+ attn_spec.metainfo["fuse_input_layernorm"] = False
+ else:
+ attn_spec.metainfo["fuse_input_layernorm"] = backend.fuse_layernorm_and_linear()
+
+ return attn_spec
+
+
+def _get_dense_mlp_module_spec(
+ config: TransformerConfig, backend: BackendSpecProvider = None
+) -> ModuleSpec:
+ """Get dense MLP module spec.
+ For hybrid models that mix dense MLP and experimental attention architectures.
+
+ Warning: This function may be deprecated in the future."""
+
+ if backend is None:
+ backend = _get_backend_spec_provider(config=config)
+
+ from megatron.core.models.gpt.gpt_layer_specs import get_mlp_module_spec_for_backend
+
+ mlp_spec = get_mlp_module_spec_for_backend(backend=backend, num_experts=None)
+ mlp_spec.metainfo["fuse_pre_mlp_layernorm"] = backend.fuse_layernorm_and_linear()
+
+ return mlp_spec
+
+
+def _get_moe_module_spec(
+ config: TransformerConfig, backend: BackendSpecProvider = None
+) -> ModuleSpec:
+ """Get MoE module spec.
+ For hybrid models that mix MoE and experimental attention architectures.
+
+ Warning: This function may be deprecated in the future."""
+
+ if backend is None:
+ backend = _get_backend_spec_provider(config=config)
+
+ from megatron.core.models.gpt.moe_module_specs import get_moe_module_spec_for_backend
+
+ moe_spec = get_moe_module_spec_for_backend(
+ backend=backend,
+ num_experts=config.num_moe_experts,
+ moe_grouped_gemm=config.moe_grouped_gemm,
+ moe_use_legacy_grouped_gemm=config.moe_use_legacy_grouped_gemm,
+ use_te_activation_func=config.use_te_activation_func,
+ )
+ moe_spec.metainfo["fuse_pre_mlp_layernorm"] = False
+ return moe_spec
diff --git a/megatron/core/models/gpt/fine_grained_callables.py b/megatron/core/models/gpt/fine_grained_callables.py
index fd1cc3d33c6..5a365b015b2 100644
--- a/megatron/core/models/gpt/fine_grained_callables.py
+++ b/megatron/core/models/gpt/fine_grained_callables.py
@@ -1,4 +1,4 @@
-# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved.
+# Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
import weakref
from contextlib import nullcontext
@@ -6,9 +6,15 @@
from typing import Optional
import torch
+from torch import Tensor
from megatron.core import tensor_parallel
+from megatron.core.packed_seq_params import PackedSeqParams
+from megatron.core.pipeline_parallel.fine_grained_activation_offload import (
+ FineGrainedActivationOffloadingInterface as off_interface,
+)
from megatron.core.pipeline_parallel.utils import ScheduleNode, make_viewless
+from megatron.core.transformer.enums import CudaGraphScope
from megatron.core.transformer.module import float16_to_fp32
from megatron.core.transformer.moe.moe_layer import MoELayer
from megatron.core.transformer.multi_token_prediction import (
@@ -16,6 +22,7 @@
get_mtp_layer_offset,
)
from megatron.core.transformer.transformer_layer import TransformerLayer, make_viewless_tensor
+from megatron.core.utils import internal_api
def weak_method(method):
@@ -35,13 +42,14 @@ def wrapped_func(*args, **kwarg):
return wrapped_func
-def should_free_input(name, is_moe, is_deepep):
+@internal_api
+def should_free_input(name, is_moe, config):
"""Determine if the node should free its input memory.
Args:
name: Node name
is_moe: Whether it's a MoE model
- is_deepep: Whether it's a DeepEP model
+ config: TransformerConfig object
Returns:
bool: Whether to free input memory
@@ -49,18 +57,30 @@ def should_free_input(name, is_moe, is_deepep):
# For dense layers [attn, fake, mlp, fake], the input is needed during backward pass
if not is_moe:
return False
+ enable_deepep = (
+ config.moe_token_dispatcher_type == "flex"
+ and config.moe_flex_dispatcher_backend == "deepep"
+ )
+ enable_hybridep = (
+ config.moe_token_dispatcher_type == "flex"
+ and config.moe_flex_dispatcher_backend == "hybridep"
+ )
# Define which nodes should free input memory
# Since we split the computing graph into multiple nodes, we can manually control
# when and how to free the input memory.
# The input and output of A2A are not needed anymore after the forward pass,
# so we can free the input memory after the forward pass.
free_input_nodes = {
- "mlp": True,
+ "mlp": not enable_hybridep,
"moe_combine": True,
- # For non-deepep mode, the input is the un-dispatched tokens and probs before dispatch A2A
- # and it's not needed anymore after the forward pass
- # For deepep mode, they are both needed in backward pass, so they cannot be freed.
- "moe_dispatch": not is_deepep,
+ # For non-DeepEP and non-HybridEP dispatcher mode, the input is the un-dispatched tokens
+ # and probs before dispatch A2A and it's not needed anymore after the forward pass
+ # For DeepEP and HybridEP dispatcher mode, they are both needed in backward pass
+ # and cannot be freed.
+ # If moe_preprocess is in cuda graph scope, tokens and probs are fixed size tensors,
+ # so they cannot be freed.
+ "moe_dispatch": not (enable_deepep or enable_hybridep)
+ and (CudaGraphScope.moe_preprocess not in config.cuda_graph_scope),
}
return free_input_nodes.get(name, False)
@@ -111,13 +131,19 @@ def forward_impl(self):
if not self.gpt_model.pre_process:
self.chunk_state.decoder_input = self.gpt_model.decoder.input_tensor
# Run GPTModel._preprocess
- decoder_input, rotary_pos_emb, rotary_pos_cos, rotary_pos_sin, sequence_len_offset = (
- self.gpt_model._preprocess(
- input_ids=self.chunk_state.input_ids,
- position_ids=self.chunk_state.position_ids,
- decoder_input=self.chunk_state.decoder_input,
- packed_seq_params=self.chunk_state.packed_seq_params,
- )
+ (
+ decoder_input,
+ rotary_pos_emb,
+ rotary_pos_cos,
+ rotary_pos_sin,
+ sequence_len_offset,
+ padding_mask,
+ ) = self.gpt_model._preprocess(
+ input_ids=self.chunk_state.input_ids,
+ position_ids=self.chunk_state.position_ids,
+ decoder_input=self.chunk_state.decoder_input,
+ packed_seq_params=self.chunk_state.packed_seq_params,
+ padding_mask=self.chunk_state.padding_mask,
)
# Saved for later use
@@ -126,6 +152,7 @@ def forward_impl(self):
self.chunk_state.rotary_pos_cos = rotary_pos_cos
self.chunk_state.rotary_pos_sin = rotary_pos_sin
self.chunk_state.sequence_len_offset = sequence_len_offset
+ self.chunk_state.padding_mask = padding_mask
return decoder_input
@@ -153,9 +180,8 @@ def forward_impl(self, hidden_states):
"""Implements the forward pass for postprocessing.
This method handles:
- 1. Final layer normalization
- 2. Output layer computation
- 3. Loss computation if labels are provided
+ 1. Output layer computation
+ 2. Loss computation if labels are provided
Args:
hidden_states: The hidden states from the transformer layers.
@@ -163,12 +189,11 @@ def forward_impl(self, hidden_states):
Returns:
The logits or loss depending on whether labels are provided.
"""
- # Final layer norm from Decoder
- if self.gpt_model.decoder.final_layernorm and not self.gpt_model.mtp_process:
- hidden_states = self.gpt_model.decoder.final_layernorm(hidden_states)
- # TENorm produces a "viewed" tensor. This will result in schedule.py's
- # deallocate_output_tensor() throwing an error, so a viewless tensor is
- # created to prevent this.
+
+ empty_decoder = len(self.gpt_model.decoder.layers) == 0
+ layer_norm = self.gpt_model.decoder.final_layernorm
+ if not self.gpt_model.config.mtp_num_layers and empty_decoder and layer_norm:
+ hidden_states = layer_norm(hidden_states)
hidden_states = make_viewless_tensor(
inp=hidden_states, requires_grad=True, keep_graph=True
)
@@ -225,12 +250,13 @@ def __init__(
it's the per_batch_state_context, o.w. nullcontext
name (str): Node name, also used to determine memory strategy
bwd_dw_callables (list): List of weight gradient functions for the layer.
- extra_args (dict): Extra arguments for the node: is_moe, enable_deepep.
+ extra_args (dict): Extra arguments for the node: is_moe, config.
"""
# determine whether to free input memory
+ config = extra_args.get("config", None)
+ assert config is not None, "model config must be passed to TransformerLayerNode."
is_moe = extra_args.get("is_moe", False)
- enable_deepep = extra_args.get("enable_deepep", False)
- free_input = should_free_input(name, is_moe, enable_deepep)
+ free_input = should_free_input(name, is_moe, config)
self.delay_wgrad_compute = extra_args.get("delay_wgrad_compute", False)
super().__init__(
@@ -246,6 +272,7 @@ def __init__(
self.submodule = submodule
self.detached = tuple()
self.before_detached = tuple()
+ self.is_mtp = extra_args.get("is_mtp", False)
# Create flags to indicate first and last layer
self.is_first_layer = extra_args.get("is_first_layer", False)
@@ -275,7 +302,13 @@ def backward_impl(self, outputs, output_grad):
detached_grad = tuple([e.grad for e in self.detached])
grads = output_grad + detached_grad
self.default_backward_func(outputs + self.before_detached, grads)
- self._release_state()
+ # release the output grad memory after backward finishes,
+ # except when delay_wgrad_comptue is enabled, the grad should be
+ # kept until all modules' backward_dw has been invoked.
+ if self.delay_wgrad_compute:
+ self.output_grads = grads
+ self.delay_grads_release = len(self.bwd_dw_callables) > 0
+
# return grads for record stream
return grads
@@ -286,9 +319,17 @@ def backward_dw(self):
with torch.cuda.nvtx.range(f"{self.name} wgrad"):
for module in self.bwd_dw_callables:
module.backward_dw()
+
+ # the output grad memory is last used in wgrad compute, should be safe to release.
+ if self.manual_grads_release:
+ assert self.delay_grads_release, "output grad memory should be valid before wgrad."
+ for tensor in self.output_grads:
+ tensor.untyped_storage().resize_(0)
+ self.output_grads = None
+
self.bwd_dw_callables = None
- def _release_state(self):
+ def __del__(self):
# Release reference as early as possible, this helps avoid memory leak.
self.before_detached = None
self.detached = None
@@ -329,12 +370,106 @@ def build_transformer_layer_callables(layer: TransformerLayer):
layer.config.moe_token_dispatcher_type == "flex"
and layer.config.moe_flex_dispatcher_backend == "deepep"
)
+ enable_hybridep = (
+ layer.config.moe_token_dispatcher_type == "flex"
+ and layer.config.moe_flex_dispatcher_backend == "hybridep"
+ )
+
+ class _BackwardDWWrapper:
+ def __init__(self):
+ self.graphed_backward_dw_callable = None
+ self.attn_dw_callable = layer.self_attention.backward_dw
+ if isinstance(layer.mlp, MoELayer):
+ self.shared_expert_dw_callable = partial(
+ layer.mlp.backward_dw, routed_experts=False, shared_experts=True
+ )
+ else:
+ self.shared_expert_dw_callable = None
+ self.cuda_graph_scope = layer.config.cuda_graph_scope
+
+ def set_graphed_backward_dw_callable(self, graphed_backward_dw_callable):
+ """Store the CUDA graphed backward weight gradient callable."""
+ self.graphed_backward_dw_callable = graphed_backward_dw_callable
+
+ def backward_dw(self):
+ """Execute weight gradients, skipping CUDA graphed components during replay."""
+ is_replay = hasattr(layer, 'cuda_graphs') and layer.cuda_graphs
+ if self.shared_expert_dw_callable is not None and (
+ not is_replay or CudaGraphScope.moe_router not in self.cuda_graph_scope
+ ):
+ self.shared_expert_dw_callable()
+ if not is_replay or CudaGraphScope.attn not in self.cuda_graph_scope:
+ self.attn_dw_callable()
+ if is_replay and self.graphed_backward_dw_callable is not None:
+ self.graphed_backward_dw_callable()
+
+ attn_backward_dw_wrapper = _BackwardDWWrapper()
def submodule_attn_forward(node: ScheduleNode, hidden_states: torch.Tensor):
"""
- Performs same attnention forward logic as GPT Model.
+ Performs same attnention forward logic as GPT Model and forward pass for
+ computations between attention and dispatch:
+ pre mlp layernorm->router->dispatch preprocess
"""
- hidden_states, _ = layer._forward_attention(
+
+ if hasattr(layer, 'cuda_graphs') and layer.cuda_graphs:
+ assert (
+ CudaGraphScope.mlp not in layer.config.cuda_graph_scope
+ and CudaGraphScope.moe not in layer.config.cuda_graph_scope
+ ), (
+ "Supported CUDA graph scope with EP overlap: "
+ "attn, moe_router, moe_preprocess, mlp, got {}".format(
+ layer.config.cuda_graph_scope
+ )
+ )
+ forward_func = layer._te_cuda_graph_replay
+ attn_backward_dw_wrapper.set_graphed_backward_dw_callable(
+ partial(layer.backward_dw_cudagraph, layer.current_microbatch)
+ )
+ else:
+ # wrapper function that keeps consistent api with cuda graph replay
+ def forward_func(
+ hidden_states: Tensor,
+ attention_mask: Optional[Tensor] = None,
+ rotary_pos_emb: Optional[Tensor] = None,
+ rotary_pos_cos: Optional[Tensor] = None,
+ rotary_pos_sin: Optional[Tensor] = None,
+ packed_seq_params: Optional[PackedSeqParams] = None,
+ sequence_len_offset: Optional[Tensor] = None,
+ ):
+ hidden_states, _ = layer._forward_attention(
+ hidden_states=hidden_states,
+ attention_mask=attention_mask,
+ rotary_pos_emb=rotary_pos_emb,
+ rotary_pos_cos=rotary_pos_cos,
+ rotary_pos_sin=rotary_pos_sin,
+ packed_seq_params=packed_seq_params,
+ sequence_len_offset=sequence_len_offset,
+ )
+ if not isinstance(layer.mlp, MoELayer):
+ return hidden_states, None, None, None
+ if layer.recompute_pre_mlp_layernorm:
+ layer.pre_mlp_norm_checkpoint = tensor_parallel.CheckpointWithoutOutput()
+ with off_interface(
+ layer.offload_mlp_norm, hidden_states, "mlp_norm"
+ ) as hidden_states:
+ pre_mlp_layernorm_output = layer.pre_mlp_norm_checkpoint.checkpoint(
+ layer.pre_mlp_layernorm, hidden_states
+ )
+ else:
+ with off_interface(
+ layer.offload_mlp_norm, hidden_states, "mlp_norm"
+ ) as hidden_states:
+ pre_mlp_layernorm_output = layer.pre_mlp_layernorm(hidden_states)
+
+ shared_expert_output = layer.mlp.shared_experts_compute(pre_mlp_layernorm_output)
+ probs, routing_map = layer.mlp.route(pre_mlp_layernorm_output)
+ local_tokens, probs = layer.mlp.preprocess(
+ pre_mlp_layernorm_output, probs, routing_map
+ )
+ return hidden_states, local_tokens, probs, shared_expert_output
+
+ hidden_states, local_tokens, probs, shared_expert_output = forward_func(
hidden_states=hidden_states,
attention_mask=node.chunk_state.attention_mask,
rotary_pos_emb=node.chunk_state.rotary_pos_emb,
@@ -343,28 +478,14 @@ def submodule_attn_forward(node: ScheduleNode, hidden_states: torch.Tensor):
packed_seq_params=node.chunk_state.packed_seq_params,
sequence_len_offset=node.chunk_state.sequence_len_offset,
)
- return hidden_states
-
- def submodule_post_attn_forward(node: ScheduleNode, hidden_states: torch.Tensor):
- """
- Run forward pass for computations between attention and dispatch:
- pre mlp layernorm->router->dispatch preprocess
- """
- if layer.recompute_pre_mlp_layernorm:
- layer.pre_mlp_norm_checkpoint = tensor_parallel.CheckpointWithoutOutput()
- pre_mlp_layernorm_output = layer.pre_mlp_norm_checkpoint.checkpoint(
- layer.pre_mlp_layernorm, hidden_states
- )
- else:
- pre_mlp_layernorm_output = layer.pre_mlp_layernorm(hidden_states)
-
- local_tokens, probs, _ = layer.mlp.router_and_preprocess(pre_mlp_layernorm_output)
+ if not isinstance(layer.mlp, MoELayer):
+ return hidden_states
# Detach here for mlp_bda residual connection
node.layer_state.residual = node.detach(hidden_states)
if layer.mlp.use_shared_expert and not layer.mlp.shared_expert_overlap:
- # Detach here for shared expert connection
- node.layer_state.pre_mlp_layernorm_output = node.detach(pre_mlp_layernorm_output)
+ # Detach here for shared expert connection in moe_combine
+ node.layer_state.shared_expert_output = node.detach(shared_expert_output)
return local_tokens, probs
@@ -375,7 +496,7 @@ def submodule_dispatch_forward(
Dispatches tokens to the experts based on the router output.
"""
token_dispatcher = layer.mlp.token_dispatcher
- if enable_deepep:
+ if enable_deepep or enable_hybridep:
# update token_probs to be the detached version, prevents
# backward graph from connecting to attn submodule
token_dispatcher._comm_manager.token_probs = probs
@@ -389,19 +510,14 @@ def submodule_moe_forward(node: ScheduleNode, dispatched_tokens: torch.Tensor):
Run forward pass for computations between dispatch and combine:
post dispatch->experts->combine preprocess
"""
- shared_expert_output = None
dispatched_probs = node.layer_state.dispatched_probs
token_dispatcher = layer.mlp.token_dispatcher
- if enable_deepep:
+ if enable_deepep or enable_hybridep:
# update dispatched_probs to be detached version, prevents
# backward graph from connecting to dispatch submodule
token_dispatcher._comm_manager.dispatched_probs = dispatched_probs
- pre_mlp_layernorm_output = getattr(node.layer_state, 'pre_mlp_layernorm_output', None)
- shared_expert_output = layer.mlp.shared_experts_compute(pre_mlp_layernorm_output)
- expert_output, mlp_bias = layer.mlp.routed_experts_compute(
- dispatched_tokens, dispatched_probs, pre_mlp_layernorm_output
- )
+ expert_output, _ = layer.mlp.routed_experts_compute(dispatched_tokens, dispatched_probs)
if layer.recompute_pre_mlp_layernorm:
# discard the output of the pre-mlp layernorm and register the recompute
@@ -410,16 +526,10 @@ def submodule_moe_forward(node: ScheduleNode, dispatched_tokens: torch.Tensor):
# release tensor reference after use
node.layer_state.dispatched_probs = None
node.layer_state.pre_mlp_layernorm_output = None
- if shared_expert_output is None:
- # Return only expert_output, since shared_expert_output causes backward on None
- return expert_output
- return expert_output, shared_expert_output
-
- def submodule_combine_forward(
- node: ScheduleNode,
- output: torch.Tensor,
- shared_expert_output: Optional[torch.Tensor] = None,
- ):
+
+ return expert_output
+
+ def submodule_combine_forward(node: ScheduleNode, output: torch.Tensor):
"""
# Triggers token combine and the remaining computation in the transformer layer.
# The `mlp_bda` computation is placed after `mlp.combine` due to data dependency.
@@ -429,14 +539,21 @@ def submodule_combine_forward(
# with another microbatch's computation and expose the communication.
"""
residual = node.layer_state.residual
-
+ shared_expert_output = getattr(node.layer_state, 'shared_expert_output', None)
output = layer.mlp.combine(output, shared_expert_output)
mlp_output_with_bias = (output, None)
-
+ if hasattr(layer, 'cuda_graphs') and layer.cuda_graphs:
+ layer.mlp.cudagraph_tensor_store.clear()
with layer.bias_dropout_add_exec_handler():
hidden_states = layer.mlp_bda(layer.training, layer.config.bias_dropout_fusion)(
mlp_output_with_bias, residual, layer.hidden_dropout
)
+ # Delay the offload of the mlp norm until after the mlp_bda has been computed
+ # because the residual is needed in the mlp_bda.
+ if layer.offload_mlp_norm:
+ hidden_states = off_interface.group_commit(
+ hidden_states, name="mlp_norm", forced_released_tensors=[residual]
+ )
output = make_viewless_tensor(
inp=hidden_states, requires_grad=hidden_states.requires_grad, keep_graph=True
)
@@ -446,6 +563,12 @@ def submodule_combine_forward(
# release tensor reference after use
node.layer_state.residual = None
+
+ # final layer norm from decoder
+ final_layernorm = node.chunk_state.model.decoder.final_layernorm
+ if not node.is_mtp and final_layernorm and node.is_last_layer:
+ output = final_layernorm(output)
+ output = make_viewless_tensor(inp=output, requires_grad=True, keep_graph=True)
return output
def mlp_wrapper(node: ScheduleNode, *args, **kwargs):
@@ -458,13 +581,12 @@ def raise_not_implemented(*args):
# Build forward and backward callable functions
attn_func = submodule_attn_forward
- post_attn_func = submodule_post_attn_forward if is_moe else raise_not_implemented
dispatch_func = submodule_dispatch_forward if is_moe else raise_not_implemented
mlp_func = submodule_moe_forward if is_moe else mlp_wrapper
combine_func = submodule_combine_forward if is_moe else raise_not_implemented
- forward_funcs = [attn_func, post_attn_func, dispatch_func, mlp_func, combine_func, None]
- backward_dw = {"attn": layer.self_attention, "mlp": layer.mlp}
+ forward_funcs = [attn_func, dispatch_func, mlp_func, combine_func, None]
+ backward_dw = {"attn": attn_backward_dw_wrapper, "mlp": layer.mlp}
return forward_funcs, backward_dw
@@ -476,24 +598,14 @@ def build_mtp_layer_callables(layer):
"""
forward_funcs, backward_dw = build_transformer_layer_callables(layer.transformer_layer)
- attn_forward, post_attn_forward, dispatch_forward, mlp_forward, combine_forward, _ = (
- forward_funcs
- )
+ attn_forward, dispatch_forward, mlp_forward, combine_forward, _ = forward_funcs
is_moe = isinstance(layer.transformer_layer.mlp, MoELayer)
assert is_moe, "MTP layer in a2a overlap only supports MoE layer for now."
def submodule_mtp_attn_forward(node, hidden_states):
# MTP Block Preprocess
if node.is_first_layer:
- # Final layer norm from Decoder
- final_layernorm = node.chunk_state.model.decoder.final_layernorm
- if final_layernorm:
- hidden_states = final_layernorm(hidden_states)
- hidden_states = make_viewless_tensor(
- inp=hidden_states, requires_grad=True, keep_graph=True
- )
- hidden_states = node.detach(hidden_states)
- offset = get_mtp_layer_offset(layer.config)
+ offset = get_mtp_layer_offset(layer.config, node.chunk_state.model.vp_stage)
node.chunk_state.mtp_hidden_states = list(torch.chunk(hidden_states, 1 + offset, dim=0))
hidden_states = node.chunk_state.mtp_hidden_states[offset]
@@ -547,24 +659,17 @@ def rng_context_wrapper(func, *args, **kwargs):
# Build forward and backward callable functions
# attn_forward already has rng context, no need to wrap
attn_func = submodule_mtp_attn_forward
- post_attn_func = partial(rng_context_wrapper, post_attn_forward)
dispatch_func = partial(rng_context_wrapper, dispatch_forward)
mlp_func = partial(rng_context_wrapper, mlp_forward)
combine_func = partial(rng_context_wrapper, combine_forward)
mtp_post_process_func = submodule_mtp_postprocess_forward
- forward_funcs = [
- attn_func,
- post_attn_func,
- dispatch_func,
- mlp_func,
- combine_func,
- mtp_post_process_func,
- ]
- backward_dw = {
- "attn": [layer.transformer_layer.self_attention, layer.eh_proj],
- "mlp": layer.transformer_layer.mlp,
- }
+ forward_funcs = [attn_func, dispatch_func, mlp_func, combine_func, mtp_post_process_func]
+ if isinstance(backward_dw["attn"], list):
+ backward_dw["attn"].append(layer.eh_proj)
+ else:
+ backward_dw["attn"] = [backward_dw["attn"], layer.eh_proj]
+
return forward_funcs, backward_dw
diff --git a/megatron/core/models/gpt/gpt_layer_specs.py b/megatron/core/models/gpt/gpt_layer_specs.py
index b61a544c5df..70f0a8244ca 100755
--- a/megatron/core/models/gpt/gpt_layer_specs.py
+++ b/megatron/core/models/gpt/gpt_layer_specs.py
@@ -40,7 +40,7 @@
from megatron.core.utils import is_te_min_version
try:
- import transformer_engine as te # type: ignore[import-untyped] # pylint: disable=unused-import
+ import transformer_engine as te # pylint: disable=unused-import
from megatron.core.extensions.transformer_engine import TEFusedMLP, TENorm
from megatron.core.extensions.transformer_engine_spec_provider import TESpecProvider
@@ -50,7 +50,7 @@
HAVE_TE = False
try:
- import nvidia_kitchen # type: ignore[import-not-found] # pylint: disable=unused-import
+ import nvidia_kitchen # pylint: disable=unused-import
from megatron.core.extensions.kitchen import KitchenSpecProvider
@@ -59,7 +59,7 @@
HAVE_KITCHEN = False
try:
- import apex # type: ignore[import-untyped] # pylint: disable=unused-import
+ import apex # pylint: disable=unused-import
from megatron.core.fusions.fused_layer_norm import FusedLayerNorm
@@ -182,6 +182,7 @@ def get_gpt_layer_with_transformer_engine_spec(
use_te_op_fuser: Optional[bool] = False,
use_kitchen: bool = False,
use_te_activation_func: bool = False,
+ fallback_to_eager_attn: bool = False,
use_kitchen_attention: bool = False,
kitchen_attention_backend: str = "sdpa",
) -> ModuleSpec:
@@ -212,7 +213,7 @@ def get_gpt_layer_with_transformer_engine_spec(
if use_kitchen:
assert HAVE_KITCHEN
backend: BackendSpecProvider = KitchenSpecProvider(
- fallback=TESpecProvider(),
+ fallback=TESpecProvider(fallback_to_eager_attn=fallback_to_eager_attn),
use_kitchen_attention=use_kitchen_attention,
kitchen_attention_backend=kitchen_attention_backend,
)
@@ -221,7 +222,7 @@ def get_gpt_layer_with_transformer_engine_spec(
if use_te_activation_func:
raise AssertionError("use_te_activation_func not compatible with using kitchen.")
else:
- backend = TESpecProvider()
+ backend = TESpecProvider(fallback_to_eager_attn=fallback_to_eager_attn)
mlp = get_mlp_module_spec_for_backend(
backend=backend,
@@ -515,17 +516,19 @@ def get_mlp_module_spec_for_backend(
)
-def get_gpt_decoder_block_spec(
+def get_gpt_decoder_layer_specs(
config: TransformerConfig,
use_transformer_engine: bool,
normalization: Optional[str] = None,
qk_l2_norm: Optional[bool] = False,
- vp_stage: Optional[int] = None,
- pp_rank: Optional[int] = None,
) -> TransformerBlockSubmodules:
"""GPT block spec."""
+ assert config.experimental_attention_variant is None, (
+ "Experimental attention variant is not supported with get_gpt_decoder_layer_specs, "
+ f"but got {config.experimental_attention_variant=}."
+ )
+
if use_transformer_engine:
- layer_norm_impl = TENorm
dense_layer_spec = get_gpt_layer_with_transformer_engine_spec(
num_experts=None,
moe_grouped_gemm=False,
@@ -535,8 +538,6 @@ def get_gpt_decoder_block_spec(
qk_l2_norm=qk_l2_norm,
use_kitchen=config.use_kitchen,
use_te_activation_func=config.use_te_activation_func,
- use_kitchen_attention=config.use_kitchen_attention,
- kitchen_attention_backend=config.kitchen_attention_backend,
)
moe_layer_spec = get_gpt_layer_with_transformer_engine_spec(
num_experts=config.num_moe_experts,
@@ -547,11 +548,8 @@ def get_gpt_decoder_block_spec(
qk_l2_norm=qk_l2_norm,
use_kitchen=config.use_kitchen,
use_te_activation_func=config.use_te_activation_func,
- use_kitchen_attention=config.use_kitchen_attention,
- kitchen_attention_backend=config.kitchen_attention_backend,
)
else:
- layer_norm_impl = LNImpl
dense_layer_spec = get_gpt_layer_local_spec(
num_experts=None,
moe_grouped_gemm=False,
@@ -561,8 +559,6 @@ def get_gpt_decoder_block_spec(
normalization=normalization,
qk_l2_norm=qk_l2_norm,
use_kitchen=config.use_kitchen,
- use_kitchen_attention=config.use_kitchen_attention,
- kitchen_attention_backend=config.kitchen_attention_backend,
)
moe_layer_spec = get_gpt_layer_local_spec(
num_experts=config.num_moe_experts,
@@ -573,8 +569,6 @@ def get_gpt_decoder_block_spec(
normalization=normalization,
qk_l2_norm=qk_l2_norm,
use_kitchen=config.use_kitchen,
- use_kitchen_attention=config.use_kitchen_attention,
- kitchen_attention_backend=config.kitchen_attention_backend,
)
# Parse config.moe_layer_freq to determine the pattern of expert/dense layers.
@@ -607,13 +601,31 @@ def get_gpt_decoder_block_spec(
else:
raise ValueError(f"Invalid layer pattern: {moe_layer_pattern}")
+ return layer_specs
+
+
+def get_gpt_decoder_block_spec(
+ config: TransformerConfig,
+ use_transformer_engine: bool,
+ normalization: Optional[str] = None,
+ qk_l2_norm: Optional[bool] = False,
+ vp_stage: Optional[int] = None,
+ pp_rank: Optional[int] = None,
+) -> TransformerBlockSubmodules:
+ """GPT block spec."""
+ layer_specs = get_gpt_decoder_layer_specs(
+ config, use_transformer_engine, normalization, qk_l2_norm
+ )
+
# Slice the layer specs to only include the layers that are built in this pipeline stage.
# Note: MCore layer_number starts at 1
num_layers_to_build = get_num_layers_to_build(config, vp_stage=vp_stage, pp_rank=pp_rank)
if config.pipeline_model_parallel_layout is not None:
layout = config.pipeline_model_parallel_layout
- assert isinstance(layout, PipelineParallelLayerLayout)
+ assert isinstance(
+ layout, PipelineParallelLayerLayout
+ ), f"Invalid pipeline model parallel layout: {layout}"
local_layer_specs = [
layer_specs[layer_id]
for layer_id in layout.get_layer_id_list(
@@ -625,6 +637,10 @@ def get_gpt_decoder_block_spec(
local_layer_specs = layer_specs[offset : offset + num_layers_to_build]
# Block spec.
+ if use_transformer_engine:
+ layer_norm_impl = TENorm
+ else:
+ layer_norm_impl = LNImpl
block_spec = TransformerBlockSubmodules(
layer_specs=local_layer_specs, layer_norm=layer_norm_impl
)
@@ -641,22 +657,17 @@ def get_gpt_mtp_block_spec(
) -> MultiTokenPredictionBlockSubmodules:
"""GPT Multi-Token Prediction (MTP) block spec."""
if use_transformer_engine:
- backend: BackendSpecProvider = (
- KitchenSpecProvider(
- fallback=TESpecProvider(),
+ if config.use_kitchen:
+ backend: BackendSpecProvider = KitchenSpecProvider(
+ fallback=TESpecProvider(fallback_to_eager_attn=config.fallback_to_eager_attn),
use_kitchen_attention=config.use_kitchen_attention,
kitchen_attention_backend=config.kitchen_attention_backend,
)
- if config.use_kitchen
- else TESpecProvider()
- )
+ else:
+ backend = TESpecProvider(fallback_to_eager_attn=config.fallback_to_eager_attn)
else:
backend = (
- KitchenSpecProvider(
- fallback=LocalSpecProvider(),
- use_kitchen_attention=config.use_kitchen_attention,
- kitchen_attention_backend=config.kitchen_attention_backend,
- )
+ KitchenSpecProvider(fallback=LocalSpecProvider())
if config.use_kitchen
else LocalSpecProvider()
)
@@ -691,7 +702,7 @@ def get_gpt_mtp_block_spec_for_backend(
mtp_num_layers = config.mtp_num_layers if config.mtp_num_layers else 0
mtp_layer_specs = [mtp_layer_spec] * mtp_num_layers
- offset = get_mtp_layer_offset(config)
+ offset = get_mtp_layer_offset(config, vp_stage=vp_stage)
# split the mtp layer specs to only include the layers that are built in this pipeline stage.
mtp_layer_specs = mtp_layer_specs[offset : offset + num_layers_to_build]
if len(mtp_layer_specs) > 0:
diff --git a/megatron/core/models/gpt/gpt_model.py b/megatron/core/models/gpt/gpt_model.py
index 9342e96ce9a..16462d6e426 100644
--- a/megatron/core/models/gpt/gpt_model.py
+++ b/megatron/core/models/gpt/gpt_model.py
@@ -1,4 +1,4 @@
-# Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved.
+# Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
from collections import OrderedDict
from typing import Dict, Literal, Optional
@@ -18,16 +18,18 @@
)
from megatron.core.models.common.language_module.language_module import LanguageModule
from megatron.core.packed_seq_params import PackedSeqParams
+from megatron.core.pipeline_parallel.fine_grained_activation_offload import (
+ FineGrainedActivationOffloadingInterface as off_interface,
+)
from megatron.core.process_groups_config import ProcessGroupCollection
from megatron.core.quantization.utils import get_quant_config_or_none
from megatron.core.tensor_parallel import gather_from_sequence_parallel_region
-from megatron.core.transformer.enums import ModelType
+from megatron.core.transformer.enums import CudaGraphScope, ModelType
from megatron.core.transformer.multi_token_prediction import (
MTPLossAutoScaler,
MTPLossLoggingHelper,
MultiTokenPredictionBlock,
roll_tensor,
- tie_output_layer_state_dict,
tie_word_embeddings_state_dict,
)
from megatron.core.transformer.spec_utils import ModuleSpec
@@ -117,6 +119,7 @@ def __init__(
self.parallel_output = parallel_output
self.share_embeddings_and_output_weights = share_embeddings_and_output_weights
self.vp_stage = vp_stage
+ self.disable_param_offloading = True
if hasattr(self.config, 'position_embedding_type'):
self.position_embedding_type = self.config.position_embedding_type
@@ -246,7 +249,7 @@ def __init__(
tp_group=self.pg_collection.tp,
)
- if self.pre_process or self.post_process:
+ if self.pre_process or self.post_process or self.mtp_process:
self.setup_embeddings_and_output_layer()
if has_config_logger_enabled(self.config):
@@ -281,6 +284,7 @@ def _preprocess(
decoder_input: Tensor = None,
inference_context: BaseInferenceContext = None,
packed_seq_params: PackedSeqParams = None,
+ padding_mask: Optional[Tensor] = None,
):
"""Preprocesses inputs for the transformer decoder.
@@ -297,7 +301,20 @@ def _preprocess(
if decoder_input is not None:
pass
elif self.pre_process:
+ if padding_mask is not None:
+ assert padding_mask.shape == input_ids.shape, (
+ f"padding_mask shape {padding_mask.shape} does not match "
+ f"input_ids shape {input_ids.shape}"
+ )
decoder_input = self.embedding(input_ids=input_ids, position_ids=position_ids)
+ if padding_mask is not None and self.config.sequence_parallel:
+ padding_mask = (
+ tensor_parallel.scatter_to_sequence_parallel_region(
+ padding_mask.transpose(0, 1).contiguous()
+ )
+ .transpose(0, 1)
+ .contiguous()
+ )
else:
# intermediate stage of pipeline
# decoder will get hidden_states from encoder.input_tensor
@@ -341,16 +358,16 @@ def _preprocess(
inference_context, self.decoder, decoder_input, self.config, packed_seq_params
)
rotary_pos_emb = self.rotary_pos_emb(
- rotary_seq_len,
- packed_seq=packed_seq_params is not None
- and packed_seq_params.qkv_format == 'thd',
+ rotary_seq_len, packed_seq_params=packed_seq_params
)
elif self.position_embedding_type == 'yarn':
if self.training or not self.config.flash_decode:
rotary_seq_len = self.rotary_pos_emb.get_rotary_seq_len(
inference_context, self.decoder, decoder_input, self.config, packed_seq_params
)
- rotary_pos_emb, _ = self.rotary_pos_emb(rotary_seq_len)
+ rotary_pos_emb, _ = self.rotary_pos_emb(
+ rotary_seq_len, packed_seq_params=packed_seq_params
+ )
else:
raise NotImplementedError(
"Flash decoding uses precomputed cos and sin for RoPE, not implemented in "
@@ -358,7 +375,9 @@ def _preprocess(
)
elif self.position_embedding_type == 'mrope' and not self.config.multi_latent_attention:
if self.training or not self.config.flash_decode:
- rotary_pos_emb = self.rotary_pos_emb(position_ids, self.mrope_section)
+ rotary_pos_emb = self.rotary_pos_emb(
+ position_ids, self.mrope_section, packed_seq_params=packed_seq_params
+ )
else:
# Flash decoding uses precomputed cos and sin for RoPE
raise NotImplementedError(
@@ -371,7 +390,7 @@ def _preprocess(
and (
(
self.config.cuda_graph_impl == "local"
- and self.config.cuda_graph_scope != "full_iteration"
+ and CudaGraphScope.full_iteration not in self.config.cuda_graph_scope
)
or self.config.flash_decode
)
@@ -398,6 +417,7 @@ def _preprocess(
rotary_pos_cos,
rotary_pos_sin,
sequence_len_offset,
+ padding_mask,
)
if rotary_pos_cos_sin is not None:
# only in the case of flashinfer fused rope will we
@@ -409,6 +429,24 @@ def _preprocess(
return preproc_output
+ def preprocess_for_fine_grained_offloading(self):
+ """Preprocess for fine-grained activation offloading."""
+ off_interface.init_chunk_handler(
+ vp_size=self.config.virtual_pipeline_model_parallel_size,
+ vp_stage=self.vp_stage,
+ min_offloaded_tensor_size=self.config.min_offloaded_tensor_size,
+ )
+ if self.disable_param_offloading:
+ for param in self.decoder.parameters():
+ off_interface.mark_not_offloadable(param)
+ if self.mtp_process:
+ for param in self.mtp.parameters():
+ off_interface.mark_not_offloadable(param)
+ if self.post_process:
+ for param in self.output_layer.parameters():
+ off_interface.mark_not_offloadable(param)
+ self.disable_param_offloading = False
+
def forward(
self,
input_ids: Tensor,
@@ -423,6 +461,7 @@ def forward(
*,
inference_params: Optional[BaseInferenceContext] = None,
loss_mask: Optional[Tensor] = None,
+ padding_mask: Optional[Tensor] = None,
) -> Tensor:
"""Forward function of the GPT Model This function passes the input tensors
through the embedding layer, and then the decoder and finally into the post
@@ -433,7 +472,12 @@ def forward(
Args:
runtime_gather_output (bool): Gather output at runtime. Default None means
`parallel_output` arg in the constructor will be used.
+ padding_mask (Tensor, optional): Padding mask for MoE routing.
+ Shape [bsz, seq_length]. True = padding (exclude), False = valid (include).
+ Only used for MoE layers to exclude padding tokens from routing computations.
"""
+ if self.config.fine_grained_activation_offloading:
+ self.preprocess_for_fine_grained_offloading()
inference_context = deprecate_inference_params(inference_context, inference_params)
@@ -443,13 +487,19 @@ def forward(
decoder_input=decoder_input,
inference_context=inference_context,
packed_seq_params=packed_seq_params,
+ padding_mask=padding_mask,
)
- (decoder_input, rotary_pos_emb, rotary_pos_cos, rotary_pos_sin, sequence_len_offset) = (
- preproc_output[:5]
- )
+ (
+ decoder_input,
+ rotary_pos_emb,
+ rotary_pos_cos,
+ rotary_pos_sin,
+ sequence_len_offset,
+ padding_mask,
+ ) = preproc_output[:6]
- rotary_pos_cos_sin = preproc_output[5] if len(preproc_output) == 6 else None
+ rotary_pos_cos_sin = preproc_output[6] if len(preproc_output) == 7 else None
# Run decoder.
hidden_states = self.decoder(
@@ -462,6 +512,7 @@ def forward(
rotary_pos_cos_sin=rotary_pos_cos_sin,
packed_seq_params=packed_seq_params,
sequence_len_offset=sequence_len_offset,
+ padding_mask=padding_mask,
**(extra_block_kwargs or {}),
)
@@ -518,7 +569,6 @@ def _postprocess(
output_weight = None
if self.share_embeddings_and_output_weights:
output_weight = self.shared_embedding_or_output_weight()
-
if mtp_in_postprocess:
hidden_states = self.mtp(
input_ids=input_ids,
@@ -538,7 +588,8 @@ def _postprocess(
if not self.post_process:
return hidden_states
- if self.mtp_process:
+ # Skip when mtp_num_layers is None or 0
+ if self.config.mtp_num_layers:
mtp_labels = labels.clone()
hidden_states_list = torch.chunk(hidden_states, 1 + self.config.mtp_num_layers, dim=0)
hidden_states = hidden_states_list[0]
@@ -546,12 +597,6 @@ def _postprocess(
# if loss_mask is not provided, use all ones as loss_mask
loss_mask = torch.ones_like(mtp_labels)
for mtp_layer_number in range(self.config.mtp_num_layers):
- # output
- mtp_logits, _ = self.output_layer(
- hidden_states_list[mtp_layer_number + 1],
- weight=output_weight,
- runtime_gather_output=runtime_gather_output,
- )
# Calc loss for the current Multi-Token Prediction (MTP) layers.
mtp_labels, _ = roll_tensor(
mtp_labels,
@@ -567,7 +612,20 @@ def _postprocess(
cp_group=self.cp_group,
packed_seq_params=packed_seq_params,
)
- mtp_loss = self.compute_language_model_loss(mtp_labels, mtp_logits)
+
+ # Compute mtp loss without storing logits to save memory.
+ mtp_loss = self.compute_output_layer_and_language_model_loss(
+ hidden_states_list[mtp_layer_number + 1],
+ labels=mtp_labels,
+ weight=self.shared_embedding_or_output_weight(),
+ sequence_parallel_enabled=self.output_layer.sequence_parallel,
+ column_parallel_linear=self.output_layer,
+ col_linear_kwargs={
+ 'weight': output_weight,
+ 'runtime_gather_output': runtime_gather_output,
+ },
+ )
+
mtp_loss = loss_mask * mtp_loss
if self.training:
# TODO(shifangx): remove the use of parallel_state here
@@ -590,6 +648,7 @@ def _postprocess(
hidden_states, mtp_loss_scale * mtp_loss / num_tokens
)
sequence_parallel_override = False
+
if in_inference_mode and inference_context.materialize_only_last_token_logits:
if inference_context.is_static_batching():
hidden_states = hidden_states[-1:, :, :]
@@ -611,9 +670,12 @@ def _postprocess(
hidden_states.squeeze(1).unsqueeze(0)
).unsqueeze(1)
- logits, _ = self.output_layer(
- hidden_states, weight=output_weight, runtime_gather_output=runtime_gather_output
- )
+ if has_config_logger_enabled(self.config) or labels is None:
+ logits, _ = self.output_layer(
+ hidden_states, weight=output_weight, runtime_gather_output=runtime_gather_output
+ )
+ else:
+ logits = None
# Restore sequence parallel execution to the output layer if necessary.
if sequence_parallel_override:
@@ -640,7 +702,17 @@ def _postprocess(
# [s b h] => [b s h]
return logits.transpose(0, 1).contiguous()
- loss = self.compute_language_model_loss(labels, logits)
+ loss = self.compute_output_layer_and_language_model_loss(
+ hidden_states,
+ labels=labels,
+ weight=self.shared_embedding_or_output_weight(),
+ sequence_parallel_enabled=self.output_layer.sequence_parallel,
+ column_parallel_linear=self.output_layer,
+ col_linear_kwargs={
+ 'weight': output_weight,
+ 'runtime_gather_output': runtime_gather_output,
+ },
+ )
return loss
@@ -678,6 +750,7 @@ def build_schedule_plan(
runtime_gather_output: Optional[bool] = None,
inference_params: Optional[BaseInferenceContext] = None,
loss_mask: Optional[Tensor] = None,
+ padding_mask: Optional[Tensor] = None,
):
"""Builds a computation schedule plan for the model.
@@ -703,11 +776,15 @@ def build_schedule_plan(
inference_params (InferenceParams, optional):
Parameters for inference. Defaults to None.
loss_mask (Optional[Tensor], optional): Loss mask. Defaults to None.
+ padding_mask (Optional[Tensor], optional): Padding mask. Defaults to None.
Returns:
TransformerModelChunkSchedulePlan: The model chunk schedule plan.
"""
+ if self.config.fine_grained_activation_offloading:
+ self.preprocess_for_fine_grained_offloading()
+
from ..common.model_chunk_schedule_plan import TransformerModelChunkSchedulePlan
return TransformerModelChunkSchedulePlan(
@@ -721,6 +798,7 @@ def build_schedule_plan(
extra_block_kwargs,
runtime_gather_output,
loss_mask,
+ padding_mask,
)
def sharded_state_dict(
@@ -749,14 +827,11 @@ def sharded_state_dict(
output_extra_state and output_extra_state.data
), f'Expected output layer extra state to be empty, got: {output_extra_state}'
- # Multi-Token Prediction (MTP) need both embedding layer and output layer in
- # mtp process stage.
+ # Multi-Token Prediction (MTP) need embedding layer in mtp process stage.
# If MTP is not placed in the pre processing stage, we need to maintain a copy of
# embedding layer in the mtp process stage and tie it to the embedding in the pre
# processing stage.
- # Also, if MTP is not placed in the post processing stage, we need to maintain a copy
- # of output layer in the mtp process stage and tie it to the output layer in the post
- # processing stage.
+ # Now MTP loss is computed in post processing stage, so the output_layer is not needed.
if self.mtp_process and not self.pre_process:
emb_weight_key = f'{prefix}embedding.word_embeddings.weight'
emb_weight = self.embedding.word_embeddings.weight
@@ -767,19 +842,5 @@ def sharded_state_dict(
tp_group=self.tp_group,
dp_cp_group=metadata['dp_cp_group'],
)
- if self.mtp_process and not self.post_process:
- # We only need to tie the output layer weight if share_embeddings_and_output_weights
- # is False. Because if share_embeddings_and_output_weights is True, the shared weight
- # will be stored in embedding layer, and output layer will not have any weight.
- if not self.share_embeddings_and_output_weights:
- output_layer_weight_key = f'{prefix}output_layer.weight'
- output_layer_weight = self.output_layer.weight
- tie_output_layer_state_dict(
- sharded_state_dict,
- output_layer_weight,
- output_layer_weight_key,
- tp_group=self.tp_group,
- dp_cp_group=metadata['dp_cp_group'],
- )
return sharded_state_dict
diff --git a/megatron/core/models/gpt/moe_module_specs.py b/megatron/core/models/gpt/moe_module_specs.py
index 4f090e4a024..62ee4537cfc 100755
--- a/megatron/core/models/gpt/moe_module_specs.py
+++ b/megatron/core/models/gpt/moe_module_specs.py
@@ -57,10 +57,12 @@ def get_moe_module_spec_for_backend(
experts = ModuleSpec(module=expert_module, submodules=expert_submodule)
# shared experts spec
- shared_experts = ModuleSpec(module=SharedExpertMLP, params={"gate": False}, submodules=mlp)
+ shared_experts = ModuleSpec(module=SharedExpertMLP, submodules=mlp)
# MoE module spec
moe_module_spec = ModuleSpec(
- module=MoELayer, submodules=MoESubmodules(experts=experts, shared_experts=shared_experts)
+ module=MoELayer,
+ submodules=MoESubmodules(experts=experts, shared_experts=shared_experts),
+ metainfo={"fuse_pre_mlp_layernorm": False},
)
return moe_module_spec
diff --git a/megatron/core/models/mamba/mamba_model.py b/megatron/core/models/mamba/mamba_model.py
index 378cf7e47d6..e4074eda806 100644
--- a/megatron/core/models/mamba/mamba_model.py
+++ b/megatron/core/models/mamba/mamba_model.py
@@ -267,9 +267,10 @@ def forward(
hidden_states.squeeze(1).unsqueeze(0)
).unsqueeze(1)
- logits, _ = self.output_layer(
- hidden_states, weight=output_weight, runtime_gather_output=runtime_gather_output
- )
+ if labels is None:
+ logits, _ = self.output_layer(
+ hidden_states, weight=output_weight, runtime_gather_output=runtime_gather_output
+ )
# Restore sequence parallel execution to the output layer if necessary.
if sequence_parallel_override:
@@ -284,6 +285,16 @@ def forward(
# [s b h] => [b s h]
return logits.transpose(0, 1).contiguous()
- loss = self.compute_language_model_loss(labels, logits)
+ loss = self.compute_output_layer_and_language_model_loss(
+ hidden_states,
+ labels,
+ weight=self.shared_embedding_or_output_weight(),
+ sequence_parallel_enabled=self.output_layer.sequence_parallel,
+ column_parallel_linear=self.output_layer,
+ col_linear_kwargs={
+ "weight": output_weight,
+ "runtime_gather_output": runtime_gather_output,
+ },
+ )
return loss
diff --git a/megatron/core/models/retro/config.py b/megatron/core/models/retro/config.py
index 1b486767264..4e45be30b2e 100644
--- a/megatron/core/models/retro/config.py
+++ b/megatron/core/models/retro/config.py
@@ -7,10 +7,11 @@
from megatron.core.transformer import TransformerConfig
from megatron.core.transformer.enums import AttnBackend
-from megatron.core.utils import is_te_min_version
+from megatron.core.utils import experimental_api, is_te_min_version
@dataclass
+@experimental_api
class RetroConfig(TransformerConfig):
"""Configuration object for Retro models."""
diff --git a/megatron/core/nccl_allocator.py b/megatron/core/nccl_allocator.py
index b46157e9d00..8eb4047634c 100644
--- a/megatron/core/nccl_allocator.py
+++ b/megatron/core/nccl_allocator.py
@@ -156,6 +156,37 @@ def init() -> None:
logging.info(f"[MCORE][NCCL_ALLOCATOR] Initialized NCCL Allocator")
+# register_mem_pool/deregister_mem_pool are used for manual (de)registration of the memory pool.
+# They are used in the case of FSDP manual registration.
+def register_mem_pool(pool, group, symmetric=True):
+ """
+ Register a memory pool to a group.
+ symmetric: bool, this is for future use.
+ """
+ backend = group._get_backend(torch.device("cuda", torch.cuda.current_device()))
+ if symmetric:
+ try:
+ backend.register_mem_pool(pool, symm=symmetric)
+ except TypeError:
+ # Older PyTorch/APIs without 'symm' keyword.
+ logging.warning(
+ f"[MCORE][NCCL_ALLOCATOR] Failed in symmetric registration."
+ f"Falling back to registration api without 'symm' keyword!!"
+ )
+ backend.register_mem_pool(pool)
+ else:
+ backend.register_mem_pool(pool)
+
+
+def deregister_mem_pool(pool, group):
+ """
+ Deregister a memory pool from a group.
+ """
+ backend = group._get_backend(torch.device("cuda", torch.cuda.current_device()))
+ if pool.snapshot():
+ backend.deregister_mem_pool(pool)
+
+
# Preserve the original APEX NCCL allocator interface for backward compatibility
class nccl_mem:
"""
@@ -314,3 +345,20 @@ def __exit__(self, *args):
f"{repr(group)}({desc}) group!!"
)
self.mem_context.__exit__(*args)
+
+
+class MemPoolAllocatorWithoutRegistration:
+ """
+ An allocator class that uses allocates memory without registering to any communication group.
+ Users are expected to register the memory manually to the communication groups.
+ """
+
+ def __init__(self, pool):
+ self.pool = pool
+ self.mem_context = torch.cuda.use_mem_pool(self.pool)
+
+ def __enter__(self):
+ self.mem_context.__enter__()
+
+ def __exit__(self, *args):
+ self.mem_context.__exit__(*args)
diff --git a/megatron/core/optimizer/__init__.py b/megatron/core/optimizer/__init__.py
index 061cb25f5b8..11aa6c49585 100644
--- a/megatron/core/optimizer/__init__.py
+++ b/megatron/core/optimizer/__init__.py
@@ -3,7 +3,7 @@
import logging
import warnings
from dataclasses import astuple
-from typing import Callable, Dict, List, Optional, Tuple, Union
+from typing import Any, Callable, Dict, List, Optional, Tuple, Union
import torch
from torch.optim import SGD as CPUSGD
@@ -35,6 +35,11 @@
from megatron.core import parallel_state
from megatron.core.optimizer.cpu_offloading.hybrid_optimizer import HybridDeviceOptimizer
+from megatron.core.optimizer_param_scheduler import (
+ ParamGroupOverride,
+ combine_param_group_overrides,
+ param_group_override_to_tuple,
+)
from megatron.core.process_groups_config import ProcessGroupCollection
from megatron.core.transformer.fsdp_dtensor_checkpoint import get_global_unique_param_name
@@ -50,66 +55,92 @@
MegatronOptimizer,
param_group_identifier_keys,
)
-from .optimizer_config import AdamOptimizerConfig, OptimizerConfig, ParamKey, SGDOptimizerConfig
+from .optimizer_config import (
+ AdamOptimizerConfig,
+ OptimizerConfig,
+ ParamKey,
+ ParamPredicate,
+ ParamWithNamePredicate,
+ SGDOptimizerConfig,
+)
logger = logging.getLogger(__name__)
-def _matches(param: torch.nn.Parameter, param_name: str, param_key: ParamKey) -> bool:
- """Returns true if passed-in parameter (with name) matches `param_key`.
+def get_standard_config_overrides(config: OptimizerConfig) -> Dict[ParamKey, ParamGroupOverride]:
+ """Get standard config overrides for the optimizer, handling decoupled LR and common wd skips.
Args:
- param (torch.nn.Parameter): Handle to parameter object.
- param_name (str): Name of parameter in underlying PyTorch module.
- param_key (ParamKey): ParamKey object.
+ config (OptimizerConfig): optimizer configuration object.
Returns:
- bool: True if parameter matches passed-in param_key.
+ Dict[ParamKey, ParamGroupOverride]: standard config overrides.
"""
-
- # Check if name matches.
- if isinstance(param_key.name, str):
- target_names = [param_key.name]
- else:
- target_names = list(param_key.name)
- for target_name in target_names:
- if param_name in target_name:
- return True
-
- # Check if attribute matches.
- if isinstance(param_key.attr, str):
- target_attrs = [param_key.attr]
+ config_overrides: Optional[Dict[ParamKey, ParamGroupOverride]] = {}
+ # First, figure out how we are going to do wd skipping. The two main approaches are:
+ # 1. The classic megatron approach of skipping all len 1 and bias parameters.
+ # 2. The Qwen3-Next approach of doing 1, other than qk layernorm parameters.
+ if config.apply_wd_to_qk_layernorm:
+ shape_1_not_qkln_param = ParamWithNamePredicate(
+ name="s1_not_qkln",
+ fn=lambda param, name: (len(param.shape) == 1 or name.endswith(".bias"))
+ and not ("q_layernorm." in name or "k_layernorm." in name),
+ )
+ param_wd_mult_key = ParamKey(with_name_predicate=shape_1_not_qkln_param)
else:
- target_attrs = list(param_key.attr)
- for target_attr in target_attrs:
- if getattr(param, target_attr, False):
- return True
+ param_length_1_match = ParamPredicate(
+ name="param_len_1", fn=lambda param: len(param.shape) == 1
+ )
+ param_wd_mult_key = ParamKey(name="*.bias", predicate=param_length_1_match)
+
+ config_overrides[param_wd_mult_key] = ParamGroupOverride(wd_mult=0.0)
- return False
+ if config.decoupled_lr is not None:
+ decoupled_lr_config: ParamGroupOverride = {"max_lr": config.decoupled_lr}
+ decoupled_param_key = ParamKey(attr="is_embedding_or_output_parameter")
+ if config.decoupled_min_lr is not None:
+ decoupled_lr_config["min_lr"] = config.decoupled_min_lr
+ config_overrides[decoupled_param_key] = decoupled_lr_config
+
+ return config_overrides
def _get_param_groups(
model_chunks: List[MegatronModule],
config: OptimizerConfig,
- config_overrides: Optional[Dict[ParamKey, OptimizerConfig]],
+ config_overrides: Optional[Dict[ParamKey, ParamGroupOverride]],
) -> List[Dict]:
"""Create parameter groups for optimizer.
Creates parameter groups from provided optimizer config object.
+ NOTE There can be more than one match between a ParamKey and a parameter.
+ What we do is merge all of the matching ParamKey overrides into a single ParamGroupOverride
+ for that parameter and use that as the key for that parameter. Any parameters that get
+ the same set of merged overrides will be mapped into the same parameter group.
+
Args:
model_chunks (List[MegatronModule]): model chunks to create parameter
groups for.
config (OptimizerConfig): optimizer configuration object.
- config_overrides (Optional[Dict[LayerKey, OptimizerConfig]): optimizer overrides,
- specified on a per-layer basis.
+ config_overrides (Optional[Dict[ParamKey, ParamGroupOverride]): optimizer overrides,
+ specified on a per-layer basis. NOTE: if you want to skip applying weight decay on bias
+ and length 1 parameters, and also do not want to do any other overrides, set this to an
+ empty dictionary rather than the default value of None.
Returns:
List of parameter groups.
"""
- # Map (wd_mult, is_expert_parallel, param_group_hyperparameters_config) to params.
+ # Map (pg_overrides, is_expert_parallel) to params.
params_map = {}
- configs_map = {}
+
+ if config_overrides is None:
+ # TODO remove this default behavior eventually.
+ # This is only needed for backwards compatibility with the old config overrides API where
+ # the config_overrides argument by default lead to bias parameters and length 1 parameters.
+ # We assume that users of decoupled LR already provide config overrides so will adapt
+ # to the new API.
+ config_overrides = get_standard_config_overrides(config=config)
for model_chunk in model_chunks:
for name, param in model_chunk.named_parameters():
@@ -117,47 +148,31 @@ def _get_param_groups(
continue
uses_default_config = False
- # Get optimizer config for this parameter.
- if config_overrides is None:
- config_for_param = config
- uses_default_config = True
+ # Get optimizer config overrides for this parameter.
+ param_overrides_list: list[ParamGroupOverride] = []
+ if config_overrides is not None:
+ for param_key, param_override in config_overrides.items():
+ if param_key.matches(param, name):
+ param_overrides_list.append(param_override)
+
+ if param_overrides_list:
+ param_override: ParamGroupOverride | None = combine_param_group_overrides(
+ param_overrides_list
+ )
else:
- config_for_param = None
- for param_key in config_overrides:
- if _matches(param, name, param_key):
- config_for_param = config_overrides[param_key]
- break
- # Fall back to default config.
- if config_for_param is None:
- config_for_param = config
- uses_default_config = True
+ param_override = None
is_expert_parallel = not getattr(param, 'allreduce', True)
- # TODO: Make sure there is a way to support old no_weight_decay_func functionality
- # and default_skip_embedding_weight_decay:
- # or (default_skip_embedding_weight_decay and "embedding" in name)
- no_wd = name.endswith(".bias") or len(param.shape) == 1
- if not no_wd:
- wd_mult = 1.0
- else:
- wd_mult = 0.0
-
- # Create config_tuple that is hash-able. Remove timers object before
- # creating config_tuple.
- config_for_param_copy = copy.deepcopy(config_for_param)
- config_for_param_copy.timers = None
- config_tuple = astuple(config_for_param_copy)
- key = (wd_mult, is_expert_parallel, config_tuple)
+ # Create config_tuple that is hash-able, and has a consistent ordering of the keys.
+ param_override_tuple: tuple[tuple[str, Any], ...] | None = (
+ param_group_override_to_tuple(param_override)
+ )
+ key = (param_override_tuple, is_expert_parallel)
if key not in params_map:
params_map[key] = []
params_map[key].append(param)
- if key in configs_map:
- assert (config_for_param, uses_default_config) == configs_map[key]
- else:
- configs_map[key] = (config_for_param, uses_default_config)
-
# Distributed checkpoint requires all ranks to have the same param groups,
# so we need to align the param groups across ranks, otherwise we may have
# runtime error when loading the checkpoint or numerical error when resuming training.
@@ -168,34 +183,47 @@ def _get_param_groups(
for key in keys:
if key not in params_key:
params_key.append(key)
-
+ # Need to pick one of the param_override_tuples to use for the param group.
param_groups = []
- for key in params_key:
- wd_mult, is_expert_parallel, _ = key
+ # Sort keys, None first.
+ for key in sorted(params_key, key=lambda x: (x[0] is not None, x[0])):
+ param_override_tuple, is_expert_parallel = key
params = params_map[key] if key in params_map else []
- config, uses_default_config = None, True
- if key not in configs_map:
- assert params == []
+ if param_override_tuple is None:
+ param_override: ParamGroupOverride = {}
else:
- config, uses_default_config = configs_map[key]
- assert config is not None
+ param_override: ParamGroupOverride = {k: v for (k, v) in param_override_tuple}
+
+ # False if param_group_override is None or empty tuple or if we do not modify the
+ # LR schedule.
+ # NOTE: "default_config" is used for logging the learning rate in training.py.
+ # so set to True if we do not modify the learning rate.
+ # if param_group['default_config']:
+ # learning_rate = param_group['lr']
+ uses_default_lr_schedule: bool = (not bool(param_override_tuple)) or not any(
+ ["lr" in k for k in param_override]
+ )
# TODO: Remove "backwards compatible" fields below eventually.
+ default_config: ParamGroupOverride = {
+ 'wd_mult': 1.0,
+ 'lr_mult': 1.0,
+ 'is_decoupled_lr': False,
+ # The following two fields may be important to keep even when we remove the
+ # above "backwards compatible" fields.
+ "max_lr": config.lr, # user may override this in param_override
+ "min_lr": config.min_lr, # user may override this in param_override
+ }
+ assert (
+ "params" not in param_override
+ ), "'params' should not be in param_override, this is a protected key"
param_group = {
'params': params,
- 'wd_mult': wd_mult, # For backwards compatibility.
- 'lr_mult': 1.0, # For backwards compatibility.
'is_expert_parallel': is_expert_parallel,
- 'is_decoupled_lr': False, # For backwards compatibility.
- 'default_config': uses_default_config,
+ 'default_config': uses_default_lr_schedule,
+ **default_config,
+ **param_override, # keep **param_override last so that users can override other fields.
}
-
- # Stick relevant fields into param_group from config object.
- if config is not None:
- param_group['max_lr'] = config.lr
- param_group['min_lr'] = config.min_lr
- # TODO: Add other relevant arguments (e.g., weight decay, optimizer)
- # here as well.
param_groups.append(param_group)
return param_groups
@@ -205,7 +233,7 @@ def _get_param_groups_and_buffers(
model_chunks: List[MegatronModule],
model_chunk_offset: int,
config: OptimizerConfig,
- config_overrides: Optional[Dict[ParamKey, OptimizerConfig]],
+ config_overrides: Optional[Dict[ParamKey, ParamGroupOverride]],
filter_fn: Callable,
buffer_name: str,
) -> Tuple[List[Dict], Dict[int, List[_ParamAndGradBuffer]]]:
@@ -216,8 +244,8 @@ def _get_param_groups_and_buffers(
groups for.
model_chunk_offset (int): offset of model_chunks in global model_chunks list.
config (OptimizerConfig): optimizer configuration object.
- config_overrides (Optional[Dict[LayerKey, OptimizerConfig]): optimizer overrides,
- specified on a per-layer basis.
+ config_overrides (Optional[Dict[ParamKey, ParamGroupOverride]): optimizer/scheduler
+ overrides, specified on the basis of ParamKey matches with each parameter.
lr (float): learning rate.
min_lr (float): minimum learning rate.
filter_fn (callable): filtering function for param_groups.
@@ -247,6 +275,7 @@ def _get_megatron_optimizer_based_on_param_groups(
data_parallel_group_idx: Optional[int] = None,
intra_dist_opt_group: Optional[torch.distributed.ProcessGroup] = None,
distributed_optimizer_instance_id: Optional[int] = 0,
+ pg_collection: Optional[ProcessGroupCollection] = None,
) -> MegatronOptimizer:
"""Get Megatron optimizer based on parameter groups.
@@ -436,13 +465,47 @@ def init_state_fn(opt, config=None):
optimizer = FP32Optimizer(optimizer, config, init_state_fn)
setattr(optimizer, 'grad_stats_parallel_group', model_parallel_group)
+ if pg_collection is None or not hasattr(pg_collection, 'tp'):
+ tp_group = parallel_state.get_tensor_model_parallel_group()
+ else:
+ tp_group = pg_collection.tp
+ # TODO(M4): plumb tp_group through optimizer constructors so this setattr disappears.
+ setattr(optimizer, 'tp_group', tp_group)
+
return optimizer
+def check_config_overrides_consistency(
+ config: OptimizerConfig, config_overrides: Optional[Dict[ParamKey, ParamGroupOverride]]
+):
+ """Check if the config overrides are consistent with the config."""
+
+ # TODO: Remove `optimizer` from this eventually (e.g., if we use Muon for some layers and
+ # Adam for other layers). This would need some more refactoring to work though (param_groups
+ # filtered by optimizer passed into _get_megatron_optimizer_based_on_param_groups).
+ if config_overrides is not None:
+ fields_to_check_for_consistency = [
+ 'overlap_param_gather_with_optimizer_step',
+ 'optimizer',
+ 'optimizer_cpu_offload',
+ ]
+ for field_name in fields_to_check_for_consistency:
+ base_field = getattr(config, field_name, None)
+ all_config_overrides = list(config_overrides.values())
+ for config_override in all_config_overrides:
+ if field_name in config_override:
+ field = config_override[field_name]
+ if field != base_field:
+ raise ValueError(
+ f"Field {field_name} should not be overriden in a config override."
+ )
+ return True
+
+
def get_megatron_optimizer(
config: OptimizerConfig,
model_chunks: List[MegatronModule],
- config_overrides: Optional[Dict[ParamKey, OptimizerConfig]] = None,
+ config_overrides: Optional[Dict[ParamKey, ParamGroupOverride]] = None,
use_gloo_process_groups: bool = True,
pg_collection: Optional[ProcessGroupCollection] = None,
dump_param_to_param_group_map: Optional[str] = None,
@@ -468,19 +531,7 @@ def get_megatron_optimizer(
log_single_rank(logger, logging.INFO, f'Setting up optimizer with config {config}')
- # TODO: Remove `optimizer` from this eventually (e.g., if we use Muon for some layers and
- # Adam for other layers). This would need some more refactoring to work though (param_groups
- # filtered by optimizer passed into _get_megatron_optimizer_based_on_param_groups).
- fields_to_check_for_consistency = [
- 'overlap_param_gather_with_optimizer_step',
- 'optimizer',
- 'optimizer_cpu_offload',
- ]
- for field_name in fields_to_check_for_consistency:
- field = getattr(config, field_name, None)
- if config_overrides is not None:
- all_configs = list(config_overrides.values())
- assert all([getattr(x, field_name, None) == field for x in all_configs])
+ check_config_overrides_consistency(config, config_overrides)
# Separate out first model chunk if overlapping param AG with optimizer step.
if config.overlap_param_gather_with_optimizer_step:
@@ -491,23 +542,23 @@ def get_megatron_optimizer(
overlap_param_gather_with_optimizer_step_flags = [False]
# Setup process groups using helper method
- process_groups = ProcessGroupCollection.setup_process_groups_for_optimizer(
+ process_groups_dict = ProcessGroupCollection.setup_process_groups_for_optimizer(
pg_collection, model_chunks, use_gloo_process_groups
)
- dp_cp_group = process_groups['dp_cp_group']
- intra_dp_cp_group = process_groups['intra_dp_cp_group']
- intra_expt_dp_group = process_groups['intra_expt_dp_group']
- mp_group = process_groups['mp_group']
- expt_tp_pp_group = process_groups['expt_tp_pp_group']
- intra_dp_cp_group_gloo = process_groups['intra_dp_cp_group_gloo']
- intra_expt_dp_group_gloo = process_groups['intra_expt_dp_group_gloo']
- intra_dist_opt_group = process_groups['intra_dist_opt_group']
+ dp_cp_group = process_groups_dict['dp_cp_group']
+ intra_dp_cp_group = process_groups_dict['intra_dp_cp_group']
+ intra_expt_dp_group = process_groups_dict['intra_expt_dp_group']
+ mp_group = process_groups_dict['mp_group']
+ expt_tp_pp_group = process_groups_dict['expt_tp_pp_group']
+ intra_dp_cp_group_gloo = process_groups_dict['intra_dp_cp_group_gloo']
+ intra_expt_dp_group_gloo = process_groups_dict['intra_expt_dp_group_gloo']
+ intra_dist_opt_group = process_groups_dict['intra_dist_opt_group']
model_parallel_rank = get_pg_rank(mp_group)
if get_pg_size(dp_cp_group) > get_pg_size(intra_dp_cp_group):
- inter_dist_opt_group = process_groups['inter_dist_opt_group']
+ inter_dist_opt_group = process_groups_dict['inter_dist_opt_group']
distributed_optimizer_instance_id = get_pg_rank(inter_dist_opt_group)
else:
distributed_optimizer_instance_id = 0
@@ -540,6 +591,7 @@ def get_megatron_optimizer(
data_parallel_group_idx=model_parallel_rank,
intra_dist_opt_group=intra_dist_opt_group,
distributed_optimizer_instance_id=distributed_optimizer_instance_id,
+ pg_collection=pg_collection,
)
)
model_chunk_offset += 1
@@ -587,6 +639,7 @@ def get_megatron_optimizer(
data_parallel_group_idx=model_parallel_rank,
intra_dist_opt_group=intra_dist_opt_group,
distributed_optimizer_instance_id=distributed_optimizer_instance_id,
+ pg_collection=pg_collection,
)
)
model_chunk_offset += 1
@@ -624,6 +677,7 @@ def get_megatron_optimizer(
data_parallel_group_idx=expt_model_parallel_rank,
intra_dist_opt_group=intra_dist_opt_group,
distributed_optimizer_instance_id=distributed_optimizer_instance_id,
+ pg_collection=pg_collection,
)
)
diff --git a/megatron/core/optimizer/clip_grads.py b/megatron/core/optimizer/clip_grads.py
index 70117858b77..cb2f23a685f 100644
--- a/megatron/core/optimizer/clip_grads.py
+++ b/megatron/core/optimizer/clip_grads.py
@@ -181,6 +181,7 @@ def count_zeros_fp32(
parameters: Union[List[torch.Tensor], torch.Tensor],
grad_stats_parallel_group: torch.distributed.ProcessGroup,
use_decoupled_grad: bool = False,
+ tp_group: Optional[torch.distributed.ProcessGroup] = None,
) -> float:
"""Counts the number of zeros in gradients associated with the passed-in list of
parameters.
@@ -218,7 +219,7 @@ def count_zeros_fp32(
grad_attr = "decoupled_grad" if use_decoupled_grad else "grad"
grad_not_none = hasattr(param, grad_attr) and getattr(param, grad_attr) is not None
is_not_shared = param_is_not_shared(param)
- is_not_tp_duplicate = param_is_not_tensor_parallel_duplicate(param)
+ is_not_tp_duplicate = param_is_not_tensor_parallel_duplicate(param, tp_group=tp_group)
if grad_not_none and is_not_shared and is_not_tp_duplicate:
grad_obj = getattr(param, grad_attr)
data_parallel_group = get_data_parallel_group_if_dtensor(grad_obj, data_parallel_group)
diff --git a/megatron/core/optimizer/cpu_offloading/optimizer_state_offloader.py b/megatron/core/optimizer/cpu_offloading/optimizer_state_offloader.py
new file mode 100644
index 00000000000..81fd116c8ba
--- /dev/null
+++ b/megatron/core/optimizer/cpu_offloading/optimizer_state_offloader.py
@@ -0,0 +1,315 @@
+# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved.
+
+"""Optimizer state offloading class."""
+
+from typing import TYPE_CHECKING, Dict, List, Tuple
+
+import torch
+
+if TYPE_CHECKING:
+ from megatron.core.optimizer.distrib_optimizer import DistributedOptimizer
+
+
+class OptimizerStateOffloader:
+ """
+ Manages offloading of optimizer states and master weights to CPU.
+ Used with DistributedOptimizer to reduce GPU memory usage.
+
+ Supports overlapped D2H/H2D transfers using CUDA streams.
+
+ Master weights can be stored in two locations:
+ - In adam optimizer state (when use_precision_aware_optimizer_no_fp8_or_ds_fp8 is True)
+ - In mcore's shard_fp32_from_float16_groups
+ """
+
+ OPTIMIZER_STATE_KEYS = ('exp_avg', 'exp_avg_sq')
+ MASTER_WEIGHT_KEY = 'master_param'
+
+ def __init__(self, distrib_optimizer: "DistributedOptimizer"):
+ """
+ Args:
+ distrib_optimizer: The DistributedOptimizer to offload states and master weights from.
+ """
+ self.dist_optimizer = distrib_optimizer
+ self.adam_optimizer = distrib_optimizer.optimizer
+
+ # Only support TE FusedAdam optimizer for now.
+ try:
+ from transformer_engine.pytorch.optimizers import FusedAdam
+
+ assert isinstance(self.adam_optimizer, FusedAdam), (
+ f"OptimizerStateOffloader requires TE FusedAdam optimizer, "
+ f"but got {type(self.adam_optimizer).__name__}"
+ )
+ except ImportError:
+ raise ImportError(
+ "OptimizerStateOffloader requires transformer_engine.pytorch.optimizers.FusedAdam"
+ )
+
+ # Check if master weights are stored in adam optimizer state
+ self.optimizer_contains_master_weights = self.adam_optimizer.master_weights
+
+ # CUDA streams for async transfers
+ self._d2h_stream = torch.cuda.Stream()
+ self._h2d_stream = torch.cuda.Stream()
+
+ # CPU buffers for optimizer states: {param: {key: cpu_tensor}}
+ self._opt_state_cpu_buffers: Dict[torch.Tensor, Dict[str, torch.Tensor]] = {}
+
+ # CPU buffers for mcore master weights, matching the structure of source groups
+ # List[List[cpu_tensor]]
+ self._shard_fp32_from_float16_cpu_buffers: List[List[torch.Tensor]] = []
+
+ # State tracking
+ self._offloaded = False
+ self._offloaded_state_keys: Tuple[str, ...] = ()
+ self._offloaded_mcore_master_weights = False
+
+ # Track whether optimizer states (exp_avg, exp_avg_sq) have been initialized.
+ # These are lazily initialized by FusedAdam during the first optimizer.step().
+ # Master weights (shard_fp32_from_float16_groups) are available from the start.
+ self._optimizer_states_initialized = False
+
+ def mark_optimizer_states_initialized(self):
+ """
+ Mark that optimizer states (exp_avg, exp_avg_sq) are now available.
+ Should be called after the first optimizer.step() completes.
+ """
+ self._optimizer_states_initialized = True
+
+ def _get_state_keys_to_offload(
+ self, offload_optimizer_states: bool, offload_master_weights: bool
+ ) -> Tuple[str, ...]:
+ """Get the state keys in FusedAdam to offload based on configuration."""
+ keys = []
+ # Skip optimizer states offloading if they haven't been initialized yet.
+ # Optimizer states are lazily initialized by FusedAdam during the first optimizer.step().
+ if self._optimizer_states_initialized:
+ if offload_optimizer_states:
+ keys.extend(self.OPTIMIZER_STATE_KEYS)
+ if offload_master_weights and self.optimizer_contains_master_weights:
+ keys.append(self.MASTER_WEIGHT_KEY)
+ return tuple(keys)
+
+ def _ensure_state_cpu_buffer(
+ self, param: torch.Tensor, state_key: str, gpu_tensor: torch.Tensor, pin_memory: bool = True
+ ) -> torch.Tensor:
+ """Get or create a CPU buffer for a state tensor."""
+ if param not in self._opt_state_cpu_buffers:
+ self._opt_state_cpu_buffers[param] = {}
+
+ if state_key not in self._opt_state_cpu_buffers[param]:
+ cpu_buffer = torch.empty(
+ gpu_tensor.size(),
+ dtype=gpu_tensor.dtype,
+ layout=gpu_tensor.layout,
+ device='cpu',
+ pin_memory=pin_memory,
+ )
+ self._opt_state_cpu_buffers[param][state_key] = cpu_buffer
+
+ return self._opt_state_cpu_buffers[param][state_key]
+
+ def _offload_shard_groups(
+ self,
+ shard_groups: List[List[torch.Tensor]],
+ cpu_buffers: List[List[torch.Tensor]],
+ pin_memory: bool = True,
+ ):
+ """Offload a shard group to CPU buffers."""
+ # Initialize CPU buffers on first call
+ if len(cpu_buffers) == 0:
+ for group in shard_groups:
+ group_buffers = []
+ for gpu_tensor in group:
+ cpu_buffer = torch.empty(
+ gpu_tensor.size(),
+ dtype=gpu_tensor.dtype,
+ layout=gpu_tensor.layout,
+ device='cpu',
+ pin_memory=pin_memory,
+ )
+ group_buffers.append(cpu_buffer)
+ cpu_buffers.append(group_buffers)
+
+ # Copy D2H
+ for group_idx, group in enumerate(shard_groups):
+ for param_idx, gpu_tensor in enumerate(group):
+ cpu_buffer = cpu_buffers[group_idx][param_idx]
+ cpu_buffer.copy_(gpu_tensor, non_blocking=pin_memory)
+ gpu_tensor.record_stream(self._d2h_stream)
+
+ def _offload_states(
+ self,
+ offload_optimizer_states: bool,
+ offload_master_weights: bool,
+ use_pin_memory: bool = True,
+ ):
+ """Offload optimizer states and/or master weights to CPU."""
+ # Offload states from adam optimizer
+ self._offloaded_state_keys = self._get_state_keys_to_offload(
+ offload_optimizer_states, offload_master_weights
+ )
+ states = self.adam_optimizer.state
+
+ for param, param_state in states.items():
+ for state_key in self._offloaded_state_keys:
+ if state_key not in param_state:
+ continue
+
+ gpu_tensor = param_state[state_key]
+ if not isinstance(gpu_tensor, torch.Tensor) or not gpu_tensor.is_cuda:
+ continue
+
+ cpu_buffer = self._ensure_state_cpu_buffer(
+ param, state_key, gpu_tensor, use_pin_memory
+ )
+ cpu_buffer.copy_(gpu_tensor, non_blocking=use_pin_memory)
+ gpu_tensor.record_stream(self._d2h_stream)
+
+ # Offload mcore master weights if not in optimizer state
+ if offload_master_weights and not self.optimizer_contains_master_weights:
+ self._offload_shard_groups(
+ self.dist_optimizer.shard_fp32_from_float16_groups,
+ self._shard_fp32_from_float16_cpu_buffers,
+ use_pin_memory,
+ )
+ self._offloaded_mcore_master_weights = True
+
+ def _release_states(self):
+ """Replace optimizer state GPU tensors with CPU tensors to free GPU memory."""
+ states = self.adam_optimizer.state
+
+ for param, param_state in states.items():
+ if param not in self._opt_state_cpu_buffers:
+ continue
+
+ for state_key in self._offloaded_state_keys:
+ if state_key not in self._opt_state_cpu_buffers[param]:
+ continue
+
+ param_state[state_key].untyped_storage().resize_(0)
+
+ if self._offloaded_mcore_master_weights:
+ for group in self.dist_optimizer.shard_fp32_from_float16_groups:
+ for gpu_tensor in group:
+ gpu_tensor.untyped_storage().resize_(0)
+
+ def _reload_shard_groups(
+ self,
+ shard_groups: List[List[torch.Tensor]],
+ cpu_buffers: List[List[torch.Tensor]],
+ is_allocate_stage: bool,
+ ):
+ """Reload shard groups from CPU to GPU."""
+ for group_idx, group in enumerate(shard_groups):
+ for param_idx, _ in enumerate(group):
+ cpu_buffer = cpu_buffers[group_idx][param_idx]
+ if is_allocate_stage:
+ shard_groups[group_idx][param_idx].untyped_storage().resize_(
+ cpu_buffer.untyped_storage().size()
+ )
+ else:
+ shard_groups[group_idx][param_idx].copy_(
+ cpu_buffer, non_blocking=cpu_buffer.is_pinned()
+ )
+
+ def _reload_states(self, is_allocate_stage: bool):
+ """
+ Reload optimizer states and/or master weights from CPU to GPU.
+
+ If is_allocate_stage is True, only allocate GPU memory for the states and master weights,
+ but do not copy the data from CPU to GPU. Otherwise, copy the data from CPU to GPU.
+ The two processes are separated to make sure that the GPU memory is allocated on the
+ default stream to avoid fragmentation.
+ """
+ # Reload states to adam optimizer
+ states = self.adam_optimizer.state
+
+ for param, param_state in states.items():
+ if param not in self._opt_state_cpu_buffers:
+ continue
+
+ for state_key in self._offloaded_state_keys:
+ if state_key not in self._opt_state_cpu_buffers[param]:
+ continue
+
+ cpu_buffer = self._opt_state_cpu_buffers[param][state_key]
+ if is_allocate_stage:
+ param_state[state_key].untyped_storage().resize_(
+ cpu_buffer.untyped_storage().size()
+ )
+ else:
+ param_state[state_key].copy_(cpu_buffer, non_blocking=cpu_buffer.is_pinned())
+
+ # Reload mcore master weights if not in optimizer state
+ if self._offloaded_mcore_master_weights:
+ self._reload_shard_groups(
+ self.dist_optimizer.shard_fp32_from_float16_groups,
+ self._shard_fp32_from_float16_cpu_buffers,
+ is_allocate_stage,
+ )
+
+ def offload(self, offload_optimizer_states: bool = True, offload_master_weights: bool = True):
+ """
+ Offload optimizer states and/or master weights to CPU.
+ Starts async D2H transfer that can overlap with other operations.
+
+ Args:
+ offload_optimizer_states: Whether to offload exp_avg, exp_avg_sq.
+ offload_master_weights: Whether to offload master weights.
+ """
+ if not offload_optimizer_states and not offload_master_weights:
+ return
+
+ # Wait for current stream finishing updating the optimizer states.
+ self._d2h_stream.wait_stream(torch.cuda.current_stream())
+
+ with torch.cuda.stream(self._d2h_stream):
+ self._offload_states(offload_optimizer_states, offload_master_weights)
+
+ self._offloaded = True
+
+ def release_gpu_memory(self):
+ """
+ Release GPU memory for optimizer states and master weights after D2H copy completes.
+
+ This is separated from offload() to allow delayed GPU memory release,
+ which is needed for mxfp8 + overlap_param_gather case where master weights
+ must remain on GPU until after _copy_main_params_to_param_buffer() is called.
+ """
+ if not self._offloaded:
+ return
+
+ self._release_states()
+
+ def reload(self):
+ """
+ Reload optimizer states and/or master weights from CPU to GPU.
+ Call before optimizer.step() to ensure states are on GPU.
+ """
+ if not self._offloaded:
+ return
+
+ # Allocate GPU memory on the current stream to avoid fragmentation.
+ self._reload_states(is_allocate_stage=True)
+
+ self._h2d_stream.wait_stream(self._d2h_stream)
+ self._h2d_stream.wait_stream(torch.cuda.current_stream())
+
+ # Reload states on the h2d stream to overlap with other operations.
+ with torch.cuda.stream(self._h2d_stream):
+ self._reload_states(is_allocate_stage=False)
+
+ self._offloaded_state_keys = ()
+ self._offloaded_mcore_master_weights = False
+ self._offloaded = False
+
+ def sync_before_step(self):
+ """
+ Wait for H2D reload to complete before optimizer.step().
+ Must be called to ensure states are on GPU before optimizer uses them.
+
+ This is separated from reload() to make it possible to move the reload ahead of time.
+ """
+ torch.cuda.current_stream().wait_stream(self._h2d_stream)
diff --git a/megatron/core/optimizer/distrib_optimizer.py b/megatron/core/optimizer/distrib_optimizer.py
index 6e093f96f7e..2f5876fa48a 100644
--- a/megatron/core/optimizer/distrib_optimizer.py
+++ b/megatron/core/optimizer/distrib_optimizer.py
@@ -49,6 +49,7 @@
from ..fp8_utils import dequantize_fp8_tensor, is_float8tensor, quantize_param_shard
from ..transformer.fsdp_dtensor_checkpoint import handle_experts_in_state_dict
from ..transformer.module import MegatronModule
+from .cpu_offloading.optimizer_state_offloader import OptimizerStateOffloader
from .grad_scaler import MegatronGradScaler
from .optimizer import MixedPrecisionOptimizer, _zero_grad_group_helper, param_group_identifier_keys
from .optimizer_config import OptimizerConfig
@@ -516,6 +517,8 @@ def __init__(
"due to checkpointing requirements."
)
+ self._state_offloader: Optional[OptimizerStateOffloader] = None
+
# when freezing sub-models we have no real optimizer
# but still need a stub DistributedOptimizer class
if optimizer is None:
@@ -604,6 +607,9 @@ def __init__(
self.optimizer.param_groups = [g["orig_group"] for g in self.opt_group_ranges]
self.optimizer.load_state_dict(self.optimizer.state_dict())
+ if self.config.offload_optimizer_states:
+ self._state_offloader = OptimizerStateOffloader(self)
+
def _get_model_param_range_map(self, param: torch.nn.Parameter):
"""
Given a model param, get the index sub-range of the param that this
@@ -2580,6 +2586,8 @@ def step_with_ready_grads(self) -> bool:
Under the hood, either launch synchronous param all-gathers or get ready to launch
asynchorous all-gathers that get overlapped with the next forward pass.
"""
+ if self._state_offloader is not None:
+ self._state_offloader.sync_before_step()
update_successful = super().step_with_ready_grads()
timers = self.config.timers
@@ -2600,4 +2608,22 @@ def step_with_ready_grads(self) -> bool:
if timers is not None:
timers('params-all-gather').stop()
+ if self._state_offloader is not None:
+ self._state_offloader.mark_optimizer_states_initialized()
+
return update_successful
+
+ def offload_states(self):
+ """Offload states to CPU."""
+ if self._state_offloader is not None:
+ self._state_offloader.offload()
+
+ def reload_offloaded_states(self):
+ """Start async reload of offloaded states."""
+ if self._state_offloader is not None:
+ self._state_offloader.reload()
+
+ def release_offloaded_gpu_states(self):
+ """Release GPU memory after D2H completes. For delayed release case."""
+ if self._state_offloader is not None:
+ self._state_offloader.release_gpu_memory()
diff --git a/megatron/core/optimizer/layer_wise_optimizer.py b/megatron/core/optimizer/layer_wise_optimizer.py
new file mode 100644
index 00000000000..de4396a5b4f
--- /dev/null
+++ b/megatron/core/optimizer/layer_wise_optimizer.py
@@ -0,0 +1,314 @@
+# Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+
+import logging
+from typing import Callable, List, Optional
+
+import torch
+from torch._utils import _flatten_dense_tensors, _unflatten_dense_tensors
+
+from megatron.core.dist_checkpointing.dict_utils import nested_values
+from megatron.core.dist_checkpointing.mapping import LocalNonpersistentObject, ShardedStateDict
+from megatron.core.process_groups_config import ProcessGroupCollection
+from megatron.core.utils import get_pg_rank, get_pg_size
+
+from .clip_grads import count_zeros_fp32, get_grad_norm_fp32
+from .optimizer import (
+ ChainedOptimizer,
+ Float16OptimizerWithFloat16Params,
+ FP32Optimizer,
+ MegatronOptimizer,
+)
+from .optimizer_config import OptimizerConfig
+
+logger = logging.getLogger(__name__)
+
+
+class LayerWiseDistributedOptimizer(ChainedOptimizer):
+ """Layer-wise distributed optimizer for Megatron-core models.
+
+ Experimental distributed optimizer wrapper that distributes weight to DP ranks by layer.
+ Implemented as ChainedOptimizer to support multiple optimizers (e.g. muon + adamW)
+ When using, keep all megatron distributed-optimizer related options OFF.
+
+ How LayerWiseDistributedOptimizer work:
+ 1. weights are splited into lists and each rank only keep its shard in its optimizer
+ 2. Megatron DDP handle allreduce grad, note that each rank have full model and grad
+ 3. optimizer is already modified so only param belong to this DP rank is updated
+ 4. grad_norm and zero counting will reduce metrics globally in step function
+ 5. Do regular update with chained optimizers, modified optimizer only update shard
+ 6. allgather updated params to every rank
+ """
+
+ def __init__(
+ self,
+ optimizers: List[MegatronOptimizer],
+ config: OptimizerConfig,
+ pg_collection: Optional[ProcessGroupCollection] = None,
+ init_state_fn_list: Optional[List[Callable]] = None,
+ ) -> None:
+ """
+ Initialize LayerWiseDistributedOptimizer.
+
+ Args:
+ optimizers: List of MegatronOptimizers.
+ config: OptimizerConfig.
+ pg_collection: ProcessGroupCollection.
+ init_state_fn_list: List of init state functions.
+ """
+
+ self.pg_collection = pg_collection
+ self.shard_params(optimizers)
+ if init_state_fn_list:
+ assert len(init_state_fn_list) == len(
+ optimizers
+ ), "init_state_fn_list must be the same length as optimizers if provided"
+
+ # wrap optimizer after sharding to avoid unnecessary master weight creation
+ # for higher precision, optimizers are wrapped with megatron already
+ if config.bf16:
+ # unwrap FP32 optimizer, possibly from reusing get_megatron_optimizer for adam
+ for i in range(len(optimizers)):
+ opt = optimizers[i]
+ if isinstance(opt, Float16OptimizerWithFloat16Params):
+ raise TypeError(
+ 'LayerWiseDistributedOptimizer received Float16 optimizer already.'
+ )
+ # unwrap FP32 optimizer from reusing get_megatron_optimizer for adam
+ if isinstance(opt, FP32Optimizer):
+ opt = opt.optimizer
+ optimizers[i] = Float16OptimizerWithFloat16Params(
+ opt, config, None, init_state_fn_list[i] if init_state_fn_list else None
+ )
+
+ super().__init__(optimizers)
+
+ # TODO(kunlun, deyuf): potential future perf optimization
+ # since allreduce is unchanged and handled by megatron DDP, they're already in
+ # contiguous gbuf. So instead of shard param by layer randomly, we can shard by
+ # buf range but keep some "extras" to keep boundary weight not sharded.
+ # This way each rank do some duplicated work but allgather_v is no longer needed
+ # All current distopt optimization can also be potentially applied
+
+ def shard_params(self, optimizers):
+ """Shard all params into lists by rank."""
+ # list of parameter are sorted by numel and assigned to ranks in ping-pong style
+ # example of 4 ranks and 10 parameters p0-p9 after sorting, then dp_cp_params_list will be
+ # [[p0, p7, p8], [p1, p6, p9], [p2, p5], [p3, p4]]
+
+ # simplify when dp_cp group size is 1
+ if get_pg_size(self.pg_collection.dp_cp) == 1:
+ self.dp_cp_params_list = None
+ self.expt_dp_params_list = None
+ return
+
+ dp_cp_idx, expt_dp_idx = 0, 0
+ dp_cp_size = get_pg_size(self.pg_collection.dp_cp)
+ expt_dp_size = get_pg_size(self.pg_collection.expt_dp)
+ # create ping-pong style loop so memory is more balanced
+ dp_cp_loop = list(range(dp_cp_size)) + list(range(dp_cp_size))[::-1]
+ expt_dp_loop = list(range(expt_dp_size)) + list(range(expt_dp_size))[::-1]
+ self.dp_cp_params_list = [[] for _ in range(dp_cp_size)]
+ self.expt_dp_params_list = [[] for _ in range(expt_dp_size)]
+ # get all param groups
+ param_groups = []
+ for optimizer in optimizers:
+ param_groups += optimizer.param_groups
+
+ # sort param in all groups by param numel and assign to each rank evenly
+ param_list = []
+ for group_index, group in enumerate(param_groups):
+ for p in group["params"]:
+ param_list.append((p, group_index))
+ param_list.sort(key=lambda x: x[0].numel())
+ param_groups_this_rank = [[] for g in param_groups]
+
+ # assign params to rank in ping-pong style loop
+ for p, group_index in param_list:
+ if param_groups[group_index].get("is_expert_parallel", False):
+ if expt_dp_loop[expt_dp_idx] == get_pg_rank(self.pg_collection.expt_dp):
+ param_groups_this_rank[group_index].append(p)
+ self.expt_dp_params_list[expt_dp_loop[expt_dp_idx]].append(p)
+ expt_dp_idx = (expt_dp_idx + 1) % len(expt_dp_loop)
+ else:
+ if dp_cp_loop[dp_cp_idx] == get_pg_rank(self.pg_collection.dp_cp):
+ param_groups_this_rank[group_index].append(p)
+ self.dp_cp_params_list[dp_cp_loop[dp_cp_idx]].append(p)
+ dp_cp_idx = (dp_cp_idx + 1) % len(dp_cp_loop)
+
+ # now we modify the group to only handle local params
+ for groups, params in zip(param_groups, param_groups_this_rank):
+ groups["params"] = params
+
+ # simplify when expt_dp group size is 1 or expert parallel is off
+ if expt_dp_size == 1 or len(self.expt_dp_params_list[0]) == 0:
+ self.expt_dp_params_list = None
+
+ @torch.no_grad()
+ def allgather_params(self) -> None:
+ """All-gather updated params from all ranks."""
+
+ # helper function to flatten local params, allgather, unflatten and copy to model params
+ def _allgather_helper(params_list, group):
+ # flatten this rank's params and create empty tensor output list
+ device = params_list[0][0].device
+ dtype = params_list[0][0].dtype
+ rank = get_pg_rank(group)
+ # for rank without params create empty tensor and participate in allgather
+ src = (
+ _flatten_dense_tensors(params_list[rank])
+ if len(params_list[rank]) > 0
+ else torch.empty(0, device=device, dtype=dtype)
+ )
+ output_list = [
+ torch.empty(sum([p.numel() for p in params]), device=device, dtype=dtype)
+ for params in params_list
+ ]
+ # single all_gather_v to collect all updated params
+ torch.distributed.all_gather(output_list, src, group=group)
+ # unflatten and copy gathered params for each rank i
+ for idx, (flat_params, params) in enumerate(zip(output_list, params_list)):
+ # skip local params and empty tensors
+ if len(params) == 0 or idx == rank:
+ continue
+ updated_params = _unflatten_dense_tensors(flat_params, params)
+ for updated_p, model_p in zip(updated_params, params):
+ model_p.data.copy_(updated_p)
+
+ if self.pg_collection is None:
+ return
+ if self.dp_cp_params_list:
+ _allgather_helper(self.dp_cp_params_list, self.pg_collection.dp_cp)
+ if self.expt_dp_params_list:
+ _allgather_helper(self.expt_dp_params_list, self.pg_collection.expt_dp)
+
+ @torch.no_grad()
+ def broadcast_params(self):
+ """All rank broadcast updated local params."""
+ # Broadcast linear layer weights to all other ranks. Kept as reference test.
+ if self.dp_cp_params_list is None:
+ return
+ for i, params in enumerate(self.dp_cp_params_list):
+ src_global_rank = torch.distributed.get_global_rank(self.pg_collection.dp_cp, i)
+ for p in params:
+ torch.distributed.broadcast(p, src_global_rank, self.pg_collection.dp_cp)
+ if self.expt_dp_params_list is None:
+ return
+ for i, params in enumerate(self.expt_dp_params_list):
+ src_global_rank = torch.distributed.get_global_rank(self.pg_collection.expt_dp, i)
+ for p in params:
+ torch.distributed.broadcast(p, src_global_rank, self.pg_collection.expt_dp)
+
+ @torch.no_grad()
+ def get_grad_norm(self):
+ # similar to dist opt, always aggregate globally
+ grads_for_norm = []
+ for optimizer in self.chained_optimizers:
+ grads_for_norm += optimizer.get_main_grads_for_grad_norm()
+ grad_norm = get_grad_norm_fp32(grads_for_norm, grad_stats_parallel_group=None)
+ return grad_norm
+
+ @torch.no_grad()
+ def count_zeros(self):
+ params = []
+ for optimizer in self.chained_optimizers:
+ params += optimizer.get_parameters()
+ return count_zeros_fp32(
+ params,
+ grad_stats_parallel_group=None,
+ use_decoupled_grad=self.config.use_precision_aware_optimizer_no_fp8_or_ds_fp8,
+ )
+
+ @torch.no_grad()
+ def step(self): # type: ignore[no-untyped-def]
+ """step function for layer-wise optimizer."""
+ update_successful, grad_norm, num_zeros_in_grad = super().step()
+
+ # All gather updated params.
+ self.allgather_params()
+
+ return update_successful, grad_norm, num_zeros_in_grad
+
+ # TODO(deyuf): need to improve dist checkpointing design to properly handle this
+ # fp32_from_fp16_params is list, each sub list could be empty if group is empty
+ # this breaks dist checkpointing assumption since extract_sharded_base drop list structure
+ # for now, we convert it to dict with index as key and convert back in load_state_dict
+ def load_state_dict(self, state_dict):
+ if len(self.chained_optimizers) == 1:
+ wrapped_state_dict = {1: state_dict}
+ else:
+ wrapped_state_dict = state_dict
+ for sd in wrapped_state_dict.values():
+ if 'fp32_from_fp16_params' in sd and isinstance(sd['fp32_from_fp16_params'], dict):
+ logger.info('[layerwise] converting fp32_from_fp16_params from dict to list')
+ sd['fp32_from_fp16_params'] = [
+ v for k, v in sorted(sd['fp32_from_fp16_params'].items())
+ ]
+ super().load_state_dict(state_dict)
+
+ def sharded_state_dict(
+ self, model_sharded_state_dict: ShardedStateDict, is_loading: bool = False, **kwargs
+ ):
+ """
+ Sharded state dict for torch_dist format checkpointing.
+ For fixed DP usage only, set replica_id to 0 for all ShardedTensor.
+ """
+ sharded_state_dict = super().sharded_state_dict(
+ model_sharded_state_dict, is_loading, **kwargs
+ )
+
+ # for fixed DP usage only
+ for sh_base in nested_values(sharded_state_dict):
+ if hasattr(sh_base, 'replica_id'):
+ assert (
+ isinstance(sh_base.replica_id, int) or len(sh_base.replica_id) == 3
+ ), f'Expected replica_id as int or (PP, TP, DP), got: {sh_base}'
+ sh_base.replica_id = (
+ 0 if isinstance(sh_base.replica_id, int) else (*sh_base.replica_id[:2], 0)
+ )
+
+ # later code assume list but chained optimizer fallback to non-list if there's only one
+ if len(self.chained_optimizers) == 1:
+ wrapped_sharded_state_dict = {1: sharded_state_dict}
+ else:
+ wrapped_sharded_state_dict = sharded_state_dict
+
+ # Adjust dict rank 0 output correct global metadata into common_dict
+ for sd in wrapped_sharded_state_dict.values():
+ # wrap empty containers into LocalNonpersistentObject so it won't be saved/loaded
+ # params is already wrapped, we only need to handle fp32_from_fp16_params and state
+ # more details in load_state_dict comment
+ if 'fp32_from_fp16_params' in sd:
+ sd['fp32_from_fp16_params'][:] = [
+ group if group else LocalNonpersistentObject(group)
+ for group in sd['fp32_from_fp16_params']
+ ]
+ sd['fp32_from_fp16_params'] = {
+ i: v for i, v in enumerate(sd['fp32_from_fp16_params'])
+ }
+ # state is a single dict and will be empty if optimizer is fully empty
+ if not sd['optimizer']['state']:
+ sd['optimizer']['state'] = LocalNonpersistentObject(sd['optimizer']['state'])
+ # group keys(e.g. 'step') might be missing or not updated
+ for i, group in enumerate(sd['optimizer']['param_groups']):
+ # keep local param tensor so we only gather metadata
+ local_params = group.pop('params')
+ # save whether this group is empty, so we can use non-empty rank for metadata
+ group['params'] = bool(local_params.unwrap())
+ all_rank_groups = [None for _ in range(torch.distributed.get_world_size())]
+ torch.distributed.all_gather_object(all_rank_groups, group)
+ # find first non-empty group if it exists
+ nonempty_rank_group = next((g for g in all_rank_groups if g['params']), group)
+ nonempty_rank_group['params'] = local_params
+ sd['optimizer']['param_groups'][i] = nonempty_rank_group
+ return sharded_state_dict
+
+ def save_state_dict_to_file(self, filename: str) -> None:
+ """Save the parameter state of the optimizer. For torch format only.
+ Args:
+ filename: The filename to save the parameter state.
+ """
+ torch.save(super().state_dict(), filename)
+
+ def load_state_dict_from_file(self, filename: str) -> None:
+ """Load the parameter state of the optimizer. For torch format only."""
+ super().load_state_dict(torch.load(filename))
diff --git a/megatron/core/optimizer/muon.py b/megatron/core/optimizer/muon.py
new file mode 100644
index 00000000000..b909ab61a8a
--- /dev/null
+++ b/megatron/core/optimizer/muon.py
@@ -0,0 +1,332 @@
+# Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+
+"""Megatron muon optimizer wrapper to handle tensor-parallel."""
+
+import logging
+from typing import Any, Callable, Dict, List, Literal, Optional
+
+import torch
+from torch.optim.optimizer import ParamsT
+
+from megatron.core.process_groups_config import ProcessGroupCollection
+from megatron.core.transformer.module import MegatronModule
+from megatron.core.utils import get_pg_size, log_single_rank
+
+from . import _get_param_groups, get_megatron_optimizer
+from .layer_wise_optimizer import LayerWiseDistributedOptimizer
+from .optimizer import (
+ ChainedOptimizer,
+ Float16OptimizerWithFloat16Params,
+ FP32Optimizer,
+ MegatronOptimizer,
+)
+from .optimizer_config import OptimizerConfig, ParamKey
+
+try:
+ from emerging_optimizers.orthogonalized_optimizers import (
+ OrthogonalizedOptimizer,
+ get_muon_scale_factor,
+ )
+ from emerging_optimizers.orthogonalized_optimizers.muon_utils import newton_schulz_tp
+
+ HAVE_EMERGING_OPTIMIZERS = True
+except ImportError:
+ HAVE_EMERGING_OPTIMIZERS = False
+ OrthogonalizedOptimizer = object
+
+
+logger = logging.getLogger(__name__)
+
+
+class TensorParallelMuon(OrthogonalizedOptimizer):
+ """Tensor Parallel Muon optimizer."""
+
+ def __init__(
+ self,
+ params: ParamsT,
+ lr: float = 3e-4,
+ momentum_beta: float = 0.95,
+ use_nesterov: bool = True,
+ weight_decay: float = 0.01,
+ use_decoupled_weight_decay: bool = True,
+ split_qkv: bool = False,
+ is_qkv_fn: Callable[[torch.Tensor], bool] | None = None,
+ qkv_split_shapes: tuple[int, int, int] | None = None,
+ fp32_matmul_prec: str = "medium",
+ coefficient_type: str = "quintic",
+ num_ns_steps: int = 5,
+ scale_mode: str = "spectral",
+ extra_scale_factor: float = 1.0,
+ pg_collection: Optional[ProcessGroupCollection] = None,
+ mode: Literal["blockwise", "duplicated", "distributed"] = "duplicated",
+ ) -> None:
+ if num_ns_steps < 1:
+ raise ValueError(f"num_ns_steps must be at least 1, got {num_ns_steps}")
+
+ def scaled_orthogonalize_fn(
+ grad: torch.Tensor,
+ tp_group: torch.distributed.ProcessGroup,
+ partition_dim: int | None = None,
+ ) -> torch.Tensor:
+ log_single_rank(
+ logger,
+ logging.DEBUG,
+ f'Orthogonalizing grad with {num_ns_steps} steps, {coefficient_type} coefficient, '
+ f'{scale_mode} scale mode, extra_scale_factor={extra_scale_factor}',
+ )
+ size = [grad.size(-2), grad.size(-1)]
+ if partition_dim is not None:
+ size[partition_dim] *= get_pg_size(tp_group)
+ orth_grad = newton_schulz_tp(
+ grad,
+ steps=num_ns_steps,
+ coefficient_type=coefficient_type,
+ tp_group=tp_group,
+ partition_dim=partition_dim,
+ mode="duplicated" if mode == "blockwise" else mode,
+ )
+ scale_factor = get_muon_scale_factor(size[0], size[1], mode=scale_mode)
+ return orth_grad * scale_factor * extra_scale_factor
+
+ self.pg_collection = pg_collection
+ self.mode = mode
+ self.split_qkv = split_qkv
+ self.is_qkv_fn = is_qkv_fn
+ self.qkv_split_shapes = qkv_split_shapes
+
+ weight_decay_method = "decoupled" if use_decoupled_weight_decay else "l2"
+ super().__init__(
+ params,
+ lr,
+ momentum_beta,
+ use_nesterov=use_nesterov,
+ weight_decay=weight_decay,
+ weight_decay_method=weight_decay_method,
+ fp32_matmul_prec=fp32_matmul_prec,
+ scaled_orthogonalize_fn=scaled_orthogonalize_fn,
+ )
+
+ def orthogonalize(self, p: torch.Tensor, grad: torch.Tensor, **kwargs: Any) -> torch.Tensor:
+ """Orthogonalize the momentum.
+
+ Args:
+ p: The parameter tensor. i is necessary to pass param tensor in addition to momentum
+ because a lot of information is only available in the param tensor,
+ attributes for example.
+ grad: The momentum tensor.
+
+ Returns:
+ The orthogonalized gradient tensor.
+ """
+ # TODO(deyuf): switch to group
+ if self.pg_collection:
+ tp_group = (
+ self.pg_collection.expt_tp
+ if getattr(p, 'expert_tp', False)
+ else self.pg_collection.tp
+ )
+ else:
+ tp_group = None
+ partition_dim = None if self.mode == "blockwise" else getattr(p, "partition_dim", None)
+ if partition_dim == -1:
+ # emerging-optimizers use None instead of -1 to indicate no tensor parallel
+ partition_dim = None
+
+ if self.split_qkv and self.is_qkv_fn(p): # type: ignore[misc]
+ # split grouped attention parameters (e.g., QKV, GQA, etc.)
+ grad_shape = grad.shape
+ log_single_rank(
+ logger,
+ logging.DEBUG,
+ f'qkv split grad shape {grad_shape}, split shapes {self.qkv_split_shapes}',
+ )
+ num_query_groups = grad_shape[0] // sum(self.qkv_split_shapes)
+ qkv_grads = torch.split(
+ grad.view(num_query_groups, sum(self.qkv_split_shapes), -1),
+ self.qkv_split_shapes,
+ dim=1,
+ )
+ qkv_grads = [g.reshape(-1, grad_shape[-1]) for g in qkv_grads]
+
+ # Apply Newton-Schulz and scales to each component, concat back
+ qkv_grads = [
+ self.scaled_orthogonalize_fn(g, tp_group, partition_dim).view(
+ num_query_groups, -1, grad_shape[-1]
+ )
+ for g in qkv_grads
+ ]
+ grad = torch.cat(qkv_grads, dim=1).view(grad_shape)
+ else:
+ grad = self.scaled_orthogonalize_fn(grad, tp_group, partition_dim)
+ return grad
+
+
+def get_megatron_muon_optimizer(
+ config: OptimizerConfig,
+ model_chunks: List[MegatronModule],
+ config_overrides: Optional[Dict[ParamKey, OptimizerConfig]] = None,
+ use_gloo_process_groups: bool = True,
+ layer_wise_distributed_optimizer: bool = False,
+ pg_collection: Optional[ProcessGroupCollection] = None,
+) -> MegatronOptimizer:
+ """This function is used to get the muon optimizer for the model chunks.
+ It is used to get the muon optimizer for the model chunks.
+
+ Args:
+ config (OptimizerConfig): optimizer configuration object.
+ model_chunks (List[MegatronModule]): model chunks to get optimizer for.
+ use_gloo_process_groups (bool): if false, disable use of Gloo process groups
+ in underlying Megatron optimizers.
+ layer_wise_distributed_optimizer (bool): if true, use layer-wise distributed optimizer.
+ Defaults to False.
+ """
+ # Muon currently use adam config. setting str here to call regular get for adam creation
+ # side effect is muon optimizer will have wrong name, i.e. config.optimizer == 'adam'
+ config.optimizer = 'adam'
+
+ assert HAVE_EMERGING_OPTIMIZERS, "Emerging Optimizers is not installed."
+
+ # dist-optim is not supported due to strong coupling with how DDP init grad buffer
+ # in thoery we can put some weight to use non-dist-muon and rest to dist-adam
+ # but there are strong dependency and assumption in DDP that prevent it
+ if config.use_distributed_optimizer:
+ raise Exception('muon with dist optimizer is not supported.')
+
+ # before this function receive properly created collection
+ if pg_collection is None:
+ pg_collection = ProcessGroupCollection.use_mpu_process_groups()
+
+ log_single_rank(logger, logging.INFO, f'Setting up emerging optimizer with config {config}')
+
+ optimizers = []
+ # record list of non/linear params
+ linear_params = []
+ nonlinear_params = []
+
+ for model_chunk in model_chunks:
+ # use config to determine qkv split shapes.
+ # no need to check tp since tp splits by head and this is per head(group) dimension
+ num_attention_heads = model_chunk.config.num_attention_heads
+ num_query_groups = model_chunk.config.num_query_groups
+ kv_channels = model_chunk.config.kv_channels
+ qkv_split_shapes = [
+ num_attention_heads // num_query_groups * kv_channels,
+ kv_channels,
+ kv_channels,
+ ]
+ for name, param in model_chunk.named_parameters():
+ if not param.requires_grad:
+ continue
+ # add flag for expert weight so optimizer can figure which tp group it uses
+ # alternatively, create new param group and save tp_group. this require more
+ # change in optimizer
+ if 'experts' in name and 'shared' not in name:
+ param.expert_tp = True
+ # add flag for qkv parameter
+ # TODO(deyuf): support MLA
+ if 'linear_qkv.weight' in name and len(param.shape) == 2:
+ param.is_qkv = True
+ # TODO(deyuf): currently only allow 2D non-embedding weight to avoid breaking
+ if (
+ not getattr(param, 'is_embedding_or_output_parameter', False)
+ and len(param.shape) == 2
+ ):
+ linear_params.append(param)
+ else:
+ nonlinear_params.append(param)
+
+ # freezing nonlinear params and get param groups for muon
+ for param in nonlinear_params:
+ param.requires_grad = False
+
+ linear_param_groups = _get_param_groups(model_chunks, config, config_overrides)
+
+ optimizer = TensorParallelMuon(
+ linear_param_groups,
+ lr=config.lr,
+ momentum_beta=config.muon_momentum,
+ use_nesterov=config.muon_use_nesterov,
+ weight_decay=config.weight_decay,
+ fp32_matmul_prec=config.muon_fp32_matmul_prec,
+ num_ns_steps=config.muon_num_ns_steps,
+ scale_mode=config.muon_scale_mode,
+ split_qkv=config.muon_split_qkv,
+ is_qkv_fn=lambda p: getattr(p, 'is_qkv', False),
+ qkv_split_shapes=qkv_split_shapes,
+ extra_scale_factor=config.muon_extra_scale_factor,
+ pg_collection=pg_collection,
+ mode=config.muon_tp_mode,
+ )
+
+ # Needed for torch_dist ckpt_format, unlike torch ckpt_format
+ # For other emerging optimizers, need to implement init_state_fn as well
+ # TODO(boxiangw): Improve usability after optimizer refactor
+ # TODO(boxiangw): support precision aware optimizer
+ def muon_init_state_fn(opt, config=None):
+ for group in opt.param_groups:
+ for p in group['params']:
+ if len(opt.state[p]) == 0:
+ opt.state[p]['momentum_buffer'] = torch.zeros_like(p.data)
+
+ def adam_init_state_fn(opt, config=None):
+ for group in opt.param_groups:
+ for p in group['params']:
+ if len(opt.state[p]) == 0:
+ if config is None or not config.use_precision_aware_optimizer:
+ opt.state[p]['exp_avg'] = torch.zeros_like(p.data)
+ opt.state[p]['exp_avg_sq'] = torch.zeros_like(p.data)
+ else:
+ opt.initialize_state(p)
+
+ # need to wrap into megatron mix precision optimizer. (only support bf16 w/o loss scale now)
+ if config.fp16:
+ raise Exception('muon with fp16 is not supported.')
+
+ reset_config_bf16 = False
+ if config.bf16:
+ if layer_wise_distributed_optimizer:
+ # creating master weight before layerwise sharding will lead to unnecessary master
+ # weight so here we delay master weight creation into layer_wise unset config.bf16
+ # will also result in all optimizers below(adam) to also not be wrapped
+ config.bf16 = False
+ reset_config_bf16 = True
+ else:
+ # if not using layer_wise wrapper, just create master weight here is fine
+ optimizer = Float16OptimizerWithFloat16Params(
+ optimizer, config, None, muon_init_state_fn
+ )
+ else:
+ optimizer = FP32Optimizer(optimizer, config, muon_init_state_fn)
+
+ optimizers.append(optimizer)
+
+ # done with muon, unfreeze nonlinear and freeze linear
+ for param in nonlinear_params:
+ param.requires_grad = True
+ for param in linear_params:
+ param.requires_grad = False
+
+ # call original get. linear params will be skipped since they're freezed
+ chained_adam = get_megatron_optimizer(
+ config,
+ model_chunks,
+ config_overrides=config_overrides,
+ use_gloo_process_groups=use_gloo_process_groups,
+ )
+
+ # unfreeze everything
+ for param in linear_params:
+ param.requires_grad = True
+
+ # chain everything together
+ init_fns = [muon_init_state_fn] + len(chained_adam.chained_optimizers) * [adam_init_state_fn]
+ optimizers += chained_adam.chained_optimizers
+
+ if layer_wise_distributed_optimizer:
+ log_single_rank(logger, logging.INFO, 'Using LayerWiseDistributedOptimizer for Muon')
+ if reset_config_bf16:
+ config.bf16 = True
+ return LayerWiseDistributedOptimizer(
+ optimizers, config, pg_collection, init_state_fn_list=init_fns
+ )
+ return ChainedOptimizer(optimizers)
diff --git a/megatron/core/optimizer/optimizer.py b/megatron/core/optimizer/optimizer.py
index 54e7f67c629..2c33d7e701d 100644
--- a/megatron/core/optimizer/optimizer.py
+++ b/megatron/core/optimizer/optimizer.py
@@ -153,7 +153,9 @@ def get_main_grads_for_grad_norm(self) -> List[torch.Tensor]:
grad = param.grad
grad_not_none = grad is not None
is_not_shared = param_is_not_shared(param)
- is_not_tp_duplicate = tensor_parallel.param_is_not_tensor_parallel_duplicate(param)
+ is_not_tp_duplicate = tensor_parallel.param_is_not_tensor_parallel_duplicate(
+ param, getattr(self, 'tp_group', None)
+ )
if grad_not_none and is_not_shared and is_not_tp_duplicate:
grads_for_norm.append(grad)
@@ -225,6 +227,7 @@ def count_zeros(self) -> float:
params,
grad_stats_parallel_group=self.get_grad_stats_parallel_group(),
use_decoupled_grad=self.config.use_precision_aware_optimizer_no_fp8_or_ds_fp8,
+ tp_group=getattr(self, 'tp_group', None),
)
@abstractmethod
diff --git a/megatron/core/optimizer/optimizer_config.py b/megatron/core/optimizer/optimizer_config.py
index d80d7e68096..a1429b7a170 100644
--- a/megatron/core/optimizer/optimizer_config.py
+++ b/megatron/core/optimizer/optimizer_config.py
@@ -1,5 +1,6 @@
-# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved.
+# Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+import fnmatch
from dataclasses import dataclass, field
from typing import Callable, Optional, Tuple, Union
@@ -8,6 +9,58 @@
from ..utils import is_te_min_version
+@dataclass(frozen=True)
+class ParamPredicate:
+ """Wraps a matching function to make it hashable for ParamKey.
+ Example:
+ >>> shape_1_param = ParamPredicate(name="s1", fn=lambda param: len(param.shape) == 1)
+ >>> shape_1_param(torch.empty(10))
+ True
+ >>> shape_1_param_copy = ParamPredicate(name="s1", fn=lambda param: len(param.shape) == 1)
+ >>> shape_1_param == shape_1_param_copy # name is used to match
+ True
+ >>> {shape_1_param, shape_1_param_copy} == {shape_1_param} # set hashing works properly
+
+ NOTE:
+ __hash__ and __eq__ are automatically generated by @dataclass(frozen=True)
+ based solely on 'name' because we set compare=False/hash=False on 'fn'.
+ """
+
+ name: str
+ fn: Callable[[torch.nn.Parameter], bool] = field(compare=False, hash=False)
+
+ def __call__(self, param: torch.nn.Parameter) -> bool:
+ return self.fn(param)
+
+
+@dataclass(frozen=True)
+class ParamWithNamePredicate:
+ """Wraps a matching function to make it hashable for ParamKey.
+ Example:
+ >>> shape_1_not_qkln_param = ParamWithNamePredicate(
+ name="s1_not_qkln",
+ fn=lambda param, name: (
+ len(param.shape) == 1 or name.endswith(".bias")
+ and not ("q_layernorm." in name or "k_layernorm." in name)
+ )
+ )
+ >>> shape_1_not_qkln_param(torch.empty(10), "interesting.bias")
+ True
+ >>> shape_1_not_qkln_param(torch.empty(10), "interesting.q_layernorm.bias")
+ False
+
+ NOTE:
+ __hash__ and __eq__ are automatically generated by @dataclass(frozen=True)
+ based solely on 'name' because we set compare=False/hash=False on 'fn'.
+ """
+
+ name: str
+ fn: Callable[[torch.nn.Parameter, str], bool] = field(compare=False, hash=False)
+
+ def __call__(self, param: torch.nn.Parameter, name: str) -> bool:
+ return self.fn(param, name)
+
+
@dataclass(frozen=True, slots=True)
class ParamKey:
"""Key to group parameters by. All such grouped parameters can share an
@@ -16,11 +69,71 @@ class ParamKey:
# TODO: Can add layer_id here later.
name: Union[str, Tuple[str]] = field(default_factory=tuple)
- """Parameter name(s)."""
+ """Parameter name(s), will use unix filesystem path syntax for matching."""
attr: Union[str, Tuple[str]] = field(default_factory=tuple)
"""Parameter attribute(s)."""
+ predicate: Union[ParamPredicate, Tuple[ParamPredicate]] = field(default_factory=tuple)
+ """Predicate(s) to match parameters by. If multiple predicates are provided, any must match."""
+
+ with_name_predicate: Union[ParamWithNamePredicate, Tuple[ParamWithNamePredicate]] = field(
+ default_factory=tuple
+ )
+ """
+ Predicate(s) to match parameters with their name. If multiple predicates are provided,
+ any must match. This is useful if you need to filter out some parameters from an otherwise
+ positive match by their name.
+ """
+
+ def matches(self, param: torch.nn.Parameter, param_name: str) -> bool:
+ """Returns true if passed-in parameter (with name) matches `param_key`.
+
+ Args:
+ param (torch.nn.Parameter): Handle to parameter object.
+ param_name (str): Name of parameter in underlying PyTorch module.
+
+ Returns:
+ bool: True if parameter matches passed-in param_key.
+ """
+
+ # Check if name matches.
+ if isinstance(self.name, str):
+ target_names = [self.name]
+ else:
+ target_names = list(self.name)
+ for target_name in target_names:
+ if fnmatch.fnmatch(param_name, target_name):
+ return True
+
+ # Check if attribute matches.
+ if isinstance(self.attr, str):
+ target_attrs = [self.attr]
+ else:
+ target_attrs = list(self.attr)
+ for target_attr in target_attrs:
+ if getattr(param, target_attr, False):
+ return True
+
+ # Check if predicate matches.
+ if isinstance(self.predicate, ParamPredicate):
+ if self.predicate(param):
+ return True
+ else:
+ for predicate in self.predicate:
+ if predicate(param):
+ return True
+
+ # Check if with_name_predicate matches.
+ if isinstance(self.with_name_predicate, ParamWithNamePredicate):
+ if self.with_name_predicate(param, param_name):
+ return True
+ else:
+ for predicate in self.with_name_predicate:
+ if predicate(param, param_name):
+ return True
+ return False
+
@dataclass
class OptimizerConfig:
@@ -29,7 +142,6 @@ class OptimizerConfig:
##############
# General
##############
-
lr: Optional[float] = None
"""Initial learning rate. Depending on decay style and initial warmup, the learning rate at each
iteration would be different.
@@ -38,9 +150,20 @@ class OptimizerConfig:
min_lr: Optional[float] = None
"""Minumum value for learning rate. The scheduler clip values below this threshold."""
+ decoupled_lr: Optional[float] = None
+ """Separate learning rate for the input and output layer."""
+
+ decoupled_min_lr: Optional[float] = None
+ """Minimum value for learning rate for the input and output layer. The scheduler clip values
+ below this threshold.
+ """
+
weight_decay: float = 0.01
"""Weight decay coefficient for L2 regularization."""
+ apply_wd_to_qk_layernorm: bool = False
+ """If true, apply weight decay to qk layernorm as a special case."""
+
##############
# Precision
##############
@@ -131,6 +254,31 @@ class OptimizerConfig:
sgd_momentum: float = 0.9
"""Momentum factor for SGD optimizer."""
+ # Muon
+ muon_momentum: float = 0.95
+ """The momentum used by the internal SGD."""
+
+ muon_split_qkv: bool = True
+ """Whether to split QKV parameters for Muon optimizer."""
+
+ muon_use_nesterov: bool = False
+ """Whether to use Nesterov-style momentum in the internal SGD."""
+
+ muon_scale_mode: str = "spectral"
+ """The mode to use for the scale factor. Defaults to "spectral"."""
+
+ muon_fp32_matmul_prec: str = "medium"
+ """The precision to use for the fp32 matmul. Defaults to "medium"."""
+
+ muon_num_ns_steps: int = 5
+ """The number of iteration steps to use in the Newton-Schulz iteration."""
+
+ muon_tp_mode: str = "blockwise"
+ """How to perform NS calculation for tensor parallel weights. Defaults to "blockwise"."""
+
+ muon_extra_scale_factor: float = 1.0
+ """Additional scale factor for the muon update."""
+
#######################
# Distributed optimizer
#######################
@@ -175,6 +323,12 @@ class OptimizerConfig:
pin_cpu_params: bool = True
"""If True, pin the optimizer parameters to CPU memory."""
+ offload_optimizer_states: bool = False
+ """
+ If True, offload optimizer states to CPU after each optimizer step and
+ reload them before the next optimizer step.
+ """
+
################
# Miscellaneous
################
diff --git a/megatron/core/optimizer/qk_clip.py b/megatron/core/optimizer/qk_clip.py
index 72127f94712..26b5787cd50 100644
--- a/megatron/core/optimizer/qk_clip.py
+++ b/megatron/core/optimizer/qk_clip.py
@@ -22,6 +22,11 @@ def clip_qk(model, log_max_only=False) -> float:
for model_chunk in model:
for transformer_layer in model_chunk.module.module.decoder.layers:
if hasattr(transformer_layer.self_attention, 'clip_qk'):
+ if (
+ transformer_layer.self_attention.core_attention.current_max_attn_logits
+ is None
+ ):
+ continue
torch.distributed.all_reduce(
transformer_layer.self_attention.core_attention.current_max_attn_logits,
op=torch.distributed.ReduceOp.MAX,
diff --git a/megatron/core/optimizer_param_scheduler.py b/megatron/core/optimizer_param_scheduler.py
index 9f771c612e8..7ff6fee35a7 100644
--- a/megatron/core/optimizer_param_scheduler.py
+++ b/megatron/core/optimizer_param_scheduler.py
@@ -3,14 +3,77 @@
"""Learning rate decay and weight decay incr functions."""
import logging
import math
-from typing import Optional
+from typing import TYPE_CHECKING, Any, Optional, TypedDict
-from megatron.core.optimizer import MegatronOptimizer
from megatron.core.utils import log_single_rank
+if TYPE_CHECKING:
+ # Avoid circular import.
+ from megatron.core.optimizer import MegatronOptimizer
+
logger = logging.getLogger(__name__)
+class ParamGroupOverride(TypedDict):
+ """Override values for a parameter group. These values may be optimizer-state/scheduler related.
+
+ These are the values you see later in param_group.get(...) calls in the
+ OptimizerParamScheduler.get_lr and get_wd methods. If you use a custom optimizer
+ or scheduler, you could override those variables instead.
+
+ Example:
+ >>> param_group_override = ParamGroupOverride(min_lr=1e-4, wd_mult=0.1)
+ >>> param_group_override == ParamGroupOverride(newvar=3) # this is ok too
+
+ """
+
+ max_lr: float
+ min_lr: float
+ start_wd: float
+ end_wd: float
+ wd_mult: float
+
+
+def param_group_override_to_tuple(
+ param_group_override: ParamGroupOverride | None,
+) -> tuple[tuple[str, Any], ...] | None:
+ """Convert a param group override to a tuple for use as a key in a dictionary.
+
+ The tuple is sorted by the keys of the param group override to handle different orderings of
+ the keys in different override dictionaries which still mean the same thing.
+ """
+ if param_group_override is None:
+ return None
+ return tuple(sorted(param_group_override.items()))
+
+
+def combine_param_group_overrides(
+ param_group_overrides: list[ParamGroupOverride | None],
+) -> ParamGroupOverride:
+ """Combine a list of param group overrides into a single param group override.
+
+ This function ensures that the overrides are not conflicting as well.
+
+ Args:
+ param_group_overrides (list[ParamGroupOverride]): list of param group overrides to combine
+
+ Returns:
+ ParamGroupOverride: combined param group override
+ """
+ combined_override = ParamGroupOverride()
+ for override in param_group_overrides:
+ if override is None:
+ continue
+ for key, value in override.items():
+ if key in combined_override:
+ if combined_override[key] != value:
+ raise ValueError(
+ f"Conflicting overrides for {key}: {combined_override[key]} and {value}"
+ )
+ combined_override[key] = value
+ return combined_override
+
+
class OptimizerParamScheduler:
"""Anneals learning rate and weight decay
@@ -38,7 +101,7 @@ class OptimizerParamScheduler:
def __init__(
self,
- optimizer: MegatronOptimizer,
+ optimizer: "MegatronOptimizer",
init_lr: float,
max_lr: float,
min_lr: float,
diff --git a/megatron/core/packed_seq_params.py b/megatron/core/packed_seq_params.py
index 330d0e03471..08ebdac67d8 100644
--- a/megatron/core/packed_seq_params.py
+++ b/megatron/core/packed_seq_params.py
@@ -1,6 +1,7 @@
# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved.
from dataclasses import dataclass
+import torch.distributed as dist
from torch import Tensor
@@ -18,3 +19,5 @@ class PackedSeqParams:
cu_seqlens_kv_padded: Tensor = None
max_seqlen_q: int = None
max_seqlen_kv: int = None
+ local_cp_size: int = None
+ cp_group: dist.ProcessGroup = None
diff --git a/megatron/core/parallel_state.py b/megatron/core/parallel_state.py
index aa90f2bc230..7aa867fd98f 100644
--- a/megatron/core/parallel_state.py
+++ b/megatron/core/parallel_state.py
@@ -6,6 +6,7 @@
import os
import warnings
from datetime import timedelta
+from math import log2
from typing import Callable, List, Optional
import numpy as np
@@ -114,6 +115,8 @@
_CONTEXT_PARALLEL_GLOBAL_RANKS = None
# Hierarchical context parallel groups
_HIERARCHICAL_CONTEXT_PARALLEL_GROUPS = None
+# Hybrid context parallel groups
+_HYBRID_DP_CP_GROUPS = {}
# Data parallel group information with context parallel combined.
_DATA_PARALLEL_GROUP_WITH_CP = None
@@ -417,6 +420,31 @@ def create_hierarchical_groups(
return hierarchical_groups, hierarchical_groups_gloo
+def create_hybrid_dp_cp_groups(rank, ranks, pg_options):
+ """
+ Creates groups required for hybrid DPxCP.
+ Creates a new group for every power of 2 up to the number of DPxCP ranks.
+ Returns a dictionary indexed by group size.
+ """
+ hybrid_dp_cp_groups = {}
+ # Generate group for every power of 2 up to the number of CP ranks
+ # We limit the allowed group sizes in order to avoid excessive overhead.
+ group_sizes = [2**i for i in range(int(log2(len(ranks))))][1:]
+ for group_size in group_sizes:
+ for i in range(0, len(ranks), group_size):
+ group = create_group(
+ ranks[i : i + group_size],
+ pg_options=pg_options,
+ group_desc=f"HYBRID_DP_CP_GROUP_{group_size}",
+ )
+ if rank in ranks[i : i + group_size]:
+ assert (
+ group_size not in hybrid_dp_cp_groups
+ ), f"Rank {rank} appears in multiple Hybrid DP CP groups of size {group_size}"
+ hybrid_dp_cp_groups[group_size] = group
+ return hybrid_dp_cp_groups
+
+
class RankGenerator(object):
"""A class for generating rank groups for different modes of parallelism."""
@@ -537,6 +565,7 @@ def initialize_model_parallel(
create_gloo_process_groups: bool = True,
high_priority_stream_groups: Optional[List[str]] = None,
sharp_enabled_group: Optional[str] = None,
+ hybrid_context_parallel: bool = False,
) -> None:
"""Initialize model data parallel groups.
@@ -888,6 +917,19 @@ def initialize_model_parallel(
if "NCCL_COLLNET_ENABLE" in os.environ:
del os.environ["NCCL_COLLNET_ENABLE"]
+ if hybrid_context_parallel:
+ global _HYBRID_DP_CP_GROUPS
+ for ranks_with_cp in decoder_rank_generator.get_ranks('dp-cp'):
+ assert (
+ len(ranks_with_cp) % 2 == 0
+ ), "Hybrid context parallel requires an even number of ranks"
+ _HYBRID_DP_CP_GROUPS.update(
+ create_hybrid_dp_cp_groups(
+ rank, ranks_with_cp, get_nccl_options("dp_cp", nccl_comm_cfgs)
+ )
+ )
+ # TODO: Are gloo groups needed for hybrid cp?
+
for ranks in decoder_rank_generator.get_ranks('dp'):
group = create_group(
ranks,
@@ -1404,6 +1446,18 @@ def get_hierarchical_context_parallel_groups(check_initialized=True):
return _HIERARCHICAL_CONTEXT_PARALLEL_GROUPS
+def get_hybrid_data_context_parallel_groups(check_initialized=True, group_size=None):
+ """Get the hybrid context parallel groups the caller rank belongs to."""
+ # If the group size is the same as the entire DPxCP group, return the original group
+ if get_data_parallel_world_size(with_context_parallel=True) == group_size:
+ if check_initialized:
+ assert _DATA_PARALLEL_GROUP_WITH_CP is not None
+ return _DATA_PARALLEL_GROUP_WITH_CP
+ if check_initialized:
+ assert _HYBRID_DP_CP_GROUPS is not None
+ return _HYBRID_DP_CP_GROUPS[group_size]
+
+
def get_embedding_group(check_initialized=True):
"""Get the embedding group the caller rank belongs to."""
if check_initialized:
diff --git a/megatron/core/pipeline_parallel/bridge_communicator.py b/megatron/core/pipeline_parallel/bridge_communicator.py
index a67ded6bf08..f1e74a2f16d 100644
--- a/megatron/core/pipeline_parallel/bridge_communicator.py
+++ b/megatron/core/pipeline_parallel/bridge_communicator.py
@@ -628,9 +628,6 @@ def send_forward_recv_backward(
dist.broadcast(
shape_tensor, src=self.current_rank, group=self.src_grid_broadcast_pg
)
- dist.broadcast(
- shape_tensor, src=self.current_rank, group=self.src_grid_broadcast_pg
- )
# Broadcast the tensors to all ranks in the group
dist.broadcast(
diff --git a/megatron/core/pipeline_parallel/fine_grained_activation_offload.py b/megatron/core/pipeline_parallel/fine_grained_activation_offload.py
new file mode 100644
index 00000000000..01c3a0c3aa0
--- /dev/null
+++ b/megatron/core/pipeline_parallel/fine_grained_activation_offload.py
@@ -0,0 +1,1259 @@
+# Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+
+from collections import deque
+from contextlib import nullcontext
+from typing import Any, Dict, Tuple
+
+import torch
+
+# CPU offload implementation for pipeline parallelism
+DEBUG = False
+DEBUG_RANK = 0
+
+from megatron.core.transformer.cuda_graphs import is_graph_capturing
+
+
+def debug_rank(message):
+ """Print debug message for a specific rank when DEBUG is enabled."""
+ # pylint: disable=bad-builtin
+ if not DEBUG:
+ return
+ assert torch.distributed.is_initialized()
+ if torch.distributed.get_rank() == DEBUG_RANK:
+ print(message)
+
+
+def print_offload_summary_table(total_offload_bytes: Dict[str, int]):
+ """
+ Print an ASCII table summarizing offload bytes across all ranks.
+
+ Gathers offload data from all ranks and prints a formatted table on rank 0,
+ with rows representing ranks and columns representing groups.
+
+ Args:
+ total_offload_bytes: Dict mapping group names to offload bytes for this rank.
+ """
+ # pylint: disable=bad-builtin
+ assert torch.distributed.is_initialized()
+ rank = torch.distributed.get_rank()
+ world_size = torch.distributed.get_world_size()
+
+ # Gather all group names across ranks
+ local_names = list(total_offload_bytes.keys())
+ all_names_list = [None] * world_size
+ torch.distributed.all_gather_object(all_names_list, local_names)
+ all_group_names = sorted(set(name for names in all_names_list for name in names))
+
+ # Gather offload bytes from all ranks: each rank sends a list of bytes per group
+ local_bytes = [total_offload_bytes.get(name, 0) for name in all_group_names]
+ all_bytes_list = [None] * world_size
+ torch.distributed.all_gather_object(all_bytes_list, local_bytes)
+
+ # Print ASCII table on rank 0
+ if rank == 0:
+ # Calculate column widths
+ col_width = max(12, max((len(name) for name in all_group_names), default=8) + 2)
+ rank_col_width = max(6, len(f"Rank {world_size - 1}") + 2)
+
+ # Build header
+ header = "Rank".ljust(rank_col_width)
+ header += "".join(name.rjust(col_width) for name in all_group_names)
+ header += "Total".rjust(col_width)
+ separator = "-" * len(header)
+
+ print("\n" + "=" * len(header))
+ print("Activation Offload Summary (MB)".center(len(header)))
+ print("=" * len(header))
+ print(header)
+ print(separator)
+
+ # Build rows for each rank
+ grand_total = 0
+ col_totals = [0] * len(all_group_names)
+ for r in range(world_size):
+ row_bytes = all_bytes_list[r]
+ row_total = sum(row_bytes)
+ grand_total += row_total
+ for i, b in enumerate(row_bytes):
+ col_totals[i] += b
+ row_str = f"Rank {r}".ljust(rank_col_width)
+ for b in row_bytes:
+ row_str += f"{b / (1024 * 1024):.2f}".rjust(col_width)
+ row_str += f"{row_total / (1024 * 1024):.2f}".rjust(col_width)
+ print(row_str)
+
+ # Print totals row
+ print(separator)
+ totals_row = "Total".ljust(rank_col_width)
+ for ct in col_totals:
+ totals_row += f"{ct / (1024 * 1024):.2f}".rjust(col_width)
+ totals_row += f"{grand_total / (1024 * 1024):.2f}".rjust(col_width)
+ print(totals_row)
+ print("=" * len(header) + "\n")
+
+ torch.distributed.barrier()
+
+
+class GPUTensorPool:
+ """
+ GPU memory pool for efficient allocation and deallocation of tensors.
+
+ Features:
+ - Supports multiple tensor shapes and dtypes, each with its own pool
+ - Dynamic allocation: tensors are created on-demand during allocation
+ - Efficient reuse: freed tensors are returned to the pool for reuse
+ - Uses queue-based management for O(1) allocation and deallocation
+
+ Example:
+ pool = GPUTensorPool(device='cuda:0')
+ tensor = pool.allocate((128, 512), dtype=torch.float32)
+ # ... use tensor ...
+ pool.free(tensor, (128, 512), dtype=torch.float32)
+ """
+
+ def __init__(self, device: str = 'cuda', pin_memory: bool = False):
+ """
+ Initialize GPU tensor pool.
+
+ Args:
+ device: GPU device, default 'cuda'
+ pin_memory: Whether to use pinned memory (mainly for CPU tensors)
+ """
+ self.device = torch.device(device)
+ self.pin_memory = pin_memory
+
+ # Maintain a separate pool for each (shape, dtype) combination
+ # Structure: {(shape, dtype): {'free': deque, 'all': list, 'allocated_count': int}}
+ self._pools: Dict[Tuple, Dict[str, Any]] = {}
+
+ # Statistics
+ self._stats = {
+ 'total_allocated': 0, # Total number of tensors ever allocated
+ 'current_in_use': 0, # Number of tensors currently in use
+ 'allocation_requests': 0, # Number of allocation requests
+ 'free_requests': 0, # Number of free requests
+ 'pool_hits': 0, # Number of times a tensor was reused from pool
+ 'pool_misses': 0, # Number of times a new tensor was created
+ }
+
+ debug_rank("GPUTensorPool: Initialized with dynamic allocation")
+
+ def _get_pool_key(self, shape: Tuple, dtype: torch.dtype) -> Tuple:
+ """Generate a unique key for the pool based on shape and dtype."""
+ return (shape, dtype)
+
+ @staticmethod
+ def _calculate_memory_size(shape: Tuple, dtype: torch.dtype) -> int:
+ """Calculate memory size in bytes."""
+ element_size = torch.tensor([], dtype=dtype).element_size()
+ numel = 1
+ for dim in shape:
+ numel *= dim
+ return numel * element_size
+
+ def allocate(self, shape: Tuple, dtype: torch.dtype = torch.float32) -> torch.Tensor:
+ """
+ Allocate a tensor with the specified shape and dtype.
+
+ Args:
+ shape: Shape of the tensor
+ dtype: Data type of the tensor, default torch.float32
+
+ Returns:
+ Allocated tensor
+ """
+ self._stats['allocation_requests'] += 1
+
+ pool_key = self._get_pool_key(shape, dtype)
+
+ # Create pool for this (shape, dtype) if it doesn't exist
+ if pool_key not in self._pools:
+ self._pools[pool_key] = {
+ 'free': deque(), # Queue of available tensors
+ 'all': [], # List of all tensors (for tracking)
+ 'allocated_count': 0, # Number of allocated tensors
+ }
+
+ pool = self._pools[pool_key]
+
+ # Try to reuse a tensor from the pool
+ if len(pool['free']) > 0:
+ tensor = pool['free'].popleft()
+ self._stats['pool_hits'] += 1
+ debug_rank(
+ f"GPUTensorPool.allocate: Reused tensor from pool, "
+ f"shape={shape}, dtype={dtype}, "
+ f"remaining in pool={len(pool['free'])}"
+ )
+ else:
+ # Allocate a new tensor
+ tensor = torch.empty(shape, dtype=dtype, device=self.device, pin_memory=self.pin_memory)
+ pool['all'].append(tensor)
+ self._stats['total_allocated'] += 1
+ self._stats['pool_misses'] += 1
+
+ memory_mb = self._calculate_memory_size(shape, dtype) / (1024**2)
+ debug_rank(
+ f"GPUTensorPool.allocate: Created new tensor, "
+ f"shape={shape}, dtype={dtype}, "
+ f"memory={memory_mb:.2f} MB, "
+ f"total_created={len(pool['all'])}"
+ )
+
+ pool['allocated_count'] += 1
+ self._stats['current_in_use'] += 1
+
+ return tensor
+
+ def free(self, tensor: torch.Tensor):
+ """
+ Return a tensor to the pool for reuse.
+
+ Args:
+ tensor: Tensor to free
+
+ Raises:
+ ValueError: If tensor doesn't belong to this pool
+ """
+ self._stats['free_requests'] += 1
+
+ shape = tensor.shape
+ dtype = tensor.dtype
+
+ pool_key = self._get_pool_key(shape, dtype)
+
+ if pool_key not in self._pools:
+ raise ValueError(
+ f"No pool exists for shape={shape}, dtype={dtype}. "
+ f"Available pools: {list(self._pools.keys())}"
+ )
+
+ pool = self._pools[pool_key]
+
+ # Verify tensor belongs to this pool (use identity check, not value comparison)
+ tensor_found = any(tensor is t for t in pool['all'])
+ if not tensor_found:
+ raise ValueError(
+ f"Attempting to free a tensor that doesn't belong to this pool "
+ f"(shape={shape}, dtype={dtype})"
+ )
+
+ # Return tensor to the free queue
+ pool['free'].append(tensor)
+ pool['allocated_count'] -= 1
+ self._stats['current_in_use'] -= 1
+
+ debug_rank(
+ f"GPUTensorPool.free: shape={shape}, dtype={dtype}, "
+ f"available in pool={len(pool['free'])}"
+ )
+
+ def get_pool_status(self, shape: Tuple = None, dtype: torch.dtype = None) -> Dict[str, Any]:
+ """
+ Get the status of the memory pool.
+
+ Args:
+ shape: If specified along with dtype, return status for that specific pool
+ dtype: Data type (required if shape is specified)
+
+ Returns:
+ Dictionary containing status information
+ """
+ if shape is not None:
+ if dtype is None:
+ raise ValueError("dtype must be specified when shape is provided")
+
+ pool_key = self._get_pool_key(shape, dtype)
+
+ if pool_key not in self._pools:
+ raise ValueError(f"No pool exists for shape={shape}, dtype={dtype}")
+
+ pool = self._pools[pool_key]
+ total_count = len(pool['all'])
+
+ return {
+ 'shape': shape,
+ 'dtype': dtype,
+ 'total_count': total_count,
+ 'allocated_count': pool['allocated_count'],
+ 'free_count': len(pool['free']),
+ 'utilization': (
+ pool['allocated_count'] / total_count * 100 if total_count > 0 else 0
+ ),
+ }
+ else:
+ # Return status for all pools
+ status = {'global_stats': self._stats.copy(), 'pools': {}}
+
+ for pool_key in self._pools:
+ shape, dtype = pool_key
+ status['pools'][pool_key] = self.get_pool_status(shape, dtype)
+
+ return status
+
+ def reset(self):
+ """Reset the pool, marking all tensors as available."""
+ debug_rank("GPUTensorPool: Resetting pool...")
+
+ for pool_key, pool in self._pools.items():
+ # Clear and refill the free queue
+ pool['free'].clear()
+ for tensor in pool['all']:
+ pool['free'].append(tensor)
+ pool['allocated_count'] = 0
+
+ self._stats['current_in_use'] = 0
+ debug_rank("GPUTensorPool: Reset complete")
+
+ def clear(self):
+ """Clear the pool and release all GPU memory."""
+ debug_rank("GPUTensorPool: Clearing pool...")
+
+ for pool_key, pool in self._pools.items():
+ # Clear all references, allowing PyTorch GC to reclaim memory
+ pool['free'].clear()
+ pool['all'].clear()
+
+ self._pools.clear()
+ self._stats['current_in_use'] = 0
+
+ # Trigger GPU cache cleanup
+ if torch.cuda.is_available():
+ torch.cuda.empty_cache()
+
+ debug_rank("GPUTensorPool: Clear complete")
+
+ def __del__(self):
+ """Destructor to ensure resources are released."""
+ self.clear()
+
+
+class OffloadTensorGroup:
+ """
+ A group of tensors to be offloaded together.
+ """
+
+ def __init__(self, name):
+ self._name = name
+ self._tensors = {}
+ self._offload_event = torch.cuda.Event()
+ self._reload_event = torch.cuda.Event()
+ self.offload = True
+ self.total_offload_bytes = 0
+ self.total_tensor_count = 0
+ # Using memory pool is for the compatibility with cuda graph.
+ # Shapes of tensors for expert_fc1 and moe_act are not known in advance,
+ # so we do not use CPU pool for them.
+ if name == "expert_fc1" or name == "moe_act":
+ self.use_cpu_pool = False
+ else:
+ self.use_cpu_pool = True
+
+ def push_tensor(self, tag, tensor):
+ """Push a tensor to the group."""
+ self._tensors[tag] = tensor
+
+ def pop_tensor(self, tag):
+ """Pop a tensor from the group."""
+ return self._tensors.pop(tag)
+
+ def record_offload_event(self, stream):
+ """Record the offload event."""
+ self._offload_event.record(stream)
+
+ def wait_offload_event(self, stream):
+ """Wait for the offload event."""
+ stream.wait_event(self._offload_event)
+
+ def record_reload_event(self, stream):
+ """Record the reload event."""
+ self._reload_event.record(stream)
+
+ def wait_reload_event(self, stream):
+ """Wait for the reload event."""
+ stream.wait_event(self._reload_event)
+
+ def update_offload_info(self, tensor):
+ """Update the offload information."""
+ self.total_offload_bytes += tensor.numel() * tensor.element_size()
+ self.total_tensor_count += 1
+
+
+class PipelineOffloadManager:
+ """
+ Singleton manager for coordinating activation offloading across pipeline stages.
+ Manages chunk handlers, synchronizes GPU-CPU transfers,
+ and handles virtual pipeline parallelism.
+ """
+
+ OFFLOAD_MGR = None
+
+ @classmethod
+ def get_instance(cls):
+ """Get the singleton instance of PipelineOffloadManager."""
+ if cls.OFFLOAD_MGR is None:
+ cls.OFFLOAD_MGR = PipelineOffloadManager()
+ return cls.OFFLOAD_MGR
+
+ @classmethod
+ def reset_instance(cls):
+ """Reset the singleton instance of PipelineOffloadManager."""
+ cls.OFFLOAD_MGR = None
+ cls.OFFLOAD_MGR = PipelineOffloadManager()
+
+ def __init__(self):
+ """Initialize the manager with queues and dedicated CUDA streams."""
+ # Queue to store chunk handlers for backward pass
+ self._queue = deque()
+ # Cache chunk handlers for each virtual pipeline stage
+ self._stages = None
+ # allocate streams and events for synchronization
+ self._d2h_stream = torch.cuda.Stream()
+ self._h2d_stream = torch.cuda.Stream()
+ # Shared CPU tensor pool for all chunks to improve reuse efficiency
+ self._cpu_tensor_pool = GPUTensorPool(device="cpu", pin_memory=True)
+
+ # Whether the manager is in warmup phase.
+ self._is_warmup = True
+ # Cache OffloadChunkHandler objects for each virtual pipeline stage and each forward pass.
+ self._cached_chunks_forward = []
+ # Cache OffloadChunkHandler objects for each virtual pipeline stage and each backward pass.
+ self._cached_chunks_backward = []
+ # Index of the current backward chunk in the cached chunks backward.
+ self._cached_chunks_index_backward = 0
+ # Index of the current forward chunk in the cached chunks forward.
+ self._cached_chunks_index_forward = 0
+
+ self.do_offload = True
+
+ # Do not offload the last X groups so that the reloading won't block the computing stream.
+ self._offload_margin = 0
+ # Sometimes we need to delay the offloading and launch it later.
+ # The delayed offload groups are stored in a queue.
+ self._delayed_offload_groups = []
+ self.reset()
+
+ @property
+ def d2h_stream(self):
+ """Get the device-to-host (GPU to CPU) transfer stream."""
+ return self._d2h_stream
+
+ @property
+ def h2d_stream(self):
+ """Get the host-to-device (CPU to GPU) transfer stream."""
+ return self._h2d_stream
+
+ @property
+ def cpu_tensor_pool(self):
+ """Get the shared CPU tensor pool."""
+ return self._cpu_tensor_pool
+
+ def push_offload_groups(self, group_hook, forced_released_tensors):
+ """Push the offload groups to the delayed queue."""
+ debug_rank(f"pushing offload groups to the delayed queue")
+ self._delayed_offload_groups.append((group_hook, forced_released_tensors))
+
+ def flush_delayed_groups(self):
+ """Flush the delayed groups."""
+ debug_rank("flushing delayed groups")
+ # Flush the delayed groups in reverse order to maintain the order of the groups.
+ for group_hook, forced_released_tensors in reversed(self._delayed_offload_groups):
+ group_hook(forced_released_tensors)
+ self._delayed_offload_groups = []
+
+ def reset(self):
+ """Reset manager state for a new training iteration."""
+ self._inside_context = False
+ self._cur_forward_chunk = None
+ self._cur_backward_chunk = None
+ # Reset CPU tensor pool to reuse all CPU tensors for next iteration
+ if hasattr(self, '_cpu_tensor_pool'):
+ self._cpu_tensor_pool.reset()
+
+ # Call post_warmup_callback after warmup to collect the offload information.
+ if self._is_warmup and len(self._cached_chunks_forward) > 0:
+ self.post_warmup_callback()
+ self._cached_chunks_index_backward = 0
+ self._cached_chunks_index_forward = 0
+
+ for chunk in self._cached_chunks_forward:
+ chunk.reset()
+ self._delayed_offload_groups = []
+
+ @property
+ def offload_summary_bytes(self) -> Dict[str, int]:
+ """Offload summary bytes per group collected after warmup."""
+ return self._offload_summary_bytes
+
+ @property
+ def offload_summary_total_bytes(self) -> int:
+ """Total offloaded bytes collected after warmup."""
+ return self._offload_summary_total_bytes
+
+ def flush(self):
+ """Flush all staged chunks to the backward queue in reverse order."""
+ # Ensure all virtual pipeline stages have the same number of chunks
+ if len(self._stages[0]) == len(self._stages[-1]):
+ lens = [len(e) for e in self._stages]
+ assert min(lens) == max(lens), "All stages must have same chunk count"
+ # Clear the last stage and push all chunks in reverse order for backward
+ self._stages[-1] = []
+ for chunks in reversed(self._stages):
+ for chunk in chunks:
+ self.push(chunk)
+ # Clear all stages after flushing
+ for i in range(self._vpp):
+ self._stages[i] = []
+
+ def disable_offload(self):
+ """Disable the offload."""
+ debug_rank("disable_offload")
+ self.do_offload = False
+ for chunk in self._cached_chunks_forward:
+ chunk.do_offload = False
+
+ def enable_offload(self):
+ """Enable the offload."""
+ debug_rank("enable_offload")
+ self.do_offload = True
+ for chunk in self._cached_chunks_forward:
+ chunk.do_offload = True
+
+ def post_warmup_callback(self):
+ """Callback after warmup."""
+ # pylint: disable=bad-builtin
+ debug_rank("post_warmup_callback")
+ self._is_warmup = False
+ assert len(self._cached_chunks_forward) == len(
+ self._cached_chunks_backward
+ ), "Cached chunks forward and backward must have the same length"
+ for chunk in self._cached_chunks_forward:
+ chunk.is_warmup = False
+ assert (
+ chunk in self._cached_chunks_backward
+ ), "Chunk not found in cached chunks backward"
+ # Update the offload margin to the maximum number of deduplicated groups
+ self._offload_margin = max(self._offload_margin, chunk.get_max_deduplicated_groups())
+ debug_rank(f"offload margin {self._offload_margin}")
+ # Find the last group with the same name in the cached chunks backward
+ last_group_with_same_name = {}
+ for chunk_idx, chunk in enumerate(reversed(self._cached_chunks_backward)):
+ for group in chunk.offload_groups:
+ last_group_with_same_name[group._name] = group
+ # Mark the last group with the same name as not offloadable to make sure
+ # the reloading won't block the main stream.
+ for name, group in last_group_with_same_name.items():
+ if self._offload_margin > 0:
+ group.offload = False
+ self._offload_margin -= 1
+ debug_rank(f"setting offload to false for group {name} at chunk index {chunk_idx}")
+ else:
+ break
+ debug_rank(f"offload margin {self._offload_margin}")
+ assert self._offload_margin == 0, "Offload margin is not 0"
+ # Dump the offload information
+ total_tensor_count = {}
+ total_offload_bytes = {}
+ for chunk in self._cached_chunks_forward:
+ for group in chunk.offload_groups:
+ if group.offload:
+ if group._name not in total_tensor_count:
+ total_tensor_count[group._name] = 0
+ total_tensor_count[group._name] += group.total_tensor_count
+ if group._name not in total_offload_bytes:
+ total_offload_bytes[group._name] = 0
+ total_offload_bytes[group._name] += group.total_offload_bytes
+ # Stop statistics at the first backward chunk after which 1F1B is running,
+ # where the memory cost will not increase anymore.
+ if chunk is self._cached_chunks_backward[0]:
+ break
+ # Cache summary for downstream consumers (e.g., unit tests).
+ self._offload_summary_bytes = dict(total_offload_bytes)
+ self._offload_summary_total_bytes = int(sum(total_offload_bytes.values()))
+ print_offload_summary_table(total_offload_bytes)
+
+ def push(self, handler):
+ """Add a chunk handler to the backward queue."""
+ debug_rank(f"pushing handler {handler}")
+ self._queue.append(handler)
+ if self._is_warmup:
+ self._cached_chunks_backward.append(handler)
+
+ def pop_backward_chunk(self, name=None):
+ """Get the next non-empty backward chunk containing the group with the given name."""
+ self._cur_backward_chunk = None
+ debug_rank(f"popping backward chunk {self._cached_chunks_index_backward}")
+ debug_rank(f"cached chunks backward {self._cached_chunks_backward}")
+ for idx, handler in enumerate(
+ self._cached_chunks_backward[self._cached_chunks_index_backward :]
+ ):
+ self._cached_chunks_index_backward += 1
+ if not handler.is_empty_chunk(name):
+ self._cur_backward_chunk = (
+ handler # set the first non-empty chunk as the current backward chunk
+ )
+ debug_rank(f"handler {handler} at index {idx} is not empty")
+ break
+ assert self._cur_backward_chunk is not None, "No non-empty chunk found"
+
+ def front_backward_chunk(self, name=None):
+ """Get the first non-empty backward chunk containing the group with the given name."""
+ for idx, handler in enumerate(
+ self._cached_chunks_backward[self._cached_chunks_index_backward :]
+ ):
+ if not handler.is_empty_chunk(name):
+ debug_rank(f"front handler {handler} at index {idx}")
+ return handler
+ return None
+
+ def init_model_chunk_offload_handler(
+ self, vp_size, vp_stage, min_offloaded_tensor_size=1024 * 1024
+ ):
+ """
+ Initialize a chunk offload handler for a model chunk (microbatch).
+
+ Args:
+ vp_size: Virtual pipeline size
+ vp_stage: Virtual pipeline stage index (None means stage 0)
+ min_offloaded_tensor_size: Minimum tensor size (in elements) to offload
+ """
+ if not self._is_warmup:
+ return
+
+ vp_size = 1 if vp_size is None else vp_size
+ if self._stages is None:
+ self._vpp = vp_size
+ self._stages = [[] for _ in range(vp_size)]
+
+ if vp_stage is None:
+ cur_vpp_rank = 0
+ else:
+ cur_vpp_rank = vp_stage
+
+ # Flush staged chunks when reaching the last virtual pipeline stage
+ if cur_vpp_rank == self._vpp - 1:
+ self.flush()
+
+ # Use shared CPU tensor pool for better reuse across chunks
+ cur_chunk = ChunkOffloadHandler(min_offloaded_tensor_size, self._cpu_tensor_pool)
+ debug_rank(f"init_model_chunk_offload_handler {cur_chunk}")
+ self._stages[cur_vpp_rank].append(cur_chunk)
+ # For the last stage, push immediately and flush
+ if cur_vpp_rank == self._vpp - 1:
+ self.push(cur_chunk)
+ self.flush()
+ self._cur_forward_chunk = cur_chunk
+ cur_chunk.vpp_rank = cur_vpp_rank
+ self._cached_chunks_forward.append(cur_chunk)
+
+ def pop_forward_chunk(self, name=None):
+ """Get the next forward pass chunk handler."""
+ debug_rank(f"pop_forward_chunk {self._cur_forward_chunk}")
+ if not self.do_offload:
+ return self._cur_forward_chunk
+ while not self._is_warmup and (
+ self._cur_forward_chunk is None or self._cur_forward_chunk.finish_all_groups(name)
+ ):
+ if self._cached_chunks_index_forward >= len(self._cached_chunks_forward):
+ self._cur_forward_chunk = None
+ break
+ self._cur_forward_chunk = self._cached_chunks_forward[self._cached_chunks_index_forward]
+ self._cached_chunks_index_forward += 1
+ debug_rank(f"new cur_forward_chunk {self._cur_forward_chunk}")
+ return self._cur_forward_chunk
+
+ def cur_forward_chunk(self):
+ """Get the current forward pass chunk handler."""
+ return self._cur_forward_chunk
+
+ def cur_backward_chunk(self):
+ """Get the current backward pass chunk handler."""
+ return self._cur_backward_chunk
+
+ def mark_not_offloadable(self, tensor: torch.Tensor):
+ """Mark the current forward chunk as not offloadable."""
+ if tensor is not None:
+ tensor.offloading_activation = False
+
+ def __enter__(self):
+ """Enter context manager to enable activation offloading hooks."""
+ debug_rank("----__enter__")
+ if self._cur_forward_chunk is None or not self.cur_forward_chunk().do_offload:
+ return
+ from megatron.core.extensions.transformer_engine import cpu_offload
+
+ if cpu_offload is not None:
+ cpu_offload.CPUOffloadEnabled = True
+ else:
+ raise RuntimeError("TE CPU offload is not available")
+ self.inside_context = True
+
+ torch._C._autograd._push_saved_tensors_default_hooks(
+ self.on_save_for_backward, self.on_get_saved_tensor
+ )
+
+ def __exit__(self, *args: Any):
+ """Exit context manager and restore original tensor saving behavior."""
+ debug_rank("----__exit__")
+ if self._cur_forward_chunk is None or not self.cur_forward_chunk().do_offload:
+ return
+ from megatron.core.extensions.transformer_engine import cpu_offload
+
+ if cpu_offload is not None:
+ cpu_offload.CPUOffloadEnabled = False
+ else:
+ raise RuntimeError("TE CPU offload is not available")
+ self.inside_context = False
+ torch._C._autograd._pop_saved_tensors_default_hooks()
+
+ def on_save_for_backward(self, tensor: torch.Tensor) -> Any:
+ """
+ Hook called when autograd saves a tensor for backward pass.
+ Returns a tag to identify the tensor later.
+ """
+ debug_rank(f"------on_save_for_backward {tensor.shape}")
+ assert self.inside_context, "Must be inside offload context"
+ return self.cur_forward_chunk().tensor_push(tensor)
+
+ def on_get_saved_tensor(self, saved_state: Any) -> torch.Tensor:
+ """
+ Hook called when autograd retrieves a saved tensor during backward pass.
+ Returns the actual tensor (potentially reloading from CPU).
+ """
+ debug_rank(f"----on_get_saved_tensor {saved_state}")
+ return self.cur_backward_chunk().tensor_pop(saved_state)
+
+
+class ChunkOffloadHandler:
+ """
+ Handles activation offloading and reloading for a single pipeline chunk (microbatch).
+ Manages tensor groups, coordinates asynchronous GPU-CPU transfers, and handles synchronization.
+ """
+
+ def offload(self, src_tensor, pin_memory=True, use_cpu_pool=True):
+ """Offload."""
+ debug_rank("--------offload")
+
+ if not src_tensor.is_contiguous():
+ src_tensor = src_tensor.contiguous()
+
+ if use_cpu_pool:
+ cpu_backup = self.cpu_tensor_pool.allocate(src_tensor.shape, dtype=src_tensor.dtype)
+ else:
+ cpu_backup = torch.empty(
+ src_tensor.shape, dtype=src_tensor.dtype, device="cpu", pin_memory=pin_memory
+ )
+
+ cpu_backup.copy_(src_tensor, non_blocking=pin_memory)
+ state = (src_tensor.device, cpu_backup, use_cpu_pool)
+ return state
+
+ def reload(self, state, non_blocking=None):
+ """Reload."""
+ debug_rank("------reload")
+ dev, cpu_backup, use_cpu_pool = state
+ if non_blocking is None:
+ non_blocking = cpu_backup.is_pinned()
+ gpu_tensor = torch.empty(
+ cpu_backup.size(), dtype=cpu_backup.dtype, layout=cpu_backup.layout, device=dev
+ )
+ gpu_tensor.copy_(cpu_backup, non_blocking=non_blocking)
+ if use_cpu_pool:
+ self.cpu_tensor_pool.free(cpu_backup)
+ return gpu_tensor
+
+ def __init__(self, min_offloaded_tensor_size, cpu_tensor_pool):
+ self.do_offload = True
+
+ # Group management for batching offload/reload operations
+ self.offload_groups = []
+ self._offloaded_group_index = 0
+ # Groups to be offloaded.
+ self._groups_to_offload = []
+ # Groups to be reloaded.
+ self._groups_to_reload = []
+ # Tensor count for the current group.
+ self._tensor_count_current_group = 0
+ # Maximum number of groups to offload or reload.
+ self._max_group_size = 0
+ # Groups being reloaded.
+ self._reloading_group = []
+ # Counter for special torch tensor types (FakeTensor, FunctionalTensor)
+ self.torch_tensor_count = 0
+ self.d2h_stream = PipelineOffloadManager.get_instance().d2h_stream
+ self.h2d_stream = PipelineOffloadManager.get_instance().h2d_stream
+ self.min_offloaded_tensor_size = min_offloaded_tensor_size
+ self.cpu_tensor_pool = cpu_tensor_pool
+ self.is_warmup = True
+
+ def reset(self):
+ """Reset the chunk offload handler."""
+ self._offloaded_group_index = 0
+ self._groups_to_offload = []
+ self._groups_to_reload = []
+ self._tensor_count_current_group = 0
+ self._reloading_group = []
+
+ def find_group_with_name(self, name: str, start_index: int = 0):
+ """Find the group with the given name starting from the given index."""
+ return next(
+ (group for group in self.offload_groups[start_index:] if group._name == name), None
+ )
+
+ def is_empty_chunk(self, name=None):
+ """Check if this chunk has no tensors to manage."""
+ debug_rank(f"------is_empty_chunk {self._max_group_size}")
+ if name is not None:
+ return self.find_group_with_name(name) is None
+ return self._max_group_size == 0
+
+ def finish_all_groups(self, name=None) -> bool:
+ """Finish all groups."""
+ debug_rank(
+ f"------finish_all_groups {self} {self._max_group_size} {self._offloaded_group_index}"
+ )
+ # TODO: check if this is correct
+ # Mark it as finished when there are no groups to offload or reload
+ if (
+ len(self._groups_to_reload) == 0
+ and len(self._groups_to_offload) == 0
+ and self._offloaded_group_index > 0
+ ):
+ return True
+ assert name is not None, "Name is required"
+ return self.find_group_with_name(name, self._offloaded_group_index) is None
+
+ def find_next_group(self, name=None):
+ """Find the next group with the given name."""
+ assert name is not None, "Name is required"
+ return self.find_group_with_name(name, self._offloaded_group_index)
+
+ def tensor_push(self, tensor):
+ """Push tensor to the offload handler."""
+ torch_stray_tensor = isinstance(
+ tensor,
+ (
+ torch._subclasses.fake_tensor.FakeTensor,
+ torch._subclasses.functional_tensor.FunctionalTensor,
+ ),
+ )
+ assert not torch_stray_tensor, "Stray tensor should not be offloaded"
+
+ # Assign unique tag based on group index and position within group
+ tensor_tag = (self._offloaded_group_index, self._tensor_count_current_group)
+ self._tensor_count_current_group += 1
+ self.offload_groups[self._offloaded_group_index - 1].push_tensor(tensor_tag, tensor)
+ debug_rank(f"--------tensor_push {tensor_tag}")
+ return tensor_tag
+
+ def tensor_pop(self, tensor_tag):
+ """Pop tensor from the offload handler."""
+ debug_rank(f"--------tensor_pop {tensor_tag}")
+ group_id, idx = tensor_tag
+ tensor = self.offload_groups[group_id - 1].pop_tensor(tensor_tag)
+ # If tensor is offloaded (stored as tuple), reload it
+ if isinstance(tensor, tuple):
+ tensor = self.reload(tensor)
+ debug_rank(f"--------tensor_pop {tensor.shape}")
+ return tensor
+
+ def tensor_need_offloading_checker(self, tensor):
+ """Check if the tensor needs to be offloaded."""
+ debug_rank(
+ f"tensor_need_offloading_checker {getattr(tensor, 'offloading_activation', None)}"
+ )
+ if tensor.numel() < self.min_offloaded_tensor_size:
+ return False
+ # Respect tensor's offload preference if specified
+ if hasattr(tensor, "offloading_activation") and not tensor.offloading_activation:
+ return False
+ return True
+
+ def bulk_offload_group(self):
+ """offload a group of tensors recorded in tensor_push()."""
+ debug_rank("------bulk_offload_group")
+ group_to_offload = self._groups_to_offload[-1]
+ torch.cuda.nvtx.range_push("activation offloading " + group_to_offload._name)
+ with torch.cuda.stream(self.d2h_stream):
+ for tensor_tag, tensor_on_device in group_to_offload._tensors.items():
+ if self.tensor_need_offloading_checker(tensor_on_device):
+ state = self.offload(
+ tensor_on_device, use_cpu_pool=group_to_offload.use_cpu_pool
+ )
+ if self.is_warmup:
+ group_to_offload.update_offload_info(tensor_on_device)
+ tensor_on_device.record_stream(self.d2h_stream)
+ group_to_offload.push_tensor(tensor_tag, state)
+ group_to_offload.record_offload_event(self.d2h_stream)
+ self._groups_to_offload.pop()
+ torch.cuda.nvtx.range_pop()
+
+ def get_max_deduplicated_groups(self):
+ """Get the maximum number of deduplicated groups."""
+ count_modules = []
+ for group in self.offload_groups:
+ if group._name not in count_modules:
+ count_modules.append(group._name)
+ return len(count_modules)
+
+ def bulk_reload_group(self):
+ """Bulk reload group."""
+ debug_rank("----bulk_reload_group")
+ group_to_reload = self._groups_to_reload[-1]
+ torch.cuda.nvtx.range_push("activation reloading " + group_to_reload._name)
+ with torch.cuda.stream(self.h2d_stream):
+ # Wait for offload to complete before reloading
+ if not is_graph_capturing():
+ group_to_reload.wait_offload_event(self.h2d_stream)
+ for tensor_tag, state in group_to_reload._tensors.items():
+ # Only reload if tensor was offloaded (stored as tuple)
+ if isinstance(state, tuple):
+ recovered_tensor = self.reload(state)
+ debug_rank(f"----recovered_tensor {recovered_tensor.shape}")
+ group_to_reload.push_tensor(tensor_tag, recovered_tensor)
+ group_to_reload.record_reload_event(self.h2d_stream)
+ self._groups_to_reload.pop()
+ # Add the group to the reloading group to wait for the reload event.
+ self._reloading_group.append(group_to_reload)
+ torch.cuda.nvtx.range_pop()
+
+ def pre_reload_last_layer(self):
+ """Pre-reload the last layer of this chunk to hide reload latency."""
+ debug_rank("pre_reload_last_layer")
+ debug_rank(f"len(self._groups_to_reload) {len(self._groups_to_reload)}")
+ if len(self._groups_to_reload) > 0:
+ # Reload the last group (last layer) early
+ self.bulk_reload_group()
+
+ def should_bulk_offload(self):
+ """Determine if the current group should be offloaded."""
+ assert len(self._groups_to_offload) > 0, "No groups to offload"
+ group = self._groups_to_offload[-1]
+ debug_rank(f"should_bulk_offload {self.is_warmup} {group.offload}")
+ # Don't offload if the chunk is not in warmup stage
+ if self.is_warmup:
+ return True
+ # Don't offload if the group is marked as not offloadable
+ if not group.offload:
+ return False
+
+ # Check if next backward chunk is this chunk (for last pipeline stage)
+ next_backward_chunk = PipelineOffloadManager.get_instance().front_backward_chunk(
+ group._name
+ )
+ if next_backward_chunk is not None and next_backward_chunk is self:
+ # Don't offload the last group with the same name if it's about to be used immediately
+ if self.find_next_group(group._name) is None:
+ debug_rank(f"next group {group._name} is not found")
+ return False
+
+ return True
+
+ def bulk_offload(self, forced_released_tensors):
+ """Offload a group of tensors and optionally release their GPU memory."""
+ debug_rank("----bulk_offload")
+ if self.should_bulk_offload():
+ self._groups_to_reload.append(self._groups_to_offload[-1])
+ self.bulk_offload_group()
+ # Manually release tensors not auto-freed by torch GC
+ if len(forced_released_tensors) > 0:
+ cur_stream = torch.cuda.current_stream()
+ for release_tensor in forced_released_tensors:
+ if self.tensor_need_offloading_checker(release_tensor):
+ # Ensure tensor is not in use before freeing
+ release_tensor.record_stream(cur_stream)
+ release_tensor.untyped_storage().resize_(0)
+
+ def on_group_commit_forward(self, forced_released_tensors):
+ """Called at the end of a layer group's forward pass to trigger offloading."""
+ if not self.do_offload:
+ return
+ debug_rank("--on_group_commit_forward")
+ # Wait for compute to finish before starting offload
+ self.d2h_stream.wait_stream(torch.cuda.current_stream())
+ self.bulk_offload(forced_released_tensors)
+
+ def bulk_reload(self):
+ """Reload the next group of tensors from CPU to GPU."""
+ debug_rank("--bulk_reload")
+ if len(self._groups_to_reload) > 0:
+ # Reload the next layer group
+ self.bulk_reload_group()
+ else:
+ # Pre-load the last layer of the next backward chunk to hide latency
+ next_backward_chunk = PipelineOffloadManager.get_instance().front_backward_chunk()
+ # Don't pre-reload the last layer if the next backward chunk hasn't finished fprop yet.
+ if (
+ next_backward_chunk is not None
+ and next_backward_chunk._offloaded_group_index
+ == next_backward_chunk._max_group_size
+ ):
+ next_backward_chunk.pre_reload_last_layer()
+
+ def on_group_commit_backward(self, name):
+ """
+ Called at the end of a layer group's backward pass.
+ Ensures correct chunk is active and synchronizes reloads.
+ """
+ if not self.do_offload:
+ return
+ debug_rank("--on_group_commit_backward")
+ cur_backward_chunk = PipelineOffloadManager.get_instance().cur_backward_chunk()
+ # Switch to this chunk if it's not already current
+ if cur_backward_chunk is not self:
+ PipelineOffloadManager.get_instance().pop_backward_chunk(name)
+ cur_backward_chunk = PipelineOffloadManager.get_instance().cur_backward_chunk()
+ assert cur_backward_chunk is self, f"Chunk mismatch {cur_backward_chunk} {self}"
+ # Wait for reload to complete before using tensors
+ if not is_graph_capturing() and len(self._reloading_group) > 0:
+ for reloading_group in self._reloading_group:
+ if reloading_group._name == name:
+ reloading_group.wait_reload_event(torch.cuda.current_stream())
+ self._reloading_group.remove(reloading_group)
+ break
+
+ def on_group_start_forward(self, name):
+ """
+ Called at the start of a layer group's forward pass.
+ Increments group index and prepares for offloading.
+ """
+ if not self.do_offload:
+ return
+ debug_rank(f"--on_group_start_forward {name}")
+ self._offloaded_group_index = self._offloaded_group_index + 1
+ if self.is_warmup:
+ self.offload_groups.append(OffloadTensorGroup(name))
+ self._max_group_size = max(self._max_group_size, self._offloaded_group_index)
+ debug_rank(f"max group size {self._max_group_size}")
+ else:
+ for group in self.offload_groups[self._offloaded_group_index - 1 :]:
+ if group._name == name:
+ break
+ self._offloaded_group_index = self._offloaded_group_index + 1
+ self._tensor_count_current_group = 0
+ self._groups_to_offload.append(self.offload_groups[self._offloaded_group_index - 1])
+ debug_rank(f"groups to offload {self._groups_to_offload}")
+
+ def on_group_start_backward(self):
+ """
+ Called at the start of a layer group's backward pass.
+ Triggers reloading of tensors from CPU.
+ """
+ if not self.do_offload:
+ return
+ debug_rank(f"--on_group_start_backward {self}")
+ # Wait for compute to finish before starting reload
+ self.h2d_stream.wait_stream(torch.cuda.current_stream())
+ self.bulk_reload()
+
+
+def fine_grained_offloading_disable_offload():
+ """Disable the offload."""
+ debug_rank("fine_grained_offloading_disable_offload")
+ PipelineOffloadManager.get_instance().disable_offload()
+
+
+def fine_grained_offloading_enable_offload():
+ """Enable the offload."""
+ debug_rank("fine_grained_offloading_enable_offload")
+ PipelineOffloadManager.get_instance().enable_offload()
+
+
+class FineGrainedOffloadingGroupCommitFunction(torch.autograd.Function):
+ """
+ Identity operation that marks the end of a layer group for offload synchronization.
+ Triggers offload during forward and synchronizes reload during backward.
+ """
+
+ @staticmethod
+ def forward(ctx, tensor, cur_forward_chunk, name, forced_released_tensors, delay_offload):
+ # pylint: disable=missing-function-docstring
+ debug_rank("FineGrainedOffloadingGroupCommitFunction forward")
+
+ if delay_offload:
+ PipelineOffloadManager.get_instance().push_offload_groups(
+ cur_forward_chunk.on_group_commit_forward, forced_released_tensors
+ )
+ else:
+ cur_forward_chunk.on_group_commit_forward(forced_released_tensors)
+ ctx.cpu_offload_handler = cur_forward_chunk
+ ctx.name = name
+ return tensor
+
+ @staticmethod
+ def backward(ctx, *grad_output):
+ # pylint: disable=missing-function-docstring
+ debug_rank("FineGrainedOffloadingGroupCommitFunction backward")
+
+ cpu_offload_handler = ctx.cpu_offload_handler
+ cpu_offload_handler.on_group_commit_backward(ctx.name)
+ return grad_output + (None, None, None, None)
+
+
+def fine_grained_offloading_group_commit(
+ tensor, name, forced_released_tensors=None, delay_offload=False
+):
+ """
+ Specify the tensors to be released after offloading.
+ forced_released_tensors is a list of tensors to be released after offloading.
+ The tensors will be untyped_storage().resize_(0) after offloading.
+ Note: specify the tensors only when they are not automatically released by torch gc.
+ """
+ # Be permissive: callers may pass a tuple/list of outputs (e.g., (q, k, v)).
+ # We only need to insert a single identity op into the autograd graph; applying
+ # it to the first tensor output is sufficient and keeps callers' code minimal.
+ if forced_released_tensors is None:
+ forced_released_tensors = []
+ if isinstance(tensor, tuple):
+ if len(tensor) == 0:
+ return tensor
+ committed0 = fine_grained_offloading_group_commit(
+ tensor[0],
+ name=name,
+ forced_released_tensors=forced_released_tensors,
+ delay_offload=delay_offload,
+ )
+ return (committed0,) + tensor[1:]
+ if isinstance(tensor, list):
+ if len(tensor) == 0:
+ return tensor
+ committed0 = fine_grained_offloading_group_commit(
+ tensor[0],
+ name=name,
+ forced_released_tensors=forced_released_tensors,
+ delay_offload=delay_offload,
+ )
+ return [committed0] + tensor[1:]
+
+ cur_forward_chunk = PipelineOffloadManager.get_instance().cur_forward_chunk()
+ if cur_forward_chunk is None:
+ return tensor
+ return FineGrainedOffloadingGroupCommitFunction.apply(
+ tensor, cur_forward_chunk, name, forced_released_tensors, delay_offload
+ )
+
+
+class FineGrainedOffloadingGroupStartFunction(torch.autograd.Function):
+ """
+ Identity operation that marks the start of a layer group for offload/reload.
+ Prepares for offload during forward and triggers reload during backward.
+ """
+
+ @staticmethod
+ def forward(ctx, tensor, cpu_offload_handler, name):
+ # pylint: disable=missing-function-docstring
+ ctx.cpu_offload_handler = cpu_offload_handler
+ debug_rank("FineGrainedOffloadingGroupStartFunction forward")
+
+ cpu_offload_handler.on_group_start_forward(name)
+ # return the identical tensor
+ return tensor
+
+ @staticmethod
+ def backward(ctx, grad_output):
+ # pylint: disable=missing-function-docstring
+ debug_rank("FineGrainedOffloadingGroupStartFunction backward")
+ cpu_offload_handler = ctx.cpu_offload_handler
+ cpu_offload_handler.on_group_start_backward()
+ return grad_output, None, None, None
+
+
+def fine_grained_offloading_group_start(tensor, name=None):
+ """Mark the start of a layer group and prepare for offload/reload."""
+ cur_forward_chunk = PipelineOffloadManager.get_instance().pop_forward_chunk(name=name)
+ if cur_forward_chunk is None:
+ return tensor
+ return FineGrainedOffloadingGroupStartFunction.apply(tensor, cur_forward_chunk, name)
+
+
+class FineGrainedOffloadingBackwardRecordFunction(torch.autograd.Function):
+ """
+ Identity operation that marks the end of a layer group for offload synchronization.
+ Triggers offload during forward and synchronizes reload during backward.
+ """
+
+ @staticmethod
+ def forward(ctx, tensor, event: torch.cuda.Event) -> torch.Tensor:
+ """Forward pass for cuda graph capture."""
+ ctx.event = event
+ return tensor
+
+ @staticmethod
+ def backward(ctx, grad_output):
+ """Record the backward event and wait for the h2d stream on cuda graph stream."""
+ h2d_stream = PipelineOffloadManager.get_instance().h2d_stream
+ torch.cuda.current_stream().record_event(ctx.event)
+ torch.cuda.current_stream().wait_stream(h2d_stream)
+ return grad_output, None
+
+
+class FineGrainedActivationOffloadingInterface:
+ """Interface for fine-grained activation offloading."""
+
+ def __init__(self, offload: bool, tensor: torch.Tensor, name: str):
+ self.offload = offload
+ self.tensor = tensor
+ self.name = name
+
+ def __enter__(self):
+ """Enter context manager to enable activation offloading hooks."""
+ if self.offload:
+ self.tensor = fine_grained_offloading_group_start(self.tensor, self.name)
+ PipelineOffloadManager.get_instance().__enter__()
+ return self.tensor
+
+ def __exit__(self, *args: Any):
+ """Exit context manager to disable activation offloading hooks."""
+ if self.offload:
+ PipelineOffloadManager.get_instance().__exit__()
+
+ @staticmethod
+ def init_chunk_handler(vp_size, vp_stage, min_offloaded_tensor_size):
+ """Initialize the chunk handler, called at the start of a microbatch forward pass."""
+ PipelineOffloadManager.get_instance().init_model_chunk_offload_handler(
+ vp_size, vp_stage, min_offloaded_tensor_size
+ )
+
+ @staticmethod
+ def get_context(flag):
+ """Get the fine-grained offload context"""
+ return PipelineOffloadManager.get_instance() if flag else nullcontext()
+
+ @staticmethod
+ def group_commit(tensor, name, forced_released_tensors=None, delay_offload=False):
+ """Group commit the tensors."""
+ return fine_grained_offloading_group_commit(
+ tensor, name, forced_released_tensors, delay_offload
+ )
+
+ @staticmethod
+ def mark_not_offloadable(tensor: torch.Tensor):
+ """Mark the tensor as not offloadable."""
+ PipelineOffloadManager.get_instance().mark_not_offloadable(tensor)
+
+ @staticmethod
+ def forward_record(event: torch.cuda.Event) -> None:
+ """Record the forward event for cuda graph capture."""
+ d2h_stream = PipelineOffloadManager.get_instance().d2h_stream
+ torch.cuda.current_stream().record_event(event)
+ torch.cuda.current_stream().wait_stream(d2h_stream)
+
+ @staticmethod
+ def backward_record(tensor, event: torch.cuda.Event) -> torch.Tensor:
+ """Record the backward event for cuda graph capture."""
+ return FineGrainedOffloadingBackwardRecordFunction.apply(tensor, event)
+
+ @staticmethod
+ def reset():
+ """Reset the chunk handler."""
+ PipelineOffloadManager.get_instance().reset()
+
+ @staticmethod
+ def reset_instance():
+ """Reset the singleton instance."""
+ PipelineOffloadManager.reset_instance()
+
+ @staticmethod
+ def flush_delayed_groups():
+ """Flush the delayed groups."""
+ PipelineOffloadManager.get_instance().flush_delayed_groups()
diff --git a/megatron/core/pipeline_parallel/hybrid_cp_schedule.py b/megatron/core/pipeline_parallel/hybrid_cp_schedule.py
new file mode 100644
index 00000000000..27b5fc87945
--- /dev/null
+++ b/megatron/core/pipeline_parallel/hybrid_cp_schedule.py
@@ -0,0 +1,660 @@
+# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved.
+
+from collections import deque
+from functools import lru_cache
+from math import ceil, log2
+from typing import Callable, List, Optional, Tuple
+
+import torch
+
+from megatron.core import parallel_state
+from megatron.core.rerun_state_machine import RerunDataIterator
+
+
+class BalancedCPScheduler:
+ """
+ This class provides the functionality to form groups of sub-samples
+ such that all DPxCP ranks have a roughly balanced workload in the group.
+ """
+
+ def __init__(self, max_seq_len_per_rank: int, dp_cp_group: torch.distributed.ProcessGroup):
+ self.max_seq_len_per_rank = max_seq_len_per_rank
+ self.num_subsamples = 0
+ self.num_subsamples_processed = 0
+ self.free_resources = []
+ self.total_hdp_gpus = dp_cp_group.size()
+
+ @lru_cache(maxsize=128)
+ def get_total_workload(self, seq_length: int, cp_size: Optional[int] = None):
+ """
+ seq_length: sequence length of a sub-sample
+ cp_size: total number of CP ranks working on this sub-sample
+
+ Note:
+ This function is used to estimate the relative workload intensity
+ of a sub-sample. This is not meant to be an accurate flops calculator.
+
+ Returns: workload of a sub-sample
+ """
+ if cp_size is None:
+ cp_size = self.gpus_needed(seq_length)
+ return (seq_length * seq_length) / cp_size
+
+ @lru_cache(maxsize=128)
+ def gpus_needed(self, seq_len: int) -> int:
+ """
+ Calculates the number of GPUs needed for a given sequence length
+ and max sequence length per CP rank.
+ This is used to determine the CP size of a sub-sample.
+
+ The number is rounded up to the next power of 2 to match the available
+ hybrid context parallel process group sizes.
+ """
+ return max(1, 2 ** ceil(log2((seq_len / self.max_seq_len_per_rank))))
+
+ def make_buckets_equal(
+ self,
+ sample_seqlens: List[Tuple[int, int]], # List of (sample_id, sequence_length) tuples
+ compute_estimator: Callable[[int], float],
+ ) -> List[deque]:
+ """
+ Makes as many buckets as unique CP sizes needed.
+ This keeps sample IDs tethered to their sequence lengths throughout the bucketing process.
+ """
+ # Extract just the sequence lengths for determining k
+ seqlens = [seq_len for _, seq_len in sample_seqlens]
+
+ # Determine k based on unique GPU categories needed
+ k = len({self.gpus_needed(L) for L in seqlens})
+
+ # Create a work target for each bucket
+ # This is the total work divided by the number of buckets
+ work = []
+ for _, s in sample_seqlens:
+ cp_size = self.gpus_needed(s)
+ work.append(compute_estimator(s, cp_size))
+ total_work = sum(work)
+ target = total_work / k
+ buckets, cur, cur_work = [], [], 0.0
+ remaining_work = total_work
+ remaining_k = k
+
+ for i, (sample_id, seq_len) in enumerate(sample_seqlens):
+ work = compute_estimator(seq_len)
+ projected = cur_work + work
+
+ # Check if we should close this bucket
+ if cur and (
+ projected > target * 1.1 # Too much work
+ or len(sample_seqlens) - i <= remaining_k - len(buckets)
+ ): # Need to save sequences for remaining buckets
+ buckets.append(deque(cur))
+ cur, cur_work = [], 0.0
+ remaining_work -= sum(compute_estimator(seq_len) for _, seq_len in cur)
+ remaining_k -= 1
+
+ cur.append((sample_id, seq_len))
+ cur_work += work
+
+ if cur:
+ buckets.append(deque(cur))
+
+ return buckets
+
+ def next_hdp_group(
+ self,
+ sample_seqlens: List[Tuple[int, int]], # List of (sample_id, sequence_length) tuples
+ compute_estimator: Callable[[int], float],
+ total_gpus: int,
+ delta: float = 0.05, # balance slack (e.g. 5 %)
+ strategy: str = "dp", # "dp" or "pp"
+ eps_bucket: float = 0.10, # ε target for bucket balance
+ ) -> Tuple[List[List[int]], List[Tuple[int, int]], List[float], List[List[int]]]:
+ """
+ Given a list of (sample_id, sequence_length) tuples, this function aims to assign
+ sequences in a group such that all GPUs in the DPxCP group have a roughly balanced
+ workload. Once each group is roughly balanced, we exit and return the
+ group and the leftover sequences.
+
+ The function performs the following passes in order to form a balanced microbatch:
+ 1. We create buckets of sequences that are roughly balanced.
+ We try to create as many buckets as possible CP sizes.
+ 2. Given a bucket has sequences available, we assign the sample
+ a. To a new set of GPUs if there are enough free GPUs.
+ b. To an existing set of GPUs with the lowest load.
+ 3. We check if the group is balanced whenever we need to move onto a new CP size
+ in the same set of GPUs.
+ 4. We trim the group if removing the last added sequence helps improve balance.
+ 5. If we run out of sequences to assign and there are empty GPUs,
+ we redistribute work to empty GPUs by recursively increasing the CP size of a
+ sample until no empty GPUs are left.
+
+ Returns (micro_batches, leftover_sample_seqlens, exec_times, sample_ids_per_gpu).
+ """
+ if not sample_seqlens:
+ return (
+ [[] for _ in range(total_gpus)],
+ [],
+ [0.0 for _ in range(total_gpus)],
+ [[] for _ in range(total_gpus)],
+ )
+
+ # Get buckets of sequences with balanced work
+ buckets = self.make_buckets_equal(sample_seqlens, compute_estimator)
+
+ # Initialize tracking structures
+ micro_batches = [[] for _ in range(total_gpus)]
+ exec_times = [0.0 for _ in range(total_gpus)]
+ sample_ids_per_gpu = [[] for _ in range(total_gpus)]
+
+ gpu_group_id = [None] * total_gpus
+ group_members = {}
+ group_size = {}
+ next_gid = 0
+
+ pp_cursor = 0
+ prev_needed = None
+ check_balance = False
+
+ while buckets:
+ # ---- Step 1 – pick the next sequence we COULD place ------------------
+ sample_seq_tuple = bucket_idx = None
+ needed = None
+
+ scan_order = (
+ range(len(buckets))
+ if strategy == "dp"
+ else [(pp_cursor + i) % len(buckets) for i in range(len(buckets))]
+ )
+
+ for idx in scan_order:
+ if not buckets[idx]:
+ continue
+ cand_tuple = buckets[idx][0] # This is now (sample_id, seq_len)
+ cand_seq_len = cand_tuple[1]
+ needed = self.gpus_needed(cand_seq_len)
+
+ # (a) Do we have an *existing* group of size `needed`?
+ candidate_gids = [gid for gid, sz in group_size.items() if sz == needed]
+
+ # (b) Or enough completely free GPUs to start a new group?
+ free_ranks = [r for r, gid in enumerate(gpu_group_id) if gid is None]
+ if candidate_gids or len(free_ranks) >= needed:
+ sample_seq_tuple, bucket_idx = cand_tuple, idx
+ break
+
+ # No place to put any remaining sequence – finish this micro‑batch
+ if sample_seq_tuple is None:
+ break
+
+ # TODO[pmannan]: PP not yet supported. Add PP scheduling.
+ if strategy == "pp":
+ pp_cursor = (bucket_idx + 1) % len(buckets)
+
+ sample_id, seq_len = sample_seq_tuple
+ needed = self.gpus_needed(seq_len)
+ if prev_needed is None:
+ prev_needed = needed
+
+ # (a) Existing groups of exactly this size
+ candidate_gids = [gid for gid, sz in group_size.items() if sz == needed]
+ if candidate_gids:
+ best_gid, best_load = min(
+ (
+ (gid, max(exec_times[r] for r in group_members[gid]))
+ for gid in candidate_gids
+ ),
+ key=lambda t: t[1],
+ )
+ else:
+ best_gid, best_load = None, float("inf")
+
+ # (b) Hypothetical **new** group from completely free GPUs
+ free_ranks = [r for r, gid in enumerate(gpu_group_id) if gid is None]
+ if len(free_ranks) >= needed:
+ free_sorted = sorted(free_ranks, key=lambda r: exec_times[r])
+ new_members = free_sorted[:needed]
+ new_load = exec_times[new_members[-1]]
+
+ if new_load < best_load:
+ best_gid = None
+ chosen_members = new_members
+ else:
+ chosen_members = group_members[best_gid]
+ else:
+ chosen_members = group_members[best_gid]
+
+ # ---- Step 2 – if we decided to create a fresh group ----------------
+ if best_gid is None:
+ best_gid = next_gid
+ next_gid += 1
+ group_members[best_gid] = chosen_members
+ group_size[best_gid] = needed
+ for r in chosen_members:
+ gpu_group_id[r] = best_gid
+
+ # ---- Step 3 – assign the sequence to every member of that group ------
+ per_gpu_cost = compute_estimator(seq_len)
+
+ for r in chosen_members:
+ micro_batches[r].append(seq_len)
+ exec_times[r] += per_gpu_cost
+ sample_ids_per_gpu[r].append(sample_id)
+
+ # Remove the sequence definitively from its bucket
+ buckets[bucket_idx].popleft()
+
+ # ---- Step 4 – tidy, balance‑check, maybe early‑exit ------------------
+ while buckets and not buckets[0]:
+ buckets.pop(0)
+ pp_cursor %= max(1, len(buckets))
+
+ # TODO: Removing this helps reduce the number of groups when we have
+ # lots of samples with same CP size.
+ # But because we don't exit as soon as we get balanced,
+ # even if there is one group available that can take the next sample,
+ # we will keep adding samples to the same group.
+ # trim_overload() does not help because it only checks if removing the
+ # last added sample helps.
+ # We cannot check after adding every sample because there will always be imbalance
+ # if we don't wait for future scheduling.
+
+ # IMPORTANT: So we need a solution here
+ if needed < prev_needed:
+ # When we get into a lower CP size in the same group,
+ # we can start checking for balance. There is still a gotcha here.
+ # Let's say we have a group of 3 GPU 0-2, then we move onto group of 2.
+ # We keep assigning group of 2 as we do in descending order but GPU 7/15
+ # never sees a microbatch assigned to it
+ # until we run out of samples with CP2.
+ # This means we are never balanced as min(exec_times) will always be 0.
+ # We need a smart way of identifying that we have run out of big samples
+ # and if we are having to assign work to a GPU already working,
+ # is it because there are empty GPUs?
+ # Would assigning work to empty GPUs first by moving onto next CP bucket help?
+ # But we need to remember to come back to this CP size bucket and then
+ # check for balance. Maybe the scheduling algorithm should look at empty
+ # GPUs and find work rather than going sequence by sequence.
+ check_balance = True
+
+ if (
+ check_balance
+ and buckets
+ and max(exec_times) - min(exec_times) <= delta * max(exec_times)
+ ):
+ break
+
+ # Gather leftovers (flatten remaining buckets, preserve order)
+ leftovers = []
+ for b in buckets:
+ for sample_seq_tuple in b:
+ leftovers.append(sample_seq_tuple)
+
+ # ---------------------------------------------------------------------------
+ def trim_overload():
+ """
+ Iteratively pop the most‑recent sequence from the *most‑loaded group*
+ whenever doing so reduces the global slack.
+ """
+ while True:
+ cur_max = max(exec_times)
+ cur_min = min(exec_times)
+ cur_slack = cur_max - cur_min
+ if cur_slack <= delta * cur_max:
+ # Slack is already within limit.
+ break
+ if cur_min == 0:
+ # There are empty GPUs that will be
+ # handled in the next step.
+ break
+
+ max_r = exec_times.index(cur_max)
+ gid = gpu_group_id[max_r]
+ members = group_members[gid]
+
+ if not micro_batches[max_r] or len(micro_batches[max_r]) <= 1:
+ break
+
+ seq = micro_batches[max_r][-1]
+ need = group_size[gid]
+ per_gpu_cost = compute_estimator(seq)
+
+ proj_times = exec_times[:]
+ for r in members:
+ proj_times[r] -= per_gpu_cost
+
+ proj_slack = max(proj_times) - min(proj_times)
+
+ # Check if trimming the workload helps imbalance
+ if proj_slack < cur_slack:
+ sample_id_to_remove = sample_ids_per_gpu[max_r][-1]
+ for r in members:
+ micro_batches[r].pop()
+ exec_times[r] -= per_gpu_cost
+ sample_ids_per_gpu[r].pop()
+ leftovers.append((sample_id_to_remove, seq))
+ else:
+ break
+
+ trim_overload()
+
+ # Track samples in this group before redistribution to empty GPUs
+ total_work_before = sum(len(mb) for mb in micro_batches)
+
+ # Check for empty GPUs and redistribute work
+ def fill_empty_gpus(
+ micro_batches, exec_times, sample_ids_per_gpu, group_members, group_size
+ ):
+ """
+ Recursively check for empty GPUs and redistribute work by increasing
+ the number of GPUs sharing samples. This ensures all GPUs have work.
+ GPUs must be allocated consecutively so we may need to push existing
+ work to other ranks in order to expand samples.
+ """
+ # Find empty GPUs
+ empty_gpus = [i for i in range(total_gpus) if not micro_batches[i]]
+ if not empty_gpus:
+ return (
+ micro_batches,
+ exec_times,
+ sample_ids_per_gpu,
+ group_members,
+ group_size,
+ ) # No empty GPUs, we're done
+
+ # Find the smallest group size that exists
+ existing_group_sizes = set(group_size.values())
+ assert (
+ existing_group_sizes
+ ), "There should be at least one group existing, cannot reditribute, "
+ "try to increase 'max-seqlen-per-cp-rank'."
+
+ min_group_size = min(existing_group_sizes)
+ # We have Hybrid DPxCP groups for every power of 2 of GPUs or the entire DPxCP group.
+ next_power = min(min_group_size * 2, total_gpus)
+
+ # Find the first group of min_group_size that can be expanded
+ expandable_gid = None
+ expandable_members = None
+ expandable_new_gpus = None
+
+ for gid, size in group_size.items():
+ if size == min_group_size:
+ members = group_members[gid]
+ needed_count = next_power - min_group_size
+ group_start_gpu = members[0]
+ group_end_gpu = members[-1]
+ empty_gpu = [idx for idx, work in enumerate(micro_batches) if not work][0]
+ assert not all(
+ work for work in micro_batches[empty_gpu : empty_gpu + needed_count]
+ ), f"Empty GPUs were detected but not enough to expand."
+ work_to_push = micro_batches[
+ group_end_gpu + 1 : empty_gpu
+ ] # This is work of all other subsequent sub-samples
+ exec_times_to_push = exec_times[group_end_gpu + 1 : empty_gpu]
+ sample_ids_to_push = sample_ids_per_gpu[group_end_gpu + 1 : empty_gpu]
+
+ new_micro_batches = [[]] * len(micro_batches)
+ new_exec_times = [0.0] * len(exec_times)
+ new_sample_ids_per_gpu = [[]] * len(sample_ids_per_gpu)
+
+ # No change in work until the group selected for expansion
+ for i in range(group_start_gpu):
+ new_micro_batches[i] = micro_batches[i]
+ new_exec_times[i] = exec_times[i]
+ new_sample_ids_per_gpu[i] = sample_ids_per_gpu[i]
+
+ # The work is distributed across the expanded group
+ for i in range(group_start_gpu, group_end_gpu + needed_count + 1):
+ new_micro_batches[i] = micro_batches[group_end_gpu]
+ new_exec_times[i] = self.get_total_workload(
+ micro_batches[group_end_gpu][0], next_power
+ )
+ new_sample_ids_per_gpu[i] = sample_ids_per_gpu[group_end_gpu]
+
+ # Any assigned work on expanded GPUs is pushed
+ for i, work in enumerate(work_to_push):
+ new_micro_batches[group_end_gpu + needed_count + 1 + i] = work
+ new_exec_times[group_end_gpu + needed_count + 1 + i] = exec_times_to_push[i]
+ new_sample_ids_per_gpu[group_end_gpu + needed_count + 1 + i] = (
+ sample_ids_to_push[i]
+ )
+
+ group_size[gid] = next_power
+ group_members[gid] = list(range(members[0], members[-1] + needed_count + 1))
+ for pushed_gid in group_size.keys():
+ if pushed_gid > gid:
+ group_members[pushed_gid] = [
+ x + needed_count for x in group_members[pushed_gid]
+ ]
+
+ return (
+ new_micro_batches,
+ new_exec_times,
+ new_sample_ids_per_gpu,
+ group_members,
+ group_size,
+ )
+
+ empty_gpus = any([not micro_batches[i] for i in range(total_gpus)])
+ while empty_gpus:
+ micro_batches, exec_times, sample_ids_per_gpu, group_members, group_size = (
+ fill_empty_gpus(
+ micro_batches, exec_times, sample_ids_per_gpu, group_members, group_size
+ )
+ )
+ empty_gpus = any([not micro_batches[i] for i in range(total_gpus)])
+
+ # Assert that no sample has been completely removed
+ total_work_after = sum(len(mb) for mb in micro_batches)
+ assert (
+ total_work_after >= total_work_before
+ ), f"Samples were removed: {total_work_before} -> {total_work_after}"
+
+ return micro_batches, leftovers, exec_times, sample_ids_per_gpu
+
+ def get_groups_and_subsamples(self, sample_id_seqlens, config):
+ """
+ This function recursively forms groups of sub-samples such that all DPxCP ranks
+ have a roughly balanced workload in the group.
+ """
+ groups = []
+ sample_id_groups = []
+ # We assign a sample_id to each sub-sample in order to track assignment to each GPU.
+ sample_id_seqlens = sorted(sample_id_seqlens, key=lambda x: x[1], reverse=True)
+ while sample_id_seqlens:
+ mb, sample_id_seqlens, exec_times, sample_ids = self.next_hdp_group(
+ sample_id_seqlens, self.get_total_workload, self.total_hdp_gpus
+ )
+ groups.append(mb)
+ if len(sample_ids) < self.total_hdp_gpus:
+ sample_ids.extend([] * (self.total_hdp_gpus - len(sample_ids)))
+ sample_id_groups.append(sample_ids)
+
+ return groups, sample_id_groups
+
+
+def hybrid_context_parallel_forward_backward(
+ forward_step_func,
+ data_iterator,
+ model,
+ num_microbatches,
+ input_tensor,
+ output_tensor_grad,
+ forward_data_store,
+ config,
+ collect_non_loss_data,
+ first_val_step,
+ forward_only,
+ no_sync_func,
+ total_num_tokens,
+ check_first_val_step,
+ model_type,
+):
+ """
+ Scheduler for Hybrid Context Parallel.
+
+ This function performs the packed sample scheduling and determines
+ 1. The number of microbatches to schedule for each CP rank
+ 2. The number of groups each CP rank should execute
+ 3. The number of sub-samples per group each CP rank should execute
+
+ A group is defined by a set of samples that can run across the CP domain without any barrier.
+ There are many reasons why we may not be able to run endless samples within a single group.
+ For example, if we have 8 GPUs,
+ if GPU 0-5 are assigned a long sample that requires CP6,
+ GPU 6-7 are assigned a short sample that requires CP2,
+ The next sample which requires CP4 can be assigned GPU 4-7.
+ But GPU 6-7 will finish first and get deadlocked if GPU 4-5 are not participating in the group.
+ """
+ from .schedules import backward_step, forward_step
+
+ def _broadcast(item):
+ if item is not None:
+ torch.distributed.broadcast(
+ item,
+ parallel_state.get_tensor_model_parallel_src_rank(),
+ group=parallel_state.get_tensor_model_parallel_group(),
+ )
+
+ def _broadcast_num_samples_this_group(num_samples_this_group):
+ dev = torch.cuda.current_device()
+ torch.distributed.barrier()
+
+ n = 0 if num_samples_this_group is None else int(num_samples_this_group.numel())
+ n = torch.tensor([n], dtype=torch.int64, device=dev)
+
+ _broadcast(n)
+ n = int(n.item())
+
+ assert n > 0, "there should be at least 1 sub samples in the group"
+ num_samples_this_group_broadcast = (
+ torch.empty(n, dtype=torch.int32, device=dev)
+ if num_samples_this_group is None
+ else num_samples_this_group
+ )
+ _broadcast(num_samples_this_group_broadcast)
+ return num_samples_this_group_broadcast
+
+ def _get_new_data_iterator(sample_id_in_group, group_id):
+ if is_first_tp_rank:
+ sub_sample_id = sample_ids_this_group[sample_id_in_group]
+ sample = batch[sub_sample_id]
+ partner_cp_size = len(
+ [True for sample_ids in sample_id_groups[group_id] if sub_sample_id in sample_ids]
+ )
+ sample["local_cp_size"] = torch.tensor(partner_cp_size, dtype=torch.int32)
+ new_data_iterator = RerunDataIterator(iter([sample]))
+ return new_data_iterator
+ else:
+ return None
+
+ # We get data once per global batch and schedule the sub-samples.
+ # TODO(pmannan): Should we wrap the data_iterator here instead of the training.py file?
+ hdp_rank = parallel_state.get_data_parallel_rank(with_context_parallel=True)
+ is_first_tp_rank = parallel_state.get_tensor_model_parallel_rank() == 0
+
+ if is_first_tp_rank:
+ data = next(data_iterator)
+ sample_id_groups = data[1]
+ batch = data[0]
+ else:
+ data, sample_id_groups, batch = None, None, None
+
+ num_samples_this_group = None
+ if is_first_tp_rank:
+ num_samples_this_group = torch.tensor(
+ [len(group[hdp_rank]) for group in sample_id_groups], dtype=torch.int32, device='cuda'
+ )
+
+ num_samples_this_group = _broadcast_num_samples_this_group(num_samples_this_group)
+ num_samples_this_group = num_samples_this_group.cpu().numpy()
+ num_total_groups = num_samples_this_group.shape[0]
+
+ current_microbatch = 0
+
+ # Upto last group, we don't need any sync.
+ with no_sync_func():
+ for j in range(num_total_groups - 1):
+ sample_ids_this_group = sample_id_groups[j][hdp_rank] if is_first_tp_rank else None
+ for i in range(num_samples_this_group[j]):
+ # Call forward step for each sub-sample
+ new_data_iterator = _get_new_data_iterator(i, j)
+ # TODO: Find the usage of current_microbatch and is_first_microbatch and
+ # how that may affect my usage.
+ output_tensor, num_tokens = forward_step(
+ forward_step_func,
+ new_data_iterator,
+ model,
+ num_microbatches,
+ input_tensor,
+ forward_data_store,
+ config,
+ collect_non_loss_data,
+ is_first_microbatch=check_first_val_step(
+ first_val_step, forward_only, current_microbatch == 0
+ ),
+ current_microbatch=current_microbatch,
+ )
+ current_microbatch += 1
+ total_num_tokens += num_tokens.item()
+ if not forward_only:
+ backward_step(
+ input_tensor, output_tensor, output_tensor_grad, model_type, config
+ )
+
+ # Create a barrier at end of each group.
+ # This barrier ensures that all ranks are prepared to change assigned CP group sizes and
+ # no rank is starting a sub-sample ahead of it's partner ranks.
+ torch.distributed.barrier(
+ parallel_state.get_data_parallel_group(with_context_parallel=True)
+ )
+
+ # For the last group, we need to run the last sub-sample out of the context handler.
+ with no_sync_func():
+ sample_ids_this_group = sample_id_groups[-1][hdp_rank] if is_first_tp_rank else None
+ for i in range(num_samples_this_group[-1] - 1):
+ new_data_iterator = _get_new_data_iterator(i, -1)
+ # Call forward step for each sub-sample
+ output_tensor, num_tokens = forward_step(
+ forward_step_func,
+ new_data_iterator,
+ model,
+ num_microbatches,
+ input_tensor,
+ forward_data_store,
+ config,
+ collect_non_loss_data,
+ is_first_microbatch=check_first_val_step(
+ first_val_step, forward_only, current_microbatch == 0
+ ),
+ current_microbatch=current_microbatch,
+ )
+ current_microbatch += 1
+ total_num_tokens += num_tokens.item()
+ if not forward_only:
+ backward_step(input_tensor, output_tensor, output_tensor_grad, model_type, config)
+
+ # The last sub-sample of the last group of the last microbatch is
+ # run out of the context handler.
+ new_data_iterator = _get_new_data_iterator(-1, -1)
+ # Call forward step for each sub-sample
+ output_tensor, num_tokens = forward_step(
+ forward_step_func,
+ new_data_iterator,
+ model,
+ num_microbatches,
+ input_tensor,
+ forward_data_store,
+ config,
+ collect_non_loss_data,
+ is_first_microbatch=check_first_val_step(
+ first_val_step, forward_only, current_microbatch == 0
+ ),
+ current_microbatch=current_microbatch,
+ )
+ total_num_tokens += num_tokens.item()
+ if not forward_only:
+ backward_step(input_tensor, output_tensor, output_tensor_grad, model_type, config)
+
+ return forward_data_store, total_num_tokens
diff --git a/megatron/core/pipeline_parallel/multimodule_communicator.py b/megatron/core/pipeline_parallel/multimodule_communicator.py
new file mode 100644
index 00000000000..dfda270ef76
--- /dev/null
+++ b/megatron/core/pipeline_parallel/multimodule_communicator.py
@@ -0,0 +1,523 @@
+# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved.
+
+import logging
+from dataclasses import dataclass
+from typing import Dict, List, Optional, Union
+
+import torch
+import torch.distributed as dist
+
+from megatron.core.hyper_comm_grid import HyperCommGrid
+from megatron.core.model_parallel_config import ModelParallelConfig
+from megatron.core.pipeline_parallel.bridge_communicator import BridgeCommunicator
+from megatron.core.pipeline_parallel.p2p_communication import P2PCommunicator
+
+# Types
+Shape = Union[List[int], torch.Size]
+
+
+@dataclass
+class RankModuleInfo:
+ """Information about a rank in a module."""
+
+ # the stage of the current rank in the current module's pipeline.
+ pp_rank: int # the stage of the current rank in the current module's pipeline
+ pp_size: int # the number of ranks in the current module's pipeline
+ p2p_communicator: Optional[P2PCommunicator]
+ # key is either the src or dst module name connected to the current module
+ # one module may have multiple bridge communicators if it has multiple
+ # incoming or outgoing connections.
+ bridge_comms_as_src_module: Optional[List[BridgeCommunicator]]
+ bridge_comms_as_dest_module: Optional[List[BridgeCommunicator]]
+ # the absolute first stage in the overall model
+ # no incoming connections
+ is_source_stage: Optional[bool] = True
+ # the absolute last stage in the overall model
+ # no outgoing connections
+ is_terminal_stage: Optional[bool] = True
+
+
+class MultiModulePipelineCommunicator:
+ """Communicator for a multi-module pipeline."""
+
+ def __init__(
+ self,
+ module_to_grid_map: Dict[str, HyperCommGrid],
+ topology: Dict[str, List[str]],
+ config: ModelParallelConfig,
+ dim_mapping: Dict[str, List[int]] = None,
+ ):
+ """
+ Initialize the MultiModulePipelineCommunicator.
+
+ Args:
+ module_to_grid_map (dict): A dictionary mapping module names to HyperCommGrids.
+ Example:
+ module_to_grid_map = {
+ 'image_encoder': image_encoder_grid,
+ 'audio_encoder': audio_encoder_grid,
+ 'llm': llm_grid,
+ 'generator': generator_grid
+ }
+ topology (dict): A dictionary mapping module names to lists of outgoing modules.
+ Example:
+ topology = {
+ 'image_encoder': ['llm'],
+ 'audio_encoder': ['llm'],
+ 'llm': ['generator'],
+ 'generator': []
+ }
+ config (ModelParallelConfig): A ModelParallelConfig object.
+ dim_mapping (Dict[str, List[int]]): Dimension mapping for sequence, batch, hidden.
+ Example:
+ dim_mapping = {'s': 0, 'h': 2, 'b': 1}
+ Default: None
+ """
+ self.module_to_grid_map = module_to_grid_map
+ self.topology = topology
+ self.config = config
+ self.dim_mapping = dim_mapping
+ self.current_rank = dist.get_rank()
+
+ # Build bridge communicators for all modules
+ self.bridge_comms = []
+ self._build_bridge_comms()
+
+ self.rank_module_map = {}
+ self._build_rank_module_info_map()
+
+ def _build_bridge_comms(self):
+ """Construct and store BridgeCommunicator objects that describe the outgoing
+ communication relationships for all of the modules.
+ """
+ for src_module_name, src_grid in self.module_to_grid_map.items():
+ for dest_module_name in self.topology[src_module_name]:
+ dest_grid = self.module_to_grid_map[dest_module_name]
+ bridge_comm = BridgeCommunicator(
+ src_grid=src_grid,
+ dest_grid=dest_grid,
+ dim_mapping=self.dim_mapping,
+ comm_dtype=self.config.pipeline_dtype,
+ src_module_name=src_module_name,
+ dest_module_name=dest_module_name,
+ )
+ self.bridge_comms.append(bridge_comm)
+
+ @property
+ def is_pp_first_stage(self):
+ """Return True if the current rank has the absolute first stage in the overall model.
+
+ The absolute first stage is defined as:
+ 1. The current rank must be in the first PP stage (pp_rank == 0) of some module
+ 2. That module must be a source module (no incoming connections in topology)
+ """
+ for module_name, rank_module_info in self.rank_module_map.items():
+ # Check if this rank is at the first PP stage of this module
+ if rank_module_info.pp_rank == 0:
+ # Check if this module is a source module (no incoming connections)
+ if self._is_source_module(module_name):
+ return True
+ return False
+
+ @property
+ def is_pp_last_stage(self):
+ """Return True if the current rank has the absolute last stage in the overall model.
+
+ The absolute last stage is defined as:
+ 1. The current rank must be in the last PP stage of some module
+ 2. That module must be a sink module (no outgoing connections in topology)
+ """
+ for module_name, rank_module_info in self.rank_module_map.items():
+ # Check if this rank is at the last PP stage of this module
+ if rank_module_info.pp_rank == rank_module_info.pp_size - 1:
+ # Check if this module is a sink module (no outgoing connections)
+ if self._is_sink_module(module_name):
+ return True
+ return False
+
+ def _is_source_module(self, module_name: str) -> bool:
+ """Check if a module is a source module (has no incoming connections)."""
+ # A module is a source if no other module lists it as a destination
+ for src_module, dest_modules in self.topology.items():
+ if module_name in dest_modules:
+ return False
+ return True
+
+ def _is_sink_module(self, module_name: str) -> bool:
+ """Check if a module is a sink module (has no outgoing connections)."""
+ return len(self.topology.get(module_name, [])) == 0
+
+ def is_current_rank_in_grid(self, grid: HyperCommGrid) -> bool:
+ """Check if the current rank is in the grid."""
+ return grid.rank_offset <= self.current_rank < grid.rank_offset + grid.size
+
+ @property
+ def num_warmup_microbatches(self):
+ """Calculate the number of warmup microbatches for the current rank.
+
+ Uses the same simple logic as P2PCommunicator:
+ total_pipeline_stages - current_rank_stage - 1
+
+ Returns:
+ int: Number of warmup microbatches for this rank
+ """
+ # Get total pipeline depth across all modules
+ total_stages = self.compute_total_pipeline_stages(self.topology, self.module_to_grid_map)
+
+ # Get current rank's position in the overall pipeline (0-indexed)
+ # Use compute_total_pipeline_stages with current rank to get cumulative position
+ if self.rank_module_map:
+ # Take the first module this rank belongs to
+ # TODO: ykarnati - improve this logic.
+ module_name = next(iter(self.rank_module_map.keys()))
+ current_stage = (
+ self.compute_total_pipeline_stages(
+ self.topology,
+ self.module_to_grid_map,
+ rank=self.current_rank,
+ module_name=module_name,
+ )
+ - 1
+ ) # Convert from 1-indexed to 0-indexed
+ else:
+ current_stage = 0
+
+ assert (
+ current_stage <= total_stages
+ ), f"current_stage: {current_stage} is greater than total_stages: {total_stages}"
+ logging.debug(
+ f"[Rank {dist.get_rank()} ][MultiModulePipelineCommunicator] "
+ f"current_stage: {current_stage} total_stages: {total_stages} "
+ f"num_warmup_microbatches: {total_stages - current_stage - 1}"
+ )
+ return total_stages - current_stage - 1
+
+ def _build_rank_module_info_map(self):
+ """For each module in the current rank, initialize the P2P communicator
+ and build the bridge communicator info for the module.
+ Each rank may hold multiple modules when colocated.
+ """
+ for module_name, module_grid in self.module_to_grid_map.items():
+ if self.is_current_rank_in_grid(module_grid):
+ # Initialize P2P communicator
+ pp_group = module_grid.get_pg('pp')
+ p2p_comm = P2PCommunicator(pp_group, self.config)
+ pp_size = dist.get_world_size(pp_group)
+ rank_in_pp_group = dist.get_group_rank(pp_group, self.current_rank)
+ pp_rank = rank_in_pp_group % pp_size
+
+ bridge_comms_as_dest_module = []
+ bridge_comms_as_src_module = []
+ # If first stage, check if the module has any incoming modules
+ # If so, initialize bridge communicator
+ if pp_rank == 0:
+ for bridge_comm in self.bridge_comms:
+ if (
+ bridge_comm.is_current_rank_in_grid(bridge_comm.dest_grid)
+ and bridge_comm.dest_module_name == module_name
+ ):
+ bridge_comms_as_dest_module.append(bridge_comm)
+ # If last stage, check if the module has any outgoing modules
+ # If so, initialize bridge communicator
+ if pp_rank == pp_size - 1:
+ for bridge_comm in self.bridge_comms:
+ if (
+ bridge_comm.is_current_rank_in_grid(bridge_comm.src_grid)
+ and bridge_comm.src_module_name == module_name
+ ):
+ bridge_comms_as_src_module.append(bridge_comm)
+ # Build RankModuleInfo for the module
+ rank_module_info = RankModuleInfo(
+ pp_rank=pp_rank,
+ pp_size=pp_size,
+ p2p_communicator=p2p_comm,
+ bridge_comms_as_dest_module=bridge_comms_as_dest_module,
+ bridge_comms_as_src_module=bridge_comms_as_src_module,
+ )
+ self.rank_module_map[module_name] = rank_module_info
+
+ def recv_forward(
+ self, tensor_shape: Optional[Shape] = None, is_first_stage: bool = False
+ ) -> Dict[str, torch.Tensor]:
+ """Receive forward activation tensor.
+
+ Args:
+ tensor_shape: Expected activation tensor shape
+
+ Returns:
+ A dictionary mapping module names to tensors.
+ """
+ logging.debug(
+ f"[Rank {dist.get_rank()} ][MultiModulePipelineCommunicator] "
+ f"[receive_forward] tensors_shape: {tensor_shape}, is_first_stage: {is_first_stage}"
+ )
+ input_dict = {}
+ for module_name, rank_module_info in self.rank_module_map.items():
+
+ if rank_module_info.pp_rank == 0:
+ # If first stage, and has incoming modules, receive forward activation
+ # from incoming modules.
+ for bridge_comm in rank_module_info.bridge_comms_as_dest_module:
+ input_dict[bridge_comm.src_module_name] = bridge_comm.recv_forward()
+ else:
+ # If not first stage, receive forward activation tensor from P2P communicator.
+ input_dict[module_name] = rank_module_info.p2p_communicator.recv_forward(
+ tensor_shapes=tensor_shape, is_first_stage=False
+ )
+ return input_dict
+
+ def send_forward(self, output_dict: Dict[str, torch.Tensor], is_last_stage: bool = False):
+ """Send forward activation tensor.
+
+ Args:
+ output_dict: A dictionary mapping module names to tensors.
+ """
+ logging.debug(
+ f"[Rank {dist.get_rank()} ][MultiModulePipelineCommunicator] "
+ f"[send_forward] output_dict keys: {output_dict.keys()}, is_last_stage: {is_last_stage}"
+ )
+ for module_name, rank_module_info in self.rank_module_map.items():
+ if rank_module_info.pp_rank == rank_module_info.pp_size - 1:
+ # If last stage, and has outgoing modules, send forward activation
+ # by using bridge communicator.
+ for bridge_comm in rank_module_info.bridge_comms_as_src_module:
+ bridge_comm.send_forward(output_dict[module_name])
+ else:
+ # If not last stage, send forward activation by using P2P communicator.
+ rank_module_info.p2p_communicator.send_forward(
+ output_dict[module_name], is_last_stage=False
+ )
+
+ def send_forward_recv_backward(
+ self,
+ output_dict: Dict[str, torch.Tensor],
+ tensor_shape: Optional[Shape] = None,
+ is_last_stage: bool = False,
+ ) -> Dict[str, torch.Tensor]:
+ """Send forward activation tensor and receive backward activation tensor.
+
+ Args:
+ output_dict: A dictionary mapping module names to tensors.
+ tensor_shape: Expected gradient tensor shape
+
+ Returns:
+ A dictionary mapping module names to tensors.
+ """
+ logging.debug(
+ f"[Rank {dist.get_rank()} ][MultiModulePipelineCommunicator] "
+ f"[send_forward_recv_backward] output_dict keys: {output_dict.keys()}, "
+ f"tensor_shape: {tensor_shape}, is_last_stage: {is_last_stage}"
+ )
+ grad_dict = {}
+ for module_name, rank_module_info in self.rank_module_map.items():
+ if rank_module_info.pp_rank == rank_module_info.pp_size - 1:
+ # If last stage, and has outgoing modules, send forward activation and
+ # receive backward gradient by using bridge communicator.
+ for bridge_comm in rank_module_info.bridge_comms_as_src_module:
+ grad_dict[bridge_comm.src_module_name] = bridge_comm.send_forward_recv_backward(
+ output_dict[module_name]
+ )
+ else:
+ # If not last stage, send forward activation and receive backward gradient
+ # by using P2P communicator.
+ grad_dict[module_name] = (
+ rank_module_info.p2p_communicator.send_forward_recv_backward(
+ output_dict[module_name], tensor_shapes=tensor_shape, is_last_stage=False
+ )
+ )
+ return grad_dict
+
+ def send_backward_recv_forward(
+ self,
+ grad_dict: Dict[str, torch.Tensor],
+ tensor_shape: Optional[Shape] = None,
+ is_first_stage: bool = False,
+ ) -> Dict[str, torch.Tensor]:
+ """Send backward activation tensor and receive forward activation tensor.
+
+ Args:
+ grad_dict: A dictionary mapping module names to tensors.
+ tensor_shape: Expected gradient tensor shape
+
+ Returns:
+ A dictionary mapping module names to tensors.
+ """
+ logging.debug(
+ f"[Rank {dist.get_rank()} ][MultiModulePipelineCommunicator] "
+ f"[send_backward_recv_forward] grad_dict keys: {grad_dict.keys()}, "
+ f"tensor_shape: {tensor_shape}, is_first_stage: {is_first_stage}"
+ )
+ input_dict = {}
+ for module_name, rank_module_info in self.rank_module_map.items():
+ if rank_module_info.pp_rank == 0:
+ for bridge_comm in rank_module_info.bridge_comms_as_dest_module:
+ # If first stage, and has incoming modules, send backward gradient and
+ # receive forward activation by using bridge communicator.
+ input_dict[bridge_comm.src_module_name] = (
+ bridge_comm.send_backward_recv_forward(
+ grad_dict[bridge_comm.src_module_name]
+ )
+ )
+ else:
+ # If not first stage, send backward gradient and receive forward activation
+ # by using P2P communicator.
+ input_dict[module_name] = (
+ rank_module_info.p2p_communicator.send_backward_recv_forward(
+ grad_dict[module_name], tensor_shapes=tensor_shape, is_first_stage=False
+ )
+ )
+ return input_dict
+
+ def recv_backward(
+ self, tensor_shape: Optional[Shape] = None, is_last_stage: bool = False
+ ) -> Dict[str, torch.Tensor]:
+ """Receive backward activation tensor.
+
+ Args:
+ tensor_shape: Expected gradient tensor shape
+
+ Returns:
+ A dictionary mapping module names to tensors.
+ """
+ logging.debug(
+ f"[Rank {dist.get_rank()} ][MultiModulePipelineCommunicator] "
+ f"[recv_backward] tensor_shape: {tensor_shape}, is_last_stage: {is_last_stage}"
+ )
+ grad_dict = {}
+ for module_name, rank_module_info in self.rank_module_map.items():
+ if rank_module_info.pp_rank == rank_module_info.pp_size - 1:
+ # If last stage, and has incoming modules, receive backward gradient
+ # by using bridge communicator.
+ for bridge_comm in rank_module_info.bridge_comms_as_src_module:
+ grad_dict[bridge_comm.src_module_name] = bridge_comm.recv_backward()
+ else:
+ # If not last stage, receive backward gradient by using P2P communicator.
+ grad_dict[module_name] = rank_module_info.p2p_communicator.recv_backward(
+ tensor_shapes=tensor_shape, is_last_stage=False
+ )
+ return grad_dict
+
+ def send_backward(self, grad_dict: Dict[str, torch.Tensor], is_first_stage: bool = False):
+ """Send backward activation tensor.
+
+ Args:
+ grad_dict: A dictionary mapping module names to tensors.
+ """
+ logging.debug(
+ f"[Rank {dist.get_rank()} ][MultiModulePipelineCommunicator] "
+ f"[send_backward] grad_dict keys: {grad_dict.keys()}, is_first_stage: {is_first_stage}"
+ )
+ for module_name, rank_module_info in self.rank_module_map.items():
+ if rank_module_info.pp_rank == 0:
+ # If first stage, and has incoming modules, send backward activation
+ # by using bridge communicator.
+ for bridge_comm in rank_module_info.bridge_comms_as_dest_module:
+ bridge_comm.send_backward(grad_dict[bridge_comm.src_module_name])
+ else:
+ # If not first stage, send backward activation by using P2P communicator.
+ rank_module_info.p2p_communicator.send_backward(
+ grad_dict[module_name], is_first_stage=False
+ )
+
+ @staticmethod
+ def compute_total_pipeline_stages(
+ topology: Dict[str, List[str]],
+ module_to_grid_map: Dict[str, HyperCommGrid],
+ rank: Optional[int] = None,
+ module_name: Optional[str] = None,
+ ) -> int:
+ """Compute the total number of pipeline stages across a multi-module chain.
+
+ Interprets ``topology`` as a directed acyclic graph (DAG) where nodes are modules
+ and edges indicate forward data flow from source to destination modules. Each node
+ is assigned a weight equal to its pipeline parallel size (number of PP stages).
+
+ The total number of stages is defined as the length of the longest path in this DAG
+ under node weights.
+
+ If ``rank`` is None (default), returns the maximum over all terminal (sink) modules of
+ the sum of PP sizes along a path ending at that terminal. For example, given:
+
+ image_encoder ->\
+ -> llm -> generator
+ audio_encoder ->/
+
+ the total is: max(pp(image_encoder), pp(audio_encoder)) + pp(llm) + pp(generator).
+
+ If ``rank`` is provided, the result is the total number of pipeline stages up to (and
+ including) the PP stage that ``rank`` occupies inside its module. In this case, the
+ weight of the target module equals (pp_rank_index(rank) + 1) instead of the module's
+ full PP size; other modules still contribute their full PP sizes. If the rank belongs to
+ multiple modules (colocation), pass ``module_name`` to disambiguate; otherwise the
+ maximum across all candidate modules containing the rank is returned.
+
+ Args:
+ topology: Mapping from a module to its list of outgoing modules.
+ module_to_grid_map: Mapping from module name to its ``HyperCommGrid``.
+
+ Returns:
+ The total number of pipeline stages along the longest path given the constraints.
+
+ Raises:
+ ValueError: If the topology contains cycles; or has no terminal nodes when
+ ``rank`` is None
+ """
+ nodes = set(module_to_grid_map.keys())
+ # Build adjacency and reverse-adjacency (predecessors).
+ adj: Dict[str, List[str]] = {node: list(topology.get(node, [])) for node in nodes}
+ preds: Dict[str, List[str]] = {node: [] for node in nodes}
+ for src, outs in adj.items():
+ for dst in outs:
+ preds[dst].append(src)
+
+ # Identify terminal nodes (no outgoing edges) for the rank=None case.
+ sinks = [node for node, outs in adj.items() if not outs]
+ if rank is None and not sinks:
+ raise ValueError(
+ "Topology must be a DAG with at least one terminal (no outgoing) module."
+ )
+
+ def pp_size(name: str) -> int:
+ grid = module_to_grid_map[name]
+ pp_dim_index = grid.dim_names.index('pp')
+ return grid.shape[pp_dim_index]
+
+ def partial_weight_for_target(target: str) -> Optional[int]:
+ if rank is None:
+ return None
+ grid = module_to_grid_map.get(target)
+ rank_groups = grid._gen_rank_enum(['pp'])
+ stage_index: Optional[int] = None
+ for group in rank_groups:
+ if rank in group:
+ stage_index = group.index(rank)
+ break
+ return stage_index + 1
+
+ def longest_path_to(target: str) -> int:
+ visiting = set()
+ partial = partial_weight_for_target(target)
+
+ def weight(name: str) -> int:
+ if partial is not None and name == target:
+ return partial
+ return pp_size(name)
+
+ def dfs(node: str) -> int:
+ if node in visiting:
+ raise ValueError("Topology contains cycles; expected a DAG.")
+ visiting.add(node)
+ best = 0
+ for p in preds.get(node, []):
+ val = dfs(p)
+ if val > best:
+ best = val
+ visiting.remove(node)
+ return weight(node) + best
+
+ return dfs(target)
+
+ if rank is None:
+ return max(longest_path_to(sink) for sink in sinks)
+
+ return longest_path_to(module_name)
diff --git a/megatron/core/pipeline_parallel/p2p_communication.py b/megatron/core/pipeline_parallel/p2p_communication.py
index 63ee9d1f537..ac839c21f18 100644
--- a/megatron/core/pipeline_parallel/p2p_communication.py
+++ b/megatron/core/pipeline_parallel/p2p_communication.py
@@ -1,4 +1,4 @@
-# Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved.
+# Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
from typing import List, Optional, Tuple, Union
@@ -214,22 +214,22 @@ def _communicate_shapes(self, tensor_send_next, tensor_send_prev, recv_prev, rec
ops = []
if send_prev_shape_tensor is not None:
send_prev_op = torch.distributed.P2POp(
- torch.distributed.isend, send_prev_shape_tensor, self.prev_rank
+ torch.distributed.isend, send_prev_shape_tensor, self.prev_rank, self.pp_group
)
ops.append(send_prev_op)
if recv_prev_shape_tensor is not None:
recv_prev_op = torch.distributed.P2POp(
- torch.distributed.irecv, recv_prev_shape_tensor, self.prev_rank
+ torch.distributed.irecv, recv_prev_shape_tensor, self.prev_rank, self.pp_group
)
ops.append(recv_prev_op)
if send_next_shape_tensor is not None:
send_next_op = torch.distributed.P2POp(
- torch.distributed.isend, send_next_shape_tensor, self.next_rank
+ torch.distributed.isend, send_next_shape_tensor, self.next_rank, self.pp_group
)
ops.append(send_next_op)
if recv_next_shape_tensor is not None:
recv_next_op = torch.distributed.P2POp(
- torch.distributed.irecv, recv_next_shape_tensor, self.next_rank
+ torch.distributed.irecv, recv_next_shape_tensor, self.next_rank, self.pp_group
)
ops.append(recv_next_op)
if len(ops) > 0:
@@ -298,13 +298,13 @@ def _communicate(
tensor_recv_prev_func = None
tensor_recv_next_func = None
- if not config.variable_seq_lengths:
- recv_prev_shape = tensor_shape
- recv_next_shape = tensor_shape
- else:
+ if config.variable_seq_lengths or config.mtp_standalone:
recv_prev_shape, recv_next_shape = self._communicate_shapes(
tensor_send_next, tensor_send_prev, recv_prev, recv_next
)
+ else:
+ recv_prev_shape = tensor_shape
+ recv_next_shape = tensor_shape
def create_tensor_recv_prev():
return torch.empty(
diff --git a/megatron/core/pipeline_parallel/schedules.py b/megatron/core/pipeline_parallel/schedules.py
index 5ab4cb57545..dadbd199ab7 100644
--- a/megatron/core/pipeline_parallel/schedules.py
+++ b/megatron/core/pipeline_parallel/schedules.py
@@ -1,7 +1,8 @@
-# Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved.
+# Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
import contextlib
from functools import partial
+from itertools import zip_longest
from typing import Callable, Iterator, List, Optional, Union
import torch
@@ -9,6 +10,9 @@
from megatron.core import parallel_state
from megatron.core.enums import ModelType
+from megatron.core.pipeline_parallel.fine_grained_activation_offload import (
+ FineGrainedActivationOffloadingInterface as off_interface,
+)
from megatron.core.pipeline_parallel.p2p_communication import P2PCommunicator
from megatron.core.pipeline_parallel.utils import (
is_pp_first_stage,
@@ -18,6 +22,7 @@
)
from megatron.core.process_groups_config import ProcessGroupCollection
from megatron.core.transformer.cuda_graphs import create_cudagraphs
+from megatron.core.transformer.enums import CudaGraphScope
from megatron.core.transformer.moe.router import MoEAuxLossAutoScaler
from megatron.core.utils import (
drain_embedding_wgrad_compute,
@@ -32,12 +37,13 @@
combined_1f1b_schedule_for_interleaved_pipelining,
combined_1f1b_schedule_for_no_pipelining,
)
+from .hybrid_cp_schedule import hybrid_context_parallel_forward_backward
# Types
Shape = Union[List[int], torch.Size]
-def get_forward_backward_func():
+def get_forward_backward_func(pp_size: Optional[int] = None, vp_size: Optional[int] = None):
"""Retrieves the appropriate forward_backward function given the
configuration of parallel_state.
@@ -120,10 +126,18 @@ def forward_step(data_iterator, model):
respective list of shapes. Thus it is not used in the other forward-backward functions
which have different shape handling.
+ Args:
+ pp_size (Optional[int]): Pipeline model parallel size to use.
+ vp_size (Optional[int]): Virtual pipeline model parallel size to use.
+ If both pp_size and vp_size are None, both values fall back to parallel_state.
+ Otherwise, provided values are used as-is and None is treated as an explicit input.
"""
- pipeline_model_parallel_size = parallel_state.get_pipeline_model_parallel_world_size()
- if pipeline_model_parallel_size > 1:
- if parallel_state.get_virtual_pipeline_model_parallel_world_size() is not None:
+ if pp_size is None and vp_size is None:
+ pp_size = parallel_state.get_pipeline_model_parallel_world_size()
+ vp_size = parallel_state.get_virtual_pipeline_model_parallel_world_size()
+
+ if pp_size > 1:
+ if vp_size is not None:
forward_backward_func = forward_backward_pipelining_with_interleaving
else:
forward_backward_func = forward_backward_pipelining_without_interleaving
@@ -270,6 +284,8 @@ def forward_step_calc_loss(
if config.calculate_per_token_loss:
MoEAuxLossAutoScaler.set_loss_scale(loss_scale)
else:
+ # See https://github.com/NVIDIA/Megatron-LM/pull/2217 for detailed explanation
+ # of scaling by cp_group_size
MoEAuxLossAutoScaler.set_loss_scale(loss_scale * cp_group_size / num_microbatches)
# Set the loss scale for Multi-Token Prediction (MTP) loss.
@@ -509,6 +525,7 @@ def forward_backward_no_pipelining(
collect_non_loss_data: bool = False,
first_val_step: Optional[bool] = None,
adjust_tensor_shapes_fn: Optional[Callable] = None, # unused
+ p2p_communicator: Optional[P2PCommunicator] = None, # unused
pg_collection: Optional[ProcessGroupCollection] = None,
):
"""Run forward and backward passes with no pipeline parallelism"""
@@ -591,6 +608,24 @@ def forward_backward_no_pipelining(
total_num_tokens,
partial(check_first_val_step, first_val_step, forward_only),
)
+ elif config.hybrid_context_parallel:
+ forward_data_store, total_num_tokens = hybrid_context_parallel_forward_backward(
+ forward_step_func,
+ data_iterator,
+ model,
+ num_microbatches,
+ input_tensor,
+ output_tensor_grad,
+ forward_data_store,
+ config,
+ collect_non_loss_data,
+ first_val_step,
+ forward_only,
+ no_sync_func,
+ total_num_tokens,
+ check_first_val_step,
+ model_type,
+ )
else:
with no_sync_func():
for i in range(num_microbatches - 1):
@@ -644,13 +679,16 @@ def forward_backward_no_pipelining(
pg_collection=pg_collection,
)
+ if not forward_only and config.fine_grained_activation_offloading:
+ off_interface.reset()
+
if config.timers is not None:
config.timers('forward-backward').stop()
if (
hasattr(config, 'cuda_graph_impl')
and config.cuda_graph_impl == "local"
- and config.cuda_graph_scope != "full_iteration"
+ and CudaGraphScope.full_iteration not in config.cuda_graph_scope
):
create_cudagraphs()
@@ -808,6 +846,110 @@ def convert_schedule_table_to_order(num_warmup_microbatches, num_model_chunks, s
return order
+def get_overlap_moe_expert_parallel_comm_order(order, num_layers_per_chunk, capture_wgrad_graph):
+ """
+ This functions gets the order for overlap_moe_expert_parallel_comm schedule for the original
+ chunk-wise order list. Each chunk is transformered to chunks with only 1 layer so that
+ layers between 2 chunks can now overlap with each other while following the graph order.
+ If capture_wgrad_graph is True, the wgrad backward graph is also added to the order by
+ decreasing the layer id by 0.5.
+
+ Args:
+ order (List[int]): The original chunk-wise order list. Positive values represent forward
+ passes for chunks, negative values represent backward passes. The absolute value
+ indicates the chunk ID (1-indexed).
+ num_layers_per_chunk (List[int]): Number of graphable layers in each chunk. The length
+ of this list equals the number of chunks.
+ capture_wgrad_graph (bool): If True, weight gradient computation graphs are added to the
+ order by appending entries with layer_id - 0.5.
+
+ Returns:
+ Tuple[List[float], List[Optional[List[int]]]]: A tuple containing:
+ - new_order: The layer-wise order list where each chunk is expanded to individual
+ layers. Positive values are forward passes, negative values are backward passes.
+ Values with .5 suffix indicate weight gradient computations.
+ - chunk_id_list: A list parallel to new_order. For forward passes, contains
+ [chunk_id, layer_index_within_chunk]. For backward passes, contains None.
+
+ Example:
+ original_order: [1, 2, -2, 1, -1, -1]
+ num_layers_per_chunk: [1, 2]
+ capture_wgrad_graph=True:
+ new_order: [1, 2, 3, 1, -3, -3.5, -2, -2.5, -1, -1.5, -1, -1.5]
+ chunk_id_list: [[0, 0], [1, 0], [1, 1], [0, 0], None,
+ None, None, None, None, None, None, None]
+ capture_wgrad_graph=False:
+ new_order: [1, 2, 3, 1, -3, -2, -1, -1]
+ chunk_id_list: [[0, 0], [1, 0], [1, 1], [0, 0], None, None, None, None]
+ """
+
+ def _add_order(new_order, chunk_id_list, c_id, layer_id, is_wgrad=False, index=None):
+ if is_wgrad:
+ new_order.append(layer_id - 0.5)
+ else:
+ new_order.append(layer_id)
+ if c_id > 0:
+ chunk_id_list.append([abs(c_id) - 1, index])
+ else:
+ chunk_id_list.append(None)
+
+ new_order = []
+ chunk_id_list = []
+ add_order = partial(_add_order, new_order, chunk_id_list)
+ first_backward_idx, last_forward_idx = None, None
+ for idx, c_id in enumerate(order):
+ if first_backward_idx is None and c_id < 0:
+ first_backward_idx = idx
+ if c_id > 0:
+ last_forward_idx = idx
+
+ def get_layer_range(c_id):
+ num_layers = num_layers_per_chunk[abs(c_id) - 1]
+ num_layers_previous_chunks = sum(num_layers_per_chunk[: abs(c_id) - 1])
+ if c_id > 0:
+ return list(
+ range(num_layers_previous_chunks + 1, num_layers_previous_chunks + num_layers + 1)
+ )
+ return list(range(-num_layers_previous_chunks - num_layers, -num_layers_previous_chunks))
+
+ # warmup stage
+ for c_id in order[:first_backward_idx]:
+ layer_range = get_layer_range(c_id)
+ new_order += layer_range
+ chunk_id_list.extend([abs(c_id) - 1, i] for i in range(len(layer_range)))
+
+ # 1f1b overlap stage
+ if first_backward_idx < last_forward_idx:
+ for c_id_b, c_id_f in zip(
+ order[first_backward_idx : last_forward_idx + 1 : 2],
+ order[first_backward_idx + 1 : last_forward_idx + 1 : 2],
+ ):
+ layer_range_f = get_layer_range(c_id_f)
+ layer_range_b = get_layer_range(c_id_b)
+ index = 0
+ for l_b, l_f in zip_longest(layer_range_b, layer_range_f, fillvalue=0):
+ # always forward graph before backward graph
+ if l_f != 0:
+ add_order(c_id_f, l_f, index=index)
+ if l_b != 0:
+ add_order(c_id_b, l_b)
+ if capture_wgrad_graph and index < len(layer_range_b) - 1:
+ add_order(c_id_b, l_b, is_wgrad=True)
+ index += 1
+ # last wgrad backward
+ if capture_wgrad_graph and layer_range_b:
+ add_order(c_id_b, layer_range_b[-1], is_wgrad=True)
+
+ # cool down stage, backward graphs only
+ for c_id in order[last_forward_idx + 1 :]:
+ for l_b in get_layer_range(c_id):
+ add_order(c_id, l_b)
+ if capture_wgrad_graph:
+ add_order(c_id, l_b, is_wgrad=True)
+
+ return new_order, chunk_id_list
+
+
def forward_backward_pipelining_with_interleaving(
*,
forward_step_func,
@@ -1904,6 +2046,8 @@ def pp_post_backward(input_tensor_grad, vp_stage=None):
pg_collection=pg_collection,
)
+ if not forward_only and config.fine_grained_activation_offloading:
+ off_interface.reset()
# Restore config.grad_sync_func and config.param_sync_func.
if forward_only:
config.grad_sync_func, config.param_sync_func = grad_sync_func, param_sync_func
@@ -1914,7 +2058,7 @@ def pp_post_backward(input_tensor_grad, vp_stage=None):
if (
hasattr(config, 'cuda_graph_impl')
and config.cuda_graph_impl == "local"
- and config.cuda_graph_scope != "full_iteration"
+ and CudaGraphScope.full_iteration not in config.cuda_graph_scope
):
create_cudagraphs()
nvtx_range_pop(suffix="misc")
@@ -2292,13 +2436,16 @@ def enable_grad_sync():
pg_collection=pg_collection,
)
+ if not forward_only and config.fine_grained_activation_offloading:
+ off_interface.reset()
+
if config.timers is not None:
config.timers('forward-backward').stop()
if (
hasattr(config, 'cuda_graph_impl')
and config.cuda_graph_impl == "local"
- and config.cuda_graph_scope != "full_iteration"
+ and CudaGraphScope.full_iteration not in config.cuda_graph_scope
):
create_cudagraphs()
diff --git a/megatron/core/pipeline_parallel/utils.py b/megatron/core/pipeline_parallel/utils.py
index fae8e5466da..bda6334fc4b 100644
--- a/megatron/core/pipeline_parallel/utils.py
+++ b/megatron/core/pipeline_parallel/utils.py
@@ -1,5 +1,6 @@
# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved.
+import logging
from abc import ABC, abstractmethod
from contextlib import contextmanager
from typing import Callable, Optional
@@ -7,7 +8,9 @@
import torch
from torch.autograd import Variable
-from megatron.core.utils import get_pg_rank, get_pg_size, make_viewless_tensor
+from megatron.core.utils import get_pg_rank, get_pg_size, log_single_rank, make_viewless_tensor
+
+logger = logging.getLogger(__name__)
def is_pp_first_stage(pp_group: torch.distributed.ProcessGroup):
@@ -80,6 +83,39 @@ def make_viewless(e):
return e
+def set_ideal_affinity_for_current_gpu():
+ """Set CPU affinity for the current GPU to optimize host-device transfers."""
+ import uuid
+
+ try:
+ import cuda.bindings.driver as cuda_driver
+ import cuda.bindings.runtime as cuda_runtime
+ except:
+ try:
+ import cuda.cuda as cuda_driver
+ import cuda.cudart as cuda_runtime
+ except:
+ raise RuntimeError("Please install cuda-python to enable GPU affinity setting")
+ import pynvml
+
+ # Get current CUDA device ID
+ err, device_id = cuda_runtime.cudaGetDevice()
+ assert err == cuda_runtime.cudaError_t.cudaSuccess
+ # Get device UUID
+ err, device_uuid = cuda_driver.cuDeviceGetUuid(device_id)
+ assert err == cuda_driver.CUresult.CUDA_SUCCESS
+ # Set CPU affinity based on GPU's NUMA node
+ pynvml.nvmlInit()
+ handle = pynvml.nvmlDeviceGetHandleByUUID("GPU-" + str(uuid.UUID(bytes=device_uuid.bytes)))
+ pynvml.nvmlDeviceSetCpuAffinity(handle)
+
+ log_single_rank(
+ logger,
+ logging.WARNING,
+ f"Set CPU affinity for all GPUs for optimal host-device transfer performance",
+ )
+
+
@contextmanager
def stream_acquire_context(stream, event):
"""Stream acquire context"""
@@ -149,6 +185,8 @@ def __init__(
self.free_input = free_input
self.inputs = None
self.outputs = None
+ self.manual_grads_release = False
+ self.delay_grads_release = False
def default_backward_func(self, outputs, output_grad):
"""Default backward function"""
@@ -230,6 +268,12 @@ def _backward(self, *output_grad):
for g in output_grad:
if g is not None:
g.record_stream(self.stream)
+ # Manually trigger the memory release of dgrad tensor
+ # to avoid delayed garbage collection. If
+ # delay_grads_release is True, dgrad is last used in
+ # wgrad compute and skip the release here.
+ if self.manual_grads_release and not self.delay_grads_release:
+ g.untyped_storage().resize_(0)
grads = self.get_grad()
self._release_state()
diff --git a/megatron/core/safe_globals.py b/megatron/core/safe_globals.py
index 20d1694f084..8bcfe788f60 100755
--- a/megatron/core/safe_globals.py
+++ b/megatron/core/safe_globals.py
@@ -14,7 +14,7 @@
from megatron.core.enums import ModelType
from megatron.core.optimizer import OptimizerConfig
from megatron.core.rerun_state_machine import RerunDiagnostic, RerunMode, RerunState
-from megatron.core.transformer.enums import AttnBackend
+from megatron.core.transformer.enums import AttnBackend, CudaGraphScope
SAFE_GLOBALS = [
SimpleNamespace,
@@ -25,6 +25,7 @@
UInt32DType,
Namespace,
AttnBackend,
+ CudaGraphScope,
ModelType,
OptimizerConfig,
RerunDiagnostic,
diff --git a/megatron/core/ssm/gated_delta_net.py b/megatron/core/ssm/gated_delta_net.py
new file mode 100644
index 00000000000..16dc3a79ebb
--- /dev/null
+++ b/megatron/core/ssm/gated_delta_net.py
@@ -0,0 +1,900 @@
+# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved.
+# Copyright (c) 2025, Songlin Yang, Jan Kautz, Ali Hatamizadeh.
+
+# Some of this code was adopted from https://github.com/huggingface/transformers
+# This source code is licensed under the Apache license found in the
+# LICENSE file in the root directory of this source tree.
+
+import logging
+from dataclasses import dataclass, replace
+from typing import List, Optional, Tuple, Union
+
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+from torch import Tensor
+
+from megatron.core.dist_checkpointing import ShardedTensor
+from megatron.core.dist_checkpointing.mapping import ReplicaId, ShardedTensorFactory
+from megatron.core.fp8_utils import get_fp8_align_size
+from megatron.core.inference.contexts import BaseInferenceContext
+from megatron.core.jit import jit_fuser
+from megatron.core.packed_seq_params import PackedSeqParams
+from megatron.core.process_groups_config import ProcessGroupCollection
+from megatron.core.ssm.mamba_context_parallel import (
+ _all_to_all_cp2hp,
+ _all_to_all_hp2cp,
+ _redo_attention_load_balancing,
+ _undo_attention_load_balancing,
+)
+from megatron.core.tensor_parallel import get_cuda_rng_tracker
+from megatron.core.transformer import TransformerConfig
+from megatron.core.transformer.identity_op import IdentityOp
+from megatron.core.transformer.module import MegatronModule
+from megatron.core.transformer.spec_utils import ModuleSpec, build_module
+from megatron.core.transformer.utils import (
+ ensure_metadata_has_dp_cp_group,
+ make_sharded_tensors_for_checkpoint,
+ sharded_state_dict_default,
+)
+from megatron.core.utils import deprecate_inference_params, nvtx_range_pop, nvtx_range_push
+
+try:
+ from fla.modules.l2norm import l2norm
+ from fla.ops.gated_delta_rule import chunk_gated_delta_rule
+
+ HAVE_FLA = True
+except ImportError:
+ chunk_gated_delta_rule = None
+
+ HAVE_FLA = False
+
+try:
+ from causal_conv1d import causal_conv1d_fn
+except ImportError:
+ causal_conv1d_fn = None
+
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass
+class GatedDeltaNetSubmodules:
+ """
+ Contains the module specs for the input linear, output norm, and output linear layers.
+ """
+
+ in_proj: Union[ModuleSpec, type] = IdentityOp
+ out_norm: Union[ModuleSpec, type] = IdentityOp
+ out_proj: Union[ModuleSpec, type] = IdentityOp
+
+
+class GatedDeltaNet(MegatronModule):
+ """Gated Delta Net (GDN) layer class
+
+ GDN layer takes input with size [s, b, h]
+ and returns output of the same size.
+ """
+
+ def __init__(
+ self,
+ config: TransformerConfig,
+ submodules: GatedDeltaNetSubmodules,
+ layer_number: int = None,
+ bias: bool = False,
+ conv_bias: bool = False,
+ conv_init: Optional[float] = None,
+ use_qk_l2norm: bool = True,
+ A_init_range: Tuple[float, float] = (1, 16),
+ pg_collection: ProcessGroupCollection = None,
+ **kwargs,
+ ):
+ """
+ Args:
+ config: The config of the model.
+ submodules: Contains the module specs for the input and output linear layers.
+ layer_number: The layer number of this GDN layer.
+ bias: Whether to use bias in the linear layers.
+ conv_bias: Whether to use bias in the causal convolution.
+ conv_init: The initialization range for the causal convolution weights.
+ use_qk_l2norm: Whether to use L2 normalization in the kernel of the gated delta rule.
+ A_init_range: The initialization range for the attention weights.
+ pg_collection: The required process groups to use for tensor model parallel and context
+ parallel.
+ """
+
+ if not HAVE_FLA:
+ raise ImportError(
+ "FLA is not installed. Please install it with `pip install flash-linear-attention`."
+ )
+
+ super().__init__(config)
+
+ # Attributes from arguments
+ self.layer_number = layer_number
+ self.bias = bias
+ self.conv_bias = conv_bias
+ self.conv_init = conv_init
+ assert A_init_range[0] >= 0 and A_init_range[1] >= A_init_range[0]
+ self.A_init_range = A_init_range
+ self.use_qk_l2norm = use_qk_l2norm
+ assert pg_collection is not None, "pg_collection must be provided for GatedDeltaNet"
+ self.pg_collection = pg_collection
+ self.cp_size = self.pg_collection.cp.size()
+ self.tp_size = self.pg_collection.tp.size()
+ self.sp_size = self.tp_size if config.sequence_parallel else 1
+
+ # Attributes from config
+ self.config = config
+ self.hidden_size = config.hidden_size
+ self.act_fn = config.activation_func
+ self.activation = self.act_fn.__name__
+ self.conv_kernel_dim = config.linear_conv_kernel_dim
+ self.key_head_dim = config.linear_key_head_dim
+ self.value_head_dim = config.linear_value_head_dim
+ self.num_key_heads = config.linear_num_key_heads
+ self.num_value_heads = config.linear_num_value_heads
+ self.qk_dim = self.key_head_dim * self.num_key_heads
+ self.v_dim = self.value_head_dim * self.num_value_heads
+ self.qk_dim_local_tp = self.qk_dim // self.tp_size
+ self.v_dim_local_tp = self.v_dim // self.tp_size
+
+ # Input projection (hidden_states -> q, k, v, gate, beta, alpha)
+ # TODO: for now, output gate is forced for GDN.
+ # We may remove this restriction in the future.
+ self.in_proj_dim = self.qk_dim * 2 + self.v_dim * 2 + self.num_value_heads * 2
+ if self.config.fp8:
+ fp8_align_size = get_fp8_align_size(self.config.fp8_recipe)
+ assert self.in_proj_dim % fp8_align_size == 0, (
+ "For FP8, the innermost dimension of the GDN layer "
+ "input projection output tensor must be a multiple of 16."
+ )
+ self.in_proj = build_module(
+ submodules.in_proj,
+ self.hidden_size,
+ self.in_proj_dim,
+ config=self.config,
+ init_method=self.config.init_method,
+ gather_output=False,
+ bias=bias,
+ skip_bias_add=False,
+ is_expert=False,
+ tp_comm_buffer_name="fc1",
+ tp_group=self.pg_collection.tp,
+ )
+
+ # Conv1d for QKV
+ self.conv_dim = self.qk_dim * 2 + self.v_dim
+ self.conv_dim_local_tp = self.conv_dim // self.tp_size
+
+ # weight shape: [conv_dim, 1, d_conv]
+ # bias shape: [conv_dim]
+ self.conv1d = nn.Conv1d(
+ in_channels=self.conv_dim_local_tp,
+ out_channels=self.conv_dim_local_tp,
+ bias=conv_bias,
+ kernel_size=self.conv_kernel_dim,
+ groups=self.conv_dim_local_tp,
+ padding=self.conv_kernel_dim - 1,
+ device=torch.cuda.current_device(),
+ dtype=config.params_dtype,
+ )
+ setattr(self.conv1d.weight, "tensor_model_parallel", True)
+ if conv_bias:
+ setattr(self.conv1d.bias, "tensor_model_parallel", True)
+
+ # Time step projection (discretization)
+ self.num_v_heads_local_tp = self.num_value_heads // self.tp_size
+ # dt_bias parameter
+ self.dt_bias = nn.Parameter(
+ torch.empty(
+ self.num_v_heads_local_tp,
+ dtype=config.params_dtype,
+ device=torch.cuda.current_device(),
+ )
+ )
+ setattr(self.dt_bias, "tensor_model_parallel", True)
+ # A_log parameter
+ self.A_log = nn.Parameter(
+ torch.empty(
+ self.num_v_heads_local_tp,
+ dtype=config.params_dtype,
+ device=torch.cuda.current_device(),
+ )
+ )
+ setattr(self.A_log, "tensor_model_parallel", True)
+
+ # Output layernorm before projection
+ self.out_norm = build_module(
+ submodules.out_norm,
+ config=self.config,
+ hidden_size=self.value_head_dim,
+ eps=self.config.layernorm_epsilon,
+ )
+
+ self.out_proj = build_module(
+ submodules.out_proj,
+ self.v_dim,
+ self.hidden_size,
+ config=self.config,
+ init_method=self.config.output_layer_init_method,
+ bias=bias,
+ input_is_parallel=True,
+ skip_bias_add=True,
+ is_expert=False,
+ tp_comm_buffer_name="fc2",
+ tp_group=self.pg_collection.tp,
+ )
+
+ self.reset_parameters()
+
+ def reset_parameters(self):
+ """Reset the parameters."""
+ if self.config.perform_initialization:
+ with get_cuda_rng_tracker().fork():
+ # conv1d.weight
+ if self.conv_init is not None:
+ nn.init.uniform_(self.conv1d.weight, -self.conv_init, self.conv_init)
+ # dt_bias
+ torch.ones(
+ self.num_v_heads_local_tp,
+ out=self.dt_bias.data,
+ dtype=self.config.params_dtype,
+ device=torch.cuda.current_device(),
+ )
+ # A_log
+ A = torch.empty(
+ self.num_v_heads_local_tp,
+ dtype=self.config.params_dtype,
+ device=torch.cuda.current_device(),
+ ).uniform_(*self.A_init_range)
+ self.A_log.data.copy_(torch.log(A))
+
+ def forward(
+ self,
+ hidden_states: Tensor,
+ attention_mask: Tensor,
+ inference_context: Optional[BaseInferenceContext] = None,
+ packed_seq_params: Optional[PackedSeqParams] = None,
+ sequence_len_offset: Optional[int] = None,
+ *,
+ inference_params: Optional[BaseInferenceContext] = None,
+ **kwargs,
+ ):
+ """
+ Perform a forward pass through the GDN module.
+
+ Args:
+ hidden_states (Tensor): Hidden states.
+ attention_mask (Tensor): Attention mask.
+ inference_context (Optional[BaseInferenceContext]): Inference context that manages
+ KV cache.
+ packed_seq_params (Optional[PackedSeqparams]): Parameters used for THD format.
+ sequence_len_offset (Optional[int]): Sequence length offset used for
+ inference CUDA graphs.
+
+ Return:
+ (Tuple[Tensor, Tensor]) GDN output and bias.
+
+ """
+ # TODO: Deal with attention_mask
+
+ inference_context = deprecate_inference_params(inference_context, inference_params)
+
+ seq_len, batch, _ = hidden_states.shape
+ seq_len = seq_len * self.sp_size * self.cp_size
+
+ if inference_context is not None:
+ assert (
+ inference_context.is_static_batching()
+ ), "GDN does not currently support dynamic inference batching."
+ assert not self.config.sequence_parallel
+ # TODO: support inference
+ raise NotImplementedError("GDN does not support inference for now.")
+
+ if packed_seq_params is not None:
+ # TODO: support packed sequence
+ raise NotImplementedError("GDN does not support packed sequence for now.")
+
+ # Input projection
+ nvtx_range_push(suffix="in_proj")
+ qkvzba, _ = self.in_proj(hidden_states)
+ nvtx_range_pop(suffix="in_proj")
+
+ # CP All to All: CP to HP
+ qkvzba = tensor_a2a_cp2hp(
+ qkvzba,
+ seq_dim=0,
+ head_dim=-1,
+ cp_group=self.pg_collection.cp,
+ split_sections=[
+ self.qk_dim_local_tp,
+ self.qk_dim_local_tp,
+ self.v_dim_local_tp,
+ self.v_dim_local_tp,
+ self.num_value_heads // self.tp_size,
+ self.num_value_heads // self.tp_size,
+ ],
+ )
+
+ # Transpose: s b x --> b s x
+ # From sbhd to bshd format
+ qkvzba = qkvzba.transpose(0, 1)
+
+ # Split, reorder, and reshape the tensor into q, k, v, gate, beta, alpha
+ qkv, gate, beta, alpha = torch.split(
+ qkvzba,
+ [
+ (self.qk_dim_local_tp * 2 + self.v_dim_local_tp) // self.cp_size,
+ self.v_dim_local_tp // self.cp_size,
+ self.num_value_heads // self.tp_size // self.cp_size,
+ self.num_value_heads // self.tp_size // self.cp_size,
+ ],
+ dim=-1,
+ )
+ gate = gate.reshape(batch, seq_len, -1, self.value_head_dim)
+ beta = beta.reshape(batch, seq_len, -1)
+ alpha = alpha.reshape(batch, seq_len, -1)
+
+ # Convolution on qkv
+ qkv = qkv.transpose(1, 2).contiguous() # b, s, d -> b, d, s
+ nvtx_range_push(suffix="conv1d")
+ qkv_channels_split_sections = [
+ self.qk_dim_local_tp,
+ self.qk_dim_local_tp,
+ self.v_dim_local_tp,
+ ]
+ conv1d_weight = get_parameter_local_cp(
+ self.conv1d.weight,
+ dim=0,
+ cp_group=self.pg_collection.cp,
+ split_sections=qkv_channels_split_sections,
+ )
+ conv1d_bias = (
+ get_parameter_local_cp(
+ self.conv1d.bias,
+ dim=0,
+ cp_group=self.pg_collection.cp,
+ split_sections=qkv_channels_split_sections,
+ )
+ if self.conv_bias
+ else None
+ )
+ if (causal_conv1d_fn is None) or self.config.deterministic_mode:
+ conv_out = F.conv1d(
+ input=qkv,
+ weight=conv1d_weight,
+ bias=conv1d_bias,
+ stride=self.conv1d.stride,
+ padding=self.conv1d.padding,
+ dilation=self.conv1d.dilation,
+ groups=self.conv_dim_local_tp // self.cp_size,
+ )
+ qkv = self.act_fn(conv_out[..., :seq_len])
+ else:
+ assert self.activation in ["silu", "swish"]
+ qkv = causal_conv1d_fn(
+ x=qkv,
+ weight=conv1d_weight.squeeze(1), # d, 1, w -> d, w
+ bias=conv1d_bias,
+ activation=self.activation,
+ )
+ nvtx_range_pop(suffix="conv1d")
+ # Split qkv into query, key, and value
+ qkv = qkv.transpose(1, 2) # b, d, s -> b, s, d
+ query, key, value = torch.split(
+ qkv,
+ [
+ self.qk_dim_local_tp // self.cp_size,
+ self.qk_dim_local_tp // self.cp_size,
+ self.v_dim_local_tp // self.cp_size,
+ ],
+ dim=-1,
+ )
+ query = query.reshape(batch, seq_len, -1, self.key_head_dim)
+ key = key.reshape(batch, seq_len, -1, self.key_head_dim)
+ value = value.reshape(batch, seq_len, -1, self.value_head_dim)
+ # Apply L2 norm to query and key
+ if self.use_qk_l2norm:
+ query = l2norm(query.contiguous())
+ key = l2norm(key.contiguous())
+ if self.num_value_heads // self.num_key_heads > 1:
+ query = query.repeat_interleave(self.num_value_heads // self.num_key_heads, dim=2)
+ key = key.repeat_interleave(self.num_value_heads // self.num_key_heads, dim=2)
+
+ # Make contiguous
+ query = query.contiguous()
+ key = key.contiguous()
+ value = value.contiguous()
+ gate = gate.contiguous()
+ beta = beta.contiguous()
+ alpha = alpha.contiguous()
+
+ # Calculate g and beta
+ nvtx_range_push(suffix="g_and_beta")
+ A_log_local_cp = get_parameter_local_cp(self.A_log, dim=0, cp_group=self.pg_collection.cp)
+ dt_bias_local_cp = get_parameter_local_cp(
+ self.dt_bias, dim=0, cp_group=self.pg_collection.cp
+ )
+ g = -A_log_local_cp.exp() * F.softplus(alpha.float() + dt_bias_local_cp) # In fp32
+ beta = beta.sigmoid()
+ nvtx_range_pop(suffix="g_and_beta")
+
+ nvtx_range_push(suffix="gated_delta_rule")
+ if self.config.deterministic_mode:
+ core_attn_out, last_recurrent_state = torch_chunk_gated_delta_rule(
+ query,
+ key,
+ value,
+ g=g,
+ beta=beta,
+ initial_state=None,
+ output_final_state=False,
+ use_qk_l2norm_in_kernel=False,
+ )
+ else:
+ core_attn_out, last_recurrent_state = chunk_gated_delta_rule(
+ query,
+ key,
+ value,
+ g=g,
+ beta=beta,
+ initial_state=None,
+ output_final_state=False,
+ use_qk_l2norm_in_kernel=False,
+ )
+ nvtx_range_pop(suffix="gated_delta_rule")
+
+ # RMSNorm
+ nvtx_range_push(suffix="gated_norm")
+ norm_out = self._apply_gated_norm(core_attn_out, gate)
+ nvtx_range_pop(suffix="gated_norm")
+
+ # Transpose: b s x --> s b x
+ # From bshd back to sbhd format
+ norm_out = norm_out.reshape(batch, seq_len, -1)
+ norm_out = norm_out.transpose(0, 1).contiguous()
+
+ # CP all to all: HP to CP
+ norm_out = tensor_a2a_hp2cp(
+ norm_out, seq_dim=0, head_dim=-1, cp_group=self.pg_collection.cp
+ )
+
+ # Output projection
+ nvtx_range_push(suffix="out_proj")
+ out, out_bias = self.out_proj(norm_out)
+ nvtx_range_pop(suffix="out_proj")
+
+ return out, out_bias
+
+ @jit_fuser
+ def _apply_gated_norm(self, x, gate):
+ # Output Norm
+ x_dtype = x.dtype
+ x = x.reshape(-1, x.shape[-1])
+ y = self.out_norm(x)
+ # Output gate
+ gate = gate.reshape(-1, gate.shape[-1])
+ y = y * self.act_fn(gate.float())
+ y = y.to(x_dtype)
+ return y
+
+ def sharded_state_dict(self, prefix="", sharded_offsets=(), metadata=None, tp_group=None):
+ """Provide a sharded state dictionary for distributed checkpointing."""
+ # Guard for cases metadata is not provided
+ metadata = ensure_metadata_has_dp_cp_group(metadata)
+
+ sharded_state_dict = {}
+ # Parameters
+ self._save_to_state_dict(sharded_state_dict, "", keep_vars=True)
+ sharded_state_dict = make_sharded_tensors_for_checkpoint(
+ sharded_state_dict,
+ prefix,
+ tensor_parallel_layers_axis_map={
+ "A_log": 0,
+ "dt_bias": 0,
+ }, # parameters sharded across TP
+ sharded_offsets=sharded_offsets,
+ tp_group=(tp_group if tp_group is not None else self.pg_collection.tp),
+ dp_cp_group=metadata['dp_cp_group'],
+ )
+ # Submodules
+ tp_group = tp_group if tp_group is not None else self.pg_collection.tp
+ for name, module in self.named_children():
+ if name == "conv1d":
+ # Add TP sharding for Conv1d
+ module_sd = module.state_dict(prefix="", keep_vars=True)
+ tp_sharding_map = {f"weight": 0}
+ if self.conv_bias:
+ tp_sharding_map[f"bias"] = 0
+ module_sharded_sd = make_sharded_tensors_for_checkpoint(
+ module_sd,
+ f"{prefix}{name}.",
+ tp_sharding_map,
+ sharded_offsets,
+ tp_group=tp_group,
+ dp_cp_group=metadata['dp_cp_group'],
+ )
+ else:
+ module_sharded_sd = sharded_state_dict_default(
+ module, f"{prefix}{name}.", sharded_offsets, metadata, tp_group=tp_group
+ )
+
+ sharded_state_dict.update(module_sharded_sd)
+
+ # At this point the TP sharding is correctly defined for each tensor, but some of the
+ # tensors must be additionally split into separate parts
+ in_proj_dim_local_tp = self.in_proj_dim // self.tp_size
+ assert sharded_state_dict[f"{prefix}in_proj.weight"].data.size(0) == in_proj_dim_local_tp, (
+ in_proj_dim_local_tp,
+ sharded_state_dict[f"{prefix}in_proj.weight"],
+ )
+
+ sharded_state_dict[f"{prefix}in_proj.weight"] = _split_tensor_factory(
+ sharded_state_dict[f"{prefix}in_proj.weight"],
+ [
+ self.qk_dim_local_tp,
+ self.qk_dim_local_tp,
+ self.v_dim_local_tp,
+ self.v_dim_local_tp,
+ self.num_value_heads // self.tp_size,
+ self.num_value_heads // self.tp_size,
+ ],
+ ["query", "key", "value", "z", "beta", "alpha"],
+ 0,
+ )
+
+ conv_layer_name_list = ["conv1d.weight"]
+ assert (
+ sharded_state_dict[f"{prefix}conv1d.weight"].data.size(0) == self.conv_dim_local_tp
+ ), (self.conv_dim_local_tp, sharded_state_dict[f"{prefix}conv1d.weight"])
+ if self.conv_bias:
+ conv_layer_name_list.append("conv1d.bias")
+ assert (
+ sharded_state_dict[f"{prefix}conv1d.bias"].data.size(0) == self.conv_dim_local_tp
+ ), (self.conv_dim_local_tp, sharded_state_dict[f"{prefix}conv1d.bias"])
+ for conv_layer_name in conv_layer_name_list:
+ sharded_state_dict[f"{prefix}{conv_layer_name}"] = _split_tensor_factory(
+ sharded_state_dict[f"{prefix}{conv_layer_name}"],
+ [self.qk_dim_local_tp, self.qk_dim_local_tp, self.v_dim_local_tp],
+ ["query", "key", "value"],
+ 0,
+ )
+
+ return sharded_state_dict
+
+
+####################
+# Sharded state dict utilities
+####################
+def _split_tensor_factory(
+ orig_sh_ten: ShardedTensor, split_sections: List[int], split_names: List[str], split_dim: int
+) -> ShardedTensorFactory:
+ """Builds a factory that splits a given ShardedTensor into several independent chunks."""
+ assert isinstance(orig_sh_ten, ShardedTensor), type(orig_sh_ten)
+ orig_sh_ten_no_data = orig_sh_ten.without_data() # remove `data` reference
+
+ if sum(split_sections) != orig_sh_ten_no_data.local_shape[split_dim]:
+ raise ValueError(
+ f"Split sections must cover the whole dimension size, "
+ f"got {split_sections=} vs dimensions size "
+ f"{orig_sh_ten_no_data.local_shape[split_dim]}"
+ )
+
+ assert not isinstance(
+ split_sections, int
+ ), "Splitting into predefined section sizes is supported (`split_sections` must be a list)"
+ assert len(split_sections) == len(split_names), (len(split_sections), len(split_names))
+
+ @torch.no_grad()
+ def sh_ten_build_fn(
+ key: str, t: torch.Tensor, replica_id: ReplicaId, flattened_range: Optional[slice]
+ ):
+ factory_sh_ten = replace(
+ orig_sh_ten_no_data,
+ key=key,
+ data=t,
+ dtype=t.dtype,
+ replica_id=replica_id,
+ flattened_range=flattened_range,
+ )
+
+ chunk_sh_tens = []
+ split_start = 0
+ for split_size, split_name in zip(split_sections, split_names):
+ split_chunks = factory_sh_ten.narrow(split_dim, split_start, split_size)
+ for sh_ten in split_chunks:
+ sh_ten.key = f"{sh_ten.key}.{split_name}"
+ chunk_sh_tens.extend(split_chunks)
+ split_start += split_size
+
+ assert split_start == orig_sh_ten_no_data.local_shape[split_dim], (
+ split_start,
+ orig_sh_ten_no_data.local_shape[split_dim],
+ )
+ assert sum(sh_ten.data.numel() for sh_ten in chunk_sh_tens) == t.numel(), (
+ chunk_sh_tens,
+ t.shape,
+ )
+ return chunk_sh_tens
+
+ @torch.no_grad()
+ def sh_ten_merge_fn(sub_state_dict):
+ return torch.cat(sub_state_dict)
+
+ return ShardedTensorFactory(
+ orig_sh_ten.key, orig_sh_ten.data, sh_ten_build_fn, sh_ten_merge_fn, orig_sh_ten.replica_id
+ )
+
+
+####################
+# Context parallel utilities
+####################
+def get_parameter_local_cp(
+ param: torch.Tensor,
+ dim: int,
+ cp_group: torch.distributed.ProcessGroup,
+ split_sections: Optional[List[int]] = None,
+) -> torch.Tensor:
+ """Get the local parameter for the current context parallel rank.
+
+ Args:
+ param (torch.Tensor): The entire parameter to get the local parameter for.
+ dim (int): The dimension to split the parameter along. Usually the dimension of head.
+ cp_group (torch.distributed.ProcessGroup): The context parallel group.
+ split_sections (Optional[List[int]]): If not None,
+ first split the parameter along the dimension dim into sections,
+ then get the local hidden parallel weights separately,
+ finally concatenate the local hidden parallel weights along the dimension dim.
+
+ Returns:
+ torch.Tensor: The local parameter for the current context parallel rank.
+ """
+
+ cp_size = cp_group.size()
+ cp_rank = cp_group.rank()
+
+ # No need to split if CP size is 1.
+ if cp_size == 1:
+ return param
+
+ # Split first if needed.
+ if split_sections is not None:
+ inputs = torch.split(param, split_sections, dim=dim)
+ outputs = []
+ for p in inputs:
+ p = get_parameter_local_cp(p, dim, cp_group)
+ outputs.append(p)
+ return torch.cat(outputs, dim=dim)
+
+ # Slice the parameter.
+ slices = [slice(None)] * param.dim()
+ dim_size = param.size(dim=dim)
+ slices[dim] = slice(cp_rank * dim_size // cp_size, (cp_rank + 1) * dim_size // cp_size)
+ param = param[slices]
+ return param
+
+
+def tensor_a2a_cp2hp(
+ tensor: torch.Tensor,
+ seq_dim: int,
+ head_dim: int,
+ cp_group: torch.distributed.ProcessGroup,
+ split_sections: Optional[List[int]] = None,
+ undo_attention_load_balancing: bool = True,
+):
+ """All-to-all context parallel to hidden parallel.
+
+ Args:
+ tensor (torch.Tensor): The tensor to all-to-all.
+ Currently only support (seq_len, batch, head_dim) shaped tensor.
+ seq_dim (int): The dimension of sequence length. Currently only supports seq_dim == 0.
+ head_dim (int): The dimension of head. Currently only supports head_dim == -1 or 2.
+ cp_group (torch.distributed.ProcessGroup): The context parallel group.
+ split_sections (Optional[List[int]]): If not None, split the tensor along the dimension
+ head_dim into sections first, then do all-to-all for each section separately,
+ finally concatenate the separated tensors along the dimension head_dim.
+ undo_attention_load_balancing (bool): Whether to undo the attention load balancing of CP.
+
+ Returns:
+ torch.Tensor: The all-to-all tensor.
+ """
+
+ cp_size = cp_group.size()
+
+ # No need to all-to-all if CP size is 1.
+ if cp_size == 1:
+ return tensor
+
+ # Limitations of mamba_context_parallel._all_to_all_cp2hp.
+ assert seq_dim == 0, f"tensor_a2a_cp2hp only supports seq_dim == 0 for now, but got {seq_dim=}"
+ assert (
+ head_dim == -1 or head_dim == 2
+ ), f"tensor_a2a_cp2hp only supports head_dim == -1 or 2 for now, but got {head_dim=}"
+ assert (
+ tensor.dim() == 3
+ ), f"tensor_a2a_cp2hp only supports 3-d input tensor for now, but got {tensor.dim()=}"
+
+ # Split first if needed.
+ if split_sections is not None:
+ inputs = torch.split(tensor, split_sections, dim=head_dim)
+ outputs = []
+ for x in inputs:
+ x = tensor_a2a_cp2hp(
+ x,
+ seq_dim=seq_dim,
+ head_dim=head_dim,
+ cp_group=cp_group,
+ undo_attention_load_balancing=False,
+ )
+ outputs.append(x)
+ tensor = torch.cat(outputs, dim=head_dim)
+ else:
+ tensor = _all_to_all_cp2hp(tensor, cp_group)
+
+ # Undo attention load balancing last if needed.
+ if undo_attention_load_balancing:
+ tensor = _undo_attention_load_balancing(tensor, cp_size)
+ return tensor
+
+
+def tensor_a2a_hp2cp(
+ tensor: torch.Tensor,
+ seq_dim: int,
+ head_dim: int,
+ cp_group: torch.distributed.ProcessGroup,
+ split_sections: Optional[List[int]] = None,
+ redo_attention_load_balancing: bool = True,
+):
+ """All-to-all hidden parallel to context parallel.
+
+ Args:
+ tensor (torch.Tensor): The tensor to all-to-all.
+ Currently only support (seq_len, batch, head_dim) shaped tensor.
+ seq_dim (int): The dimension of sequence length. Currently only supports seq_dim == 0.
+ head_dim (int): The dimension of head. Currently only supports head_dim == -1 or 2.
+ cp_group (torch.distributed.ProcessGroup): The context parallel group.
+ split_sections (Optional[List[int]]): If not None, first split the tensor along the
+ dimension head_dim into sections, then do all-to-all for each section separately,
+ finally concatenate the separated tensors along the dimension head_dim.
+ redo_attention_load_balancing (bool): Whether to redo the attention load balancing of HP.
+
+ Returns:
+ torch.Tensor: The all-to-all tensor.
+ """
+
+ cp_size = cp_group.size()
+
+ # No need to all-to-all if CP size is 1.
+ if cp_size == 1:
+ return tensor
+
+ # Limitations of mamba_context_parallel._all_to_all_hp2cp.
+ assert seq_dim == 0, f"tensor_a2a_cp2hp only supports seq_dim == 0 for now, but got {seq_dim=}"
+ assert (
+ head_dim == -1 or head_dim == 2
+ ), f"tensor_a2a_cp2hp only supports head_dim == -1 or 2 for now, but got {head_dim=}"
+ assert (
+ tensor.dim() == 3
+ ), f"tensor_a2a_cp2hp only supports 3-d input tensor for now, but got {tensor.dim()=}"
+
+ # Redo attention load balancing first if needed.
+ if redo_attention_load_balancing:
+ tensor = _redo_attention_load_balancing(tensor, cp_size)
+
+ # Split first if needed.
+ if split_sections is not None:
+ inputs = torch.split(tensor, split_sections, dim=head_dim)
+ outputs = []
+ for x in inputs:
+ x = tensor_a2a_hp2cp(
+ x,
+ seq_dim=seq_dim,
+ head_dim=head_dim,
+ cp_group=cp_group,
+ redo_attention_load_balancing=False,
+ )
+ outputs.append(x)
+ tensor = torch.cat(outputs, dim=head_dim)
+ else:
+ tensor = _all_to_all_hp2cp(tensor, cp_group)
+
+ return tensor
+
+
+####################
+# Torch native gated delta rule
+####################
+def torch_chunk_gated_delta_rule(
+ query,
+ key,
+ value,
+ g,
+ beta,
+ chunk_size=64,
+ initial_state=None,
+ output_final_state=False,
+ use_qk_l2norm_in_kernel=False,
+):
+ # pylint: disable=line-too-long
+ '''
+ Torch-native implementation of chunked gated delta rule for deterministic mode.
+ Need this because FLA is not deterministic.
+
+ Reference: https://github.com/huggingface/transformers/blob/144c8ce2809a2e21914017652700e1ecb450501e/src/transformers/models/qwen3_next/modeling_qwen3_next.py#L470-L547
+ '''
+
+ initial_dtype = query.dtype
+ if use_qk_l2norm_in_kernel:
+ query = l2norm(query, dim=-1, eps=1e-6)
+ key = l2norm(key, dim=-1, eps=1e-6)
+ query, key, value, beta, g = [
+ x.transpose(1, 2).contiguous().to(torch.float32) for x in (query, key, value, beta, g)
+ ]
+
+ batch_size, num_heads, sequence_length, k_head_dim = key.shape
+ v_head_dim = value.shape[-1]
+ pad_size = (chunk_size - sequence_length % chunk_size) % chunk_size
+ query = F.pad(query, (0, 0, 0, pad_size))
+ key = F.pad(key, (0, 0, 0, pad_size))
+ value = F.pad(value, (0, 0, 0, pad_size))
+ beta = F.pad(beta, (0, pad_size))
+ g = F.pad(g, (0, pad_size))
+ total_sequence_length = sequence_length + pad_size
+ scale = 1 / (query.shape[-1] ** 0.5)
+ query = query * scale
+
+ v_beta = value * beta.unsqueeze(-1)
+ k_beta = key * beta.unsqueeze(-1)
+ # reshape to chunks
+ query, key, value, k_beta, v_beta = [
+ x.reshape(x.shape[0], x.shape[1], -1, chunk_size, x.shape[-1])
+ for x in (query, key, value, k_beta, v_beta)
+ ]
+ g = g.reshape(g.shape[0], g.shape[1], -1, chunk_size)
+ mask = torch.triu(
+ torch.ones(chunk_size, chunk_size, dtype=torch.bool, device=query.device), diagonal=0
+ )
+
+ # chunk decay
+ g = g.cumsum(dim=-1)
+ decay_mask = ((g.unsqueeze(-1) - g.unsqueeze(-2)).tril().exp().float()).tril()
+ attn = -((k_beta @ key.transpose(-1, -2)) * decay_mask).masked_fill(mask, 0)
+ for i in range(1, chunk_size):
+ row = attn[..., i, :i].clone()
+ sub = attn[..., :i, :i].clone()
+ attn[..., i, :i] = row + (row.unsqueeze(-1) * sub).sum(-2)
+ attn = attn + torch.eye(chunk_size, dtype=attn.dtype, device=attn.device)
+ value = attn @ v_beta
+ k_cumdecay = attn @ (k_beta * g.exp().unsqueeze(-1))
+ last_recurrent_state = (
+ torch.zeros(batch_size, num_heads, k_head_dim, v_head_dim).to(value)
+ if initial_state is None
+ else initial_state.to(value)
+ )
+ core_attn_out = torch.zeros_like(value)
+ mask = torch.triu(
+ torch.ones(chunk_size, chunk_size, dtype=torch.bool, device=query.device), diagonal=1
+ )
+
+ # for each chunk
+ for i in range(0, total_sequence_length // chunk_size):
+ q_i, k_i, v_i = query[:, :, i], key[:, :, i], value[:, :, i]
+ attn = (q_i @ k_i.transpose(-1, -2) * decay_mask[:, :, i]).masked_fill_(mask, 0)
+ v_prime = (k_cumdecay[:, :, i]) @ last_recurrent_state
+ v_new = v_i - v_prime
+ attn_inter = (q_i * g[:, :, i, :, None].exp()) @ last_recurrent_state
+ core_attn_out[:, :, i] = attn_inter + attn @ v_new
+ last_recurrent_state = (
+ last_recurrent_state * g[:, :, i, -1, None, None].exp()
+ + (k_i * (g[:, :, i, -1, None] - g[:, :, i]).exp()[..., None]).transpose(-1, -2) @ v_new
+ )
+
+ if not output_final_state:
+ last_recurrent_state = None
+ core_attn_out = core_attn_out.reshape(
+ core_attn_out.shape[0], core_attn_out.shape[1], -1, core_attn_out.shape[-1]
+ )
+ core_attn_out = core_attn_out[:, :, :sequence_length]
+ core_attn_out = core_attn_out.transpose(1, 2).contiguous().to(initial_dtype)
+ return core_attn_out, last_recurrent_state
diff --git a/megatron/core/ssm/mamba_block.py b/megatron/core/ssm/mamba_block.py
index 1abd12186e3..29e9b123674 100644
--- a/megatron/core/ssm/mamba_block.py
+++ b/megatron/core/ssm/mamba_block.py
@@ -1,4 +1,4 @@
-# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved.
+# Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# Copyright (c) 2024, Tri Dao, Albert Gu.
# Some of this code was adopted from https://github.com/state-spaces/mamba/
@@ -22,6 +22,7 @@
from megatron.core.ssm.mamba_hybrid_layer_allocation import Symbols as LayerSymbols
from megatron.core.ssm.mamba_hybrid_layer_allocation import allocate_layers
from megatron.core.transformer import TransformerConfig
+from megatron.core.transformer.enums import CudaGraphScope
from megatron.core.transformer.identity_op import IdentityOp
from megatron.core.transformer.module import MegatronModule
from megatron.core.transformer.spec_utils import ModuleSpec, build_module
@@ -245,7 +246,7 @@ def forward(
(
(
self.config.cuda_graph_impl == "local"
- and self.config.cuda_graph_scope != "full_iteration"
+ and CudaGraphScope.full_iteration not in self.config.cuda_graph_scope
)
or self.config.flash_decode
)
diff --git a/megatron/core/ssm/mamba_mixer.py b/megatron/core/ssm/mamba_mixer.py
index c4b52872c2b..c9ccf826ad0 100644
--- a/megatron/core/ssm/mamba_mixer.py
+++ b/megatron/core/ssm/mamba_mixer.py
@@ -309,18 +309,26 @@ def __init__(
self.act = nn.SiLU()
with get_cuda_rng_tracker().fork():
- # Initialize dt bias so that F.softplus(dt_bias) is between dt_min and dt_max
- dt = torch.exp(
- torch.rand(
+ if self.config.perform_initialization:
+ # Initialize dt bias so that F.softplus(dt_bias) is between dt_min and dt_max
+ dt = torch.exp(
+ torch.rand(
+ self.nheads_local_tp,
+ device=torch.cuda.current_device(),
+ dtype=config.params_dtype,
+ )
+ * (math.log(dt_max) - math.log(dt_min))
+ + math.log(dt_min)
+ ).clamp(min=dt_init_floor)
+ # Inverse of softplus: https://github.com/pytorch/pytorch/issues/72759
+ inv_dt = dt + torch.log(-torch.expm1(-dt))
+ else:
+ inv_dt = torch.empty(
self.nheads_local_tp,
device=torch.cuda.current_device(),
dtype=config.params_dtype,
)
- * (math.log(dt_max) - math.log(dt_min))
- + math.log(dt_min)
- ).clamp(min=dt_init_floor)
- # Inverse of softplus: https://github.com/pytorch/pytorch/issues/72759
- inv_dt = dt + torch.log(-torch.expm1(-dt))
+
self.dt_bias = nn.Parameter(inv_dt)
setattr(self.dt_bias, "tensor_model_parallel", True)
diff --git a/megatron/core/tensor_parallel/__init__.py b/megatron/core/tensor_parallel/__init__.py
index 98cc5efec82..2140ee54b37 100644
--- a/megatron/core/tensor_parallel/__init__.py
+++ b/megatron/core/tensor_parallel/__init__.py
@@ -29,9 +29,11 @@
from .random import (
CheckpointWithoutOutput,
checkpoint,
+ convert_cuda_rng_state,
get_cuda_rng_tracker,
get_data_parallel_rng_tracker_name,
get_expert_parallel_rng_tracker_name,
+ is_graph_safe_cuda_rng_tracker,
model_parallel_cuda_manual_seed,
)
from .utils import (
@@ -64,9 +66,11 @@
"scatter_to_sequence_parallel_region",
# random.py
"checkpoint",
+ "convert_cuda_rng_state",
"get_cuda_rng_tracker",
"model_parallel_cuda_manual_seed",
"get_expert_parallel_rng_tracker_name",
+ "is_graph_safe_cuda_rng_tracker",
"CheckpointWithoutOutput",
# utils.py
"split_tensor_along_last_dim",
diff --git a/megatron/core/tensor_parallel/layers.py b/megatron/core/tensor_parallel/layers.py
index 13acddebc79..d3ec11aaf5c 100644
--- a/megatron/core/tensor_parallel/layers.py
+++ b/megatron/core/tensor_parallel/layers.py
@@ -1,4 +1,4 @@
-# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved.
+# Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# Parts of the code here are adapted from PyTorch
# repo: https://github.com/pytorch/pytorch
@@ -56,6 +56,8 @@
HAVE_TE = False
_MODEL_PARALLEL_ATTRIBUTE_DEFAULTS = {
+ "expert_tp": False,
+ "is_qkv": False,
"tensor_model_parallel": False,
"partition_dim": -1,
"partition_stride": 1,
@@ -84,12 +86,16 @@
dist_reduce_scatter_func = torch.distributed._reduce_scatter_base
-def param_is_not_tensor_parallel_duplicate(param):
+def param_is_not_tensor_parallel_duplicate(param, tp_group=None):
"""Returns true if the passed-in parameter is not a duplicate parameter
on another TP rank."""
- return (hasattr(param, "tensor_model_parallel") and param.tensor_model_parallel) or (
- get_tensor_model_parallel_rank() == 0
- )
+ if hasattr(param, "tensor_model_parallel") and param.tensor_model_parallel:
+ return True
+ # Prefer provided tp_group when available (new explicit path).
+ if tp_group is not None:
+ return tp_group.rank() == 0
+ # Fallback to legacy global state (back-compat).
+ return get_tensor_model_parallel_rank() == 0
def set_tensor_model_parallel_attributes(tensor, is_parallel, dim, stride):
@@ -247,6 +253,10 @@ def __init__(
rank=get_pg_rank(self.tp_group),
world_size=get_pg_size(self.tp_group),
)
+ else:
+ set_tensor_model_parallel_attributes(
+ tensor=self.weight, is_parallel=True, dim=0, stride=1
+ )
else:
self.weight = Parameter(
torch.empty(
@@ -258,6 +268,10 @@ def __init__(
)
if config.perform_initialization:
_initialize_affine_weight_gpu(self.weight, init_method, partition_dim=0, stride=1)
+ else:
+ set_tensor_model_parallel_attributes(
+ tensor=self.weight, is_parallel=True, dim=0, stride=1
+ )
def forward(self, input_):
"""Forward.
@@ -859,6 +873,10 @@ def __init__(
rank=rank,
world_size=world_size,
)
+ else:
+ set_tensor_model_parallel_attributes(
+ tensor=self.weight, is_parallel=True, dim=0, stride=stride
+ )
else:
self.weight = Parameter(
torch.empty(
@@ -876,6 +894,10 @@ def __init__(
stride=stride,
is_expert=self.is_expert,
)
+ else:
+ set_tensor_model_parallel_attributes(
+ tensor=self.weight, is_parallel=True, dim=0, stride=stride
+ )
setattr(self.weight, "allreduce", not (self.is_expert and self.expert_parallel))
else:
@@ -1176,6 +1198,10 @@ def __init__(
rank=rank,
world_size=world_size,
)
+ else:
+ set_tensor_model_parallel_attributes(
+ tensor=self.weight, is_parallel=True, dim=1, stride=stride
+ )
else:
self.weight = Parameter(
torch.empty(
@@ -1193,6 +1219,10 @@ def __init__(
stride=stride,
is_expert=self.is_expert,
)
+ else:
+ set_tensor_model_parallel_attributes(
+ tensor=self.weight, is_parallel=True, dim=1, stride=stride
+ )
setattr(self.weight, "allreduce", not (self.is_expert and self.expert_parallel))
if bias:
diff --git a/megatron/core/tensor_parallel/random.py b/megatron/core/tensor_parallel/random.py
index 54cac0e41e3..5d5389a52d2 100644
--- a/megatron/core/tensor_parallel/random.py
+++ b/megatron/core/tensor_parallel/random.py
@@ -1,4 +1,4 @@
-# Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved.
+# Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# Parts of the code here are adapted from PyTorch
# repo: https://github.com/pytorch/pytorch
@@ -111,6 +111,41 @@ def cb():
_lazy_call(cb)
+def convert_cuda_rng_state(
+ state: Union[torch.Tensor, torch.Generator], to_graphable: bool = False
+) -> Union[torch.Tensor, torch.Generator]:
+ """
+ Convert the cuda rng state tensor to the graphable version,
+ or from the graphable version to the non-graphable tensor version.
+ """
+ if to_graphable:
+ if isinstance(state, torch.Tensor):
+ # Convert to the graphable version.
+ # Store current rng state.
+ orig_cuda_rng_state = _get_cuda_rng_state(graph_safe=False)
+ # Set rng state to the desired one
+ _set_cuda_rng_state(state, graph_safe=False)
+ # Get the graphable state
+ graphable_state = _get_cuda_rng_state(clone=True, graph_safe=True)
+ # And set the state to the original state we started with.
+ _set_cuda_rng_state(orig_cuda_rng_state, graph_safe=False)
+ return graphable_state
+ elif isinstance(state, torch.Generator):
+ # already graphable, just return it.
+ return state
+ else:
+ raise ValueError(f"Invalid state type: {type(state)}")
+ else:
+ if isinstance(state, torch.Tensor):
+ # already non-graphable, just return it.
+ return state
+ elif isinstance(state, torch.Generator):
+ # Convert to the non-graphable tensor version.
+ return state.get_state()
+ else:
+ raise ValueError(f"Invalid state type: {type(state)}")
+
+
def get_expert_parallel_rng_tracker_name():
"""Get the expert parallel rng tracker name"""
global _EXPERT_PARALLEL_RNG_TRACKER_NAME
@@ -161,6 +196,10 @@ def reset(self):
# Seeds are just for book keeping and ensure no seed is set twice.
self.seeds_ = set()
+ # Name of the rng state currently being used in the generator.
+ # The default one is "default-rng" and won't be pushed to the self.states_ dictionary.
+ self._current_state_name = "default-rng"
+
def get_states(self):
"""Get rng states. Copy the dictionary so we have direct
pointers to the states, not just a pointer to the dictionary."""
@@ -207,10 +246,14 @@ def fork(self, name=_MODEL_PARALLEL_RNG_TRACKER_NAME):
# Check if we have added the state
if name not in self.states_:
raise Exception('cuda rng state {} is not added'.format(name))
- # Store current rng state.
+ # Store current rng state and name. Store in self.states_ if it's not the default state.
orig_cuda_rng_state = _get_cuda_rng_state(graph_safe=self.use_cudagraphable_rng)
- # Set rng state to the desired one
+ orig_state_name = self._current_state_name
+ if orig_state_name != "default-rng":
+ self.states_[orig_state_name] = orig_cuda_rng_state
+ # Set rng state and name to the desired one.
_set_cuda_rng_state(self.states_[name], graph_safe=self.use_cudagraphable_rng)
+ self._current_state_name = name
# Record cpu RNG state
cpu_rng_state = torch.get_rng_state()
# Do the stuff we wanted to do.
@@ -220,10 +263,19 @@ def fork(self, name=_MODEL_PARALLEL_RNG_TRACKER_NAME):
# Throw a warning if cpu RNG state changed
if not torch.all(cpu_rng_state == torch.get_rng_state()).item():
logging.getLogger(__name__).warning('CPU RNG state changed within GPU RNG context')
+ # Check if the current state name is the same as the desired state name.
+ if self._current_state_name != name:
+ raise Exception(
+ f'current state name {self._current_state_name} is not the same as the desired '
+ f'state name {name}.'
+ )
# Update the current rng state for later use.
self.states_[name] = _get_cuda_rng_state(graph_safe=self.use_cudagraphable_rng)
- # And set the state to the original state we started with.
+ # And set the state and name to the original state we started with.
+ if orig_state_name != "default-rng":
+ orig_cuda_rng_state = self.states_[orig_state_name]
_set_cuda_rng_state(orig_cuda_rng_state, graph_safe=self.use_cudagraphable_rng)
+ self._current_state_name = orig_state_name
# RNG tracker object.
@@ -377,10 +429,24 @@ def model_parallel_cuda_manual_seed(
_CUDA_RNG_STATE_TRACKER.add(_EXPERT_PARALLEL_RNG_TRACKER_NAME, expert_parallel_seed)
+def is_graph_safe_cuda_rng_tracker(cuda_rng_tracker):
+ """Check if the cuda rng tracker is graph safe version."""
+ if HAVE_TE and is_te_min_version("1.5.0"):
+ from megatron.core.extensions.transformer_engine import TECudaRNGStatesTracker
+
+ if isinstance(cuda_rng_tracker, TECudaRNGStatesTracker):
+ return True
+ if getattr(cuda_rng_tracker, "use_cudagraphable_rng", False):
+ return True
+ return False
+
+
def _get_all_rng_states():
"""Get all the rng states."""
cpu_rng_state = torch.get_rng_state()
- cuda_rng_state = _get_cuda_rng_state()
+ cuda_rng_state = _get_cuda_rng_state(
+ graph_safe=is_graph_safe_cuda_rng_tracker(get_cuda_rng_tracker())
+ )
cuda_rng_state_tracker = get_cuda_rng_tracker().get_states()
return cpu_rng_state, cuda_rng_state, cuda_rng_state_tracker
@@ -388,7 +454,9 @@ def _get_all_rng_states():
def _set_all_rng_states(cpu_rng_state, cuda_rng_state, cuda_rng_state_tracker):
"""Set all the rng states."""
torch.set_rng_state(cpu_rng_state)
- _set_cuda_rng_state(cuda_rng_state)
+ _set_cuda_rng_state(
+ cuda_rng_state, graph_safe=is_graph_safe_cuda_rng_tracker(get_cuda_rng_tracker())
+ )
get_cuda_rng_tracker().set_states(cuda_rng_state_tracker)
@@ -510,10 +578,14 @@ def forward(ctx, run_function, checkpoint_without_output_obj, *args):
@staticmethod
def backward(ctx, *args):
"""Backward pass."""
- inputs = ctx.saved_tensors
+ # Get the inputs from the context instead of the saved tensors
+ # because the saved tensors are already cached by the recomputation.
+ # This is to avoid double-reloading the inputs in CPU offloading scenario.
+ inputs = ctx.inputs
outputs = ctx.outputs
torch.autograd.backward(outputs, args)
ctx.outputs = None
+ ctx.inputs = None
grads = tuple(inp.grad if isinstance(inp, torch.Tensor) else inp for inp in inputs)
return (None, None) + grads
@@ -555,6 +627,11 @@ def checkpoint(self, run_function, *args):
def _recompute(self, _):
"""Used as a hook to recompute the output."""
+
+ if self.ctx is None:
+ # The recomputation has been triggered already. Just return.
+ return
+
if not torch.autograd._is_checkpoint_valid():
raise RuntimeError(
"Checkpointing is not compatible with .grad(), "
@@ -573,8 +650,19 @@ def _recompute(self, _):
recompute_ctx = contextlib.nullcontext()
fp8_ctx = contextlib.nullcontext()
+ # Store the inputs for backward pass
+ inputs = self.ctx.saved_tensors
+
+ def detach(t):
+ if isinstance(t, torch.Tensor):
+ requires_grad = t.requires_grad
+ t = t.detach()
+ t.requires_grad_(requires_grad)
+ return t
+
+ inputs = tuple(detach(t) for t in inputs)
with torch.enable_grad(), fp8_ctx, recompute_ctx:
- outputs = self.run_function(*self.ctx.saved_tensors)
+ outputs = self.run_function(*inputs)
self.run_function = None
self.rng_states = None
@@ -590,6 +678,7 @@ def _recompute(self, _):
output.untyped_storage().copy_(recomputation_output.untyped_storage())
self.ctx.outputs = outputs
+ self.ctx.inputs = inputs
self.outputs = None
self.ctx = None
diff --git a/megatron/core/transformer/attention.py b/megatron/core/transformer/attention.py
index 2f26edb8c30..c3c7dad250a 100644
--- a/megatron/core/transformer/attention.py
+++ b/megatron/core/transformer/attention.py
@@ -1,4 +1,5 @@
-# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved.
+# Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+
import copy
from abc import ABC, abstractmethod
from dataclasses import dataclass
@@ -9,6 +10,7 @@
from megatron.core import tensor_parallel
from megatron.core.inference.contexts import BaseInferenceContext
+from megatron.core.jit import jit_fuser
from megatron.core.models.common.embeddings.rope_utils import (
apply_rotary_pos_emb,
apply_rotary_pos_emb_with_cos_sin,
@@ -22,6 +24,9 @@
get_tensor_model_parallel_rank,
get_tensor_model_parallel_world_size,
)
+from megatron.core.pipeline_parallel.fine_grained_activation_offload import (
+ FineGrainedActivationOffloadingInterface as off_interface,
+)
from megatron.core.process_groups_config import ProcessGroupCollection
from megatron.core.tensor_parallel.mappings import all_gather_last_dim_from_tensor_parallel_region
from megatron.core.transformer.identity_op import IdentityOp
@@ -41,7 +46,7 @@
from ..models.common.embeddings.yarn_rotary_pos_embedding import (
_yarn_get_concentration_factor_from_config,
)
-from .enums import AttnMaskType
+from .enums import AttnMaskType, CudaGraphScope
from .transformer_config import TransformerConfig
try:
@@ -202,6 +207,7 @@ def __init__(
self.key_hidden_size = self.hidden_size_per_attention_head
self.val_hidden_size = self.hidden_size_per_attention_head
+ # TODO: This is built twice when using MLA, should be refactored.
if self.config.num_query_groups < world_size:
# TE throws an assertion error if num_kv_heads / num_query_groups
# is not divisible by TP size.
@@ -226,6 +232,21 @@ def __init__(
and "core_attn" in self.config.recompute_modules
)
+ self.offload_qkv_linear = (
+ self.config.fine_grained_activation_offloading
+ and "qkv_linear" in self.config.offload_modules
+ )
+
+ self.offload_core_attention = (
+ self.config.fine_grained_activation_offloading
+ and "core_attn" in self.config.offload_modules
+ )
+
+ self.offload_attn_proj = (
+ self.config.fine_grained_activation_offloading
+ and "attn_proj" in self.config.offload_modules
+ )
+
# Output.
self.linear_proj = build_module(
submodules.linear_proj,
@@ -504,7 +525,9 @@ def _adjust_key_value_for_inference(
return query, key, value, rotary_pos_emb, attn_mask_type, block_table
@abstractmethod
- def get_query_key_value_tensors(self, hidden_states, key_value_states, split_qkv=True):
+ def get_query_key_value_tensors(
+ self, hidden_states, key_value_states, output_gate, split_qkv=True
+ ):
"""
This method needs to be implemented based on whether the derived class
is "self-attn" or "cross-attn".
@@ -795,19 +818,32 @@ def forward(
self.k_layernorm is None or isinstance(self.k_layernorm, IdentityOp),
]
)
+ output_gate = self.config.attention_output_gate
# Check if fused_single_qkv_rope is requested but either unavailable or not
# supported for the current use case.
if self.attention_type != "cross":
assert not (
self.config.fused_single_qkv_rope and split_qkv
), "fused_single_qkv_rope requested but not available/supported for the config."
+ if output_gate:
+ assert split_qkv, "output_gate is not supported for unsplit mixed_qkv tensor."
+
+ with off_interface(self.offload_qkv_linear, hidden_states, "qkv_linear") as hidden_states:
+ qkv_output = self.get_query_key_value_tensors(
+ hidden_states, key_value_states, output_gate=output_gate, split_qkv=split_qkv
+ )
+ if self.offload_qkv_linear:
+ # `qkv_output` may be a tuple; commit supports tuple/list and will keep structure.
+ qkv_output = off_interface.group_commit(
+ qkv_output, name="qkv_linear", forced_released_tensors=[]
+ )
- qkv_output = self.get_query_key_value_tensors(
- hidden_states, key_value_states, split_qkv=split_qkv
- )
attn_mask_type = self.attn_mask_type
block_table = None
- if split_qkv:
+ gate = None
+ if output_gate and split_qkv:
+ query, key, value, gate = qkv_output
+ elif split_qkv:
query, key, value = qkv_output
else:
mixed_qkv, qkv_split_arg_list = qkv_output
@@ -851,7 +887,7 @@ def forward(
if (
in_decode_mode
and self.config.cuda_graph_impl == "local"
- and self.config.cuda_graph_scope != "full_iteration"
+ and CudaGraphScope.full_iteration not in self.config.cuda_graph_scope
and inference_context.is_static_batching()
):
raise ValueError(f"CUDA graphs must use flash decode with static batching!")
@@ -871,7 +907,7 @@ def forward(
)
)
- if packed_seq_params is not None:
+ if packed_seq_params is not None and packed_seq_params.qkv_format == 'thd':
query = query.squeeze(1)
key = key.squeeze(1)
value = value.squeeze(1)
@@ -886,7 +922,7 @@ def forward(
):
q_pos_emb, k_pos_emb = rotary_pos_emb
- if packed_seq_params is not None:
+ if packed_seq_params is not None and packed_seq_params.qkv_format == 'thd':
if packed_seq_params.cu_seqlens_q_padded is not None:
cu_seqlens_q = packed_seq_params.cu_seqlens_q_padded
else:
@@ -952,15 +988,18 @@ def forward(
else:
if inference_context is None or inference_context.is_static_batching():
# Static batching attention kernel.
- core_attn_out = self.core_attention(
- query,
- key,
- value,
- attention_mask,
- attn_mask_type=attn_mask_type,
- attention_bias=attention_bias,
- packed_seq_params=packed_seq_params,
- )
+ with off_interface(
+ self.offload_core_attention and self.training, query, "core_attn"
+ ) as query:
+ core_attn_out = self.core_attention(
+ query,
+ key,
+ value,
+ attention_mask,
+ attn_mask_type=attn_mask_type,
+ attention_bias=attention_bias,
+ packed_seq_params=packed_seq_params,
+ )
else:
# Dynamic batching attention kernel.
@@ -980,6 +1019,10 @@ def forward(
block_table,
)
core_attn_out = rearrange(core_attn_out, 's b h d -> s b (h d)')
+ if self.offload_core_attention and self.training:
+ core_attn_out = off_interface.group_commit(
+ core_attn_out, name="core_attn", forced_released_tensors=[query, key, value]
+ )
if packed_seq_params is not None and packed_seq_params.qkv_format == 'thd':
# reshape to same output shape as unpacked case
@@ -989,16 +1032,36 @@ def forward(
core_attn_out = core_attn_out.reshape(core_attn_out.size(0), 1, -1)
nvtx_range_pop(suffix="core_attention")
+ # Output gate
+ if gate is not None:
+ nvtx_range_push(suffix="output_gate")
+ core_attn_out = self._apply_output_gate(core_attn_out, gate)
+ nvtx_range_pop(suffix="output_gate")
+
# =================
# Output. [sq, b, h]
# =================
nvtx_range_push(suffix="linear_proj")
- output, bias = self.linear_proj(core_attn_out)
+ with off_interface(self.offload_attn_proj, core_attn_out, "attn_proj") as core_attn_out:
+ output, bias = self.linear_proj(core_attn_out)
+ if self.offload_attn_proj:
+ output = off_interface.group_commit(
+ output, name="attn_proj", forced_released_tensors=[core_attn_out]
+ )
nvtx_range_pop(suffix="linear_proj")
return output, bias
+ @jit_fuser
+ def _apply_output_gate(self, x, gate):
+ x_dtype = x.dtype
+ gate = gate.contiguous()
+ gate = gate.view(*x.shape)
+ x = x * torch.sigmoid(gate.float())
+ x = x.to(x_dtype)
+ return x
+
def set_for_recompute_input_layernorm(self):
"""Set the attention layer for recompute input_layernorm. Only needed for fp8."""
raise NotImplementedError("set_for_recompute_input_layernorm is not implemented.")
@@ -1037,10 +1100,13 @@ def __init__(
pg_collection=pg_collection,
)
+ self.linear_qkv_out_dim = self.query_projection_size + 2 * self.kv_projection_size
+ if self.config.attention_output_gate:
+ self.linear_qkv_out_dim += self.config.kv_channels * self.config.num_attention_heads
self.linear_qkv = build_module(
submodules.linear_qkv,
self.config.hidden_size,
- self.query_projection_size + 2 * self.kv_projection_size,
+ self.linear_qkv_out_dim,
config=self.config,
init_method=self.config.init_method,
gather_output=False,
@@ -1142,14 +1208,26 @@ def _compare(srcs, tgts, names, parallelism):
"TP",
)
- def get_query_key_value_tensors(self, hidden_states, key_value_states=None, split_qkv=True):
+ def get_query_key_value_tensors(
+ self, hidden_states, key_value_states=None, output_gate=False, split_qkv=True
+ ):
"""
- Derives `query`, `key` and `value` tensors from `hidden_states`. If `split_qkv=False`, then
- the unsplit mixed_qkv tensor is returned.
+ Derives `query`, `key`, `value` tensors from `hidden_states`.
+ If `output_gate` is True, then also derives `gate` tensor.
+ If `split_qkv=False`, then the unsplit mixed_qkv tensor is returned.
"""
- # Attention heads [sq, b, h] --> [sq, b, ng * (np/ng + 2) * hn)]
+ # If no output gate: Attention heads [sq, b, h] --> [sq, b, ng * (np/ng + 2) * hn)]
+ # If have output gate: Attention heads [sq, b, h] --> [sq, b, ng * (2 * np/ng + 2) * hn)]
mixed_qkv, _ = self.linear_qkv(hidden_states)
+ num_query_heads_per_group = (
+ self.num_attention_heads_per_partition // self.num_query_groups_per_partition
+ )
+ num_qkv_heads_per_group = num_query_heads_per_group + 2
+ if output_gate:
+ num_qkv_heads_per_group += num_query_heads_per_group
+ # If no output gate: [sq, b, hp] --> [sq, b, ng, (np/ng + 2) * hn]
+ # If have output gate: [sq, b, hp] --> [sq, b, ng, (2 * np/ng + 2) * hn]
if self.config.num_query_groups < self.world_size:
# Note that weights are interleaved in the following manner:
# q1 q2 k1 v1 | q3 q4 k2 v2 | q5 q6 k3 v3 | ...
@@ -1173,39 +1251,47 @@ def get_query_key_value_tensors(self, hidden_states, key_value_states=None, spli
# [sq, b, hp] --> [sq, b, ng, (np/ng + 2) * hn]
new_tensor_shape = mixed_qkv.size()[:-1] + (
self.num_query_groups_per_partition,
- (
- (self.num_attention_heads_per_partition // self.num_query_groups_per_partition + 2)
- * self.hidden_size_per_attention_head
- ),
+ num_qkv_heads_per_group * self.hidden_size_per_attention_head,
)
mixed_qkv = mixed_qkv.view(*new_tensor_shape)
- split_arg_list = [
- (
- self.num_attention_heads_per_partition
- // self.num_query_groups_per_partition
- * self.hidden_size_per_attention_head
- ),
- self.hidden_size_per_attention_head,
- self.hidden_size_per_attention_head,
- ]
-
- # Return unsplit mixed_qkv and split_arg_list
- if not split_qkv:
- return mixed_qkv, split_arg_list
-
- if SplitAlongDim is not None:
+ # Split the tensor into query, gate, key, and value.
+ if output_gate:
+ if not split_qkv:
+ raise ValueError("split_qkv not supported for gated attention yet.")
+ # If have output gate: [sq, b, ng, (2 * np/ng + 2) * hn]
+ # --> [sq, b, ng, np/ng * hn], [sq, b, ng, np/ng * hn],
+ # [sq, b, ng, hn], [sq, b, ng, hn]
+ split_arg_list = [
+ num_query_heads_per_group * self.hidden_size_per_attention_head,
+ num_query_heads_per_group * self.hidden_size_per_attention_head,
+ self.hidden_size_per_attention_head,
+ self.hidden_size_per_attention_head,
+ ]
- # [sq, b, ng, (np/ng + 2) * hn]
- # --> [sq, b, ng, np/ng * hn], [sq, b, ng, hn], [sq, b, ng, hn]
- (query, key, value) = SplitAlongDim(mixed_qkv, 3, split_arg_list)
+ if SplitAlongDim is not None:
+ (query, gate, key, value) = SplitAlongDim(mixed_qkv, 3, split_arg_list)
+ else:
+ (query, gate, key, value) = torch.split(mixed_qkv, split_arg_list, dim=3)
else:
+ # If no output gate: [sq, b, ng, (np/ng + 2) * hn]
+ # --> [sq, b, ng, np/ng * hn], None, [sq, b, ng, hn], [sq, b, ng, hn]
+ split_arg_list = [
+ num_query_heads_per_group * self.hidden_size_per_attention_head,
+ self.hidden_size_per_attention_head,
+ self.hidden_size_per_attention_head,
+ ]
- # [sq, b, ng, (np/ng + 2) * hn]
- # --> [sq, b, ng, np/ng * hn], [sq, b, ng, hn], [sq, b, ng, hn]
- (query, key, value) = torch.split(mixed_qkv, split_arg_list, dim=3)
+ # Return unsplit mixed_qkv and split_arg_list
+ if not split_qkv:
+ return mixed_qkv, split_arg_list
- # [sq, b, ng, np/ng * hn] -> [sq, b, np, hn]
+ if SplitAlongDim is not None:
+ (query, key, value) = SplitAlongDim(mixed_qkv, 3, split_arg_list)
+ else:
+ (query, key, value) = torch.split(mixed_qkv, split_arg_list, dim=3)
+
+ # Query [sq, b, ng, np/ng * hn] -> [sq, b, np, hn]
query = query.reshape(query.size(0), query.size(1), -1, self.hidden_size_per_attention_head)
if self.config.num_query_groups < self.world_size:
@@ -1229,6 +1315,11 @@ def get_query_key_value_tensors(self, hidden_states, key_value_states=None, spli
if self.config.test_mode:
self.run_realtime_tests()
+ if output_gate:
+ # Gate [sq, b, ng, np/ng * hn] -> [sq, b, np, hn]
+ gate = gate.reshape(*gate.shape[:2], -1, self.hidden_size_per_attention_head)
+ return query, key, value, gate
+
return query, key, value
def backward_dw(self) -> NoReturn:
@@ -1402,11 +1493,15 @@ def __init__(
is_expert=False,
)
- def get_query_key_value_tensors(self, hidden_states, key_value_states, split_qkv=True):
+ def get_query_key_value_tensors(
+ self, hidden_states, key_value_states, output_gate=False, split_qkv=True
+ ):
"""
Derives `query` tensor from `hidden_states`, and `key`/`value` tensors
from `key_value_states`.
"""
+ assert not output_gate, "Output gate is not supported in cross attention for now."
+
assert split_qkv, "split_qkv must be True for CrossAttention"
# Attention heads [sk, b, h] --> [sk, b, (np * 2 * hn)]
mixed_kv, _ = self.linear_kv(key_value_states)
diff --git a/megatron/core/transformer/cuda_graphs.py b/megatron/core/transformer/cuda_graphs.py
index 7b81eb723ed..ec02555233b 100644
--- a/megatron/core/transformer/cuda_graphs.py
+++ b/megatron/core/transformer/cuda_graphs.py
@@ -1,14 +1,16 @@
-# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved.
+# Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
import gc
import inspect
import logging
+import math
import os
import time
from collections import defaultdict
from contextlib import nullcontext
from dataclasses import fields, is_dataclass
from enum import Enum
+from math import ceil
from typing import Any, Dict, List, Optional
import torch
@@ -21,8 +23,9 @@
get_all_rng_states,
get_cuda_rng_tracker,
)
+from megatron.core.transformer.enums import CudaGraphScope
from megatron.core.transformer.identity_op import IdentityOp
-from megatron.core.transformer.module import MegatronModule
+from megatron.core.transformer.module import GraphableMegatronModule, MegatronModule
from megatron.core.transformer.transformer_config import TransformerConfig
from megatron.core.utils import (
get_attr_wrapped_model,
@@ -1087,9 +1090,12 @@ def __init__(
), "RNG tracker does not support cudagraphs!"
assert config.cuda_graph_impl == "local", "Option cuda_graph_impl=local not enabled."
- assert "expandable_segments:True" not in os.getenv("PYTORCH_CUDA_ALLOC_CONF", ""), (
- "expandable_segments:True may not be safe when using CUDA Graphs, and may result in"
- "a crash due to illegal memory access or other undefined behaviour."
+ assert (
+ "expandable_segments:True" not in os.getenv("PYTORCH_CUDA_ALLOC_CONF", "")
+ or os.getenv("NCCL_GRAPH_REGISTER", "") == "0"
+ ), (
+ "Setting NCCL_GRAPH_REGISTER=0 to avoid illegal memory access when using "
+ "CUDA Graph with PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True."
)
self.cudagraph_runners = []
@@ -1347,23 +1353,40 @@ def _layer_is_graphable(layer, config):
Check if a layer is graphable.
"""
+ # Only GraphableMegatronModule can be graphed.
+ if not isinstance(layer, GraphableMegatronModule):
+ return False
+
+ # If cuda_graph_scope is not set, every layer is graphed.
+ if not config.cuda_graph_scope:
+ return True
+
# import modules here to avoid a circular import
from megatron.core.ssm.mamba_layer import MambaLayer
from megatron.core.transformer.identity_op import IdentityOp
+ from megatron.core.transformer.mlp import MLP
+ from megatron.core.transformer.moe.moe_layer import MoELayer
from megatron.core.transformer.transformer_layer import TransformerLayer
- if isinstance(layer, MambaLayer) and config.cuda_graph_scope == "full":
+ if isinstance(layer, MambaLayer) and CudaGraphScope.mamba in config.cuda_graph_scope:
# mamba layer.
return True
if isinstance(layer, TransformerLayer):
- if config.cuda_graph_scope == 'attn':
- if not (
- isinstance(layer.self_attention, IdentityOp)
- and isinstance(layer.cross_attention, IdentityOp)
- ):
- # attn layer.
- return True
- else:
+ if CudaGraphScope.attn in config.cuda_graph_scope and not (
+ isinstance(layer.self_attention, IdentityOp)
+ and isinstance(layer.cross_attention, IdentityOp)
+ ):
+ # attn layer.
+ return True
+ if (
+ CudaGraphScope.moe in config.cuda_graph_scope
+ or CudaGraphScope.moe_router in config.cuda_graph_scope
+ or CudaGraphScope.moe_preprocess in config.cuda_graph_scope
+ ) and isinstance(layer.mlp, MoELayer):
+ # moe layer.
+ return True
+ if CudaGraphScope.mlp in config.cuda_graph_scope and isinstance(layer.mlp, MLP):
+ # mlp layer.
return True
return False
@@ -1382,18 +1405,17 @@ def __init__(self, model, config, seq_length, micro_batch_size, optimizers=[]):
assert (
config.cuda_graph_impl == "transformer_engine"
), "Option cuda_graph_impl=transformer_engine not enabled."
- assert "expandable_segments:True" not in os.getenv("PYTORCH_CUDA_ALLOC_CONF", ""), (
- "expandable_segments:True may not be safe when using CUDA Graphs, and may result in"
- "a crash due to illegal memory access or other undefined behaviour."
+ assert (
+ "expandable_segments:True" not in os.getenv("PYTORCH_CUDA_ALLOC_CONF", "")
+ or os.getenv("NCCL_GRAPH_REGISTER", "") == "0"
+ ), (
+ "Setting NCCL_GRAPH_REGISTER=0 to avoid illegal memory access when using "
+ "CUDA Graph with PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True."
)
- assert config.cuda_graph_scope != "full_iteration", (
+ assert CudaGraphScope.full_iteration not in config.cuda_graph_scope, (
"full_iteration cuda graph is not supported for cuda_graph_impl=transformer_engine. "
"Please use cuda_graph_impl=local instead."
)
- assert config.cuda_graph_scope in [
- 'full',
- 'attn',
- ], f"--cuda-graph-scope should be full or attn, got {config.cuda_graph_scope}."
self.model = model
self.config = config
@@ -1402,6 +1424,9 @@ def __init__(self, model, config, seq_length, micro_batch_size, optimizers=[]):
self.optimizers = optimizers
self.num_model_chunks = len(model)
+ # Number of microbatches to capture. The value will be set in _get_cuda_graph_input_data().
+ self.num_microbatches = None
+
# Get callables with captureable layers.
self.chunks_with_decoder = []
self.num_layers_per_chunk = []
@@ -1476,83 +1501,295 @@ def __init__(self, model, config, seq_length, micro_batch_size, optimizers=[]):
f'{len(self.flattened_callables)} graphable layers.',
)
- def _get_cuda_graph_input_data(self):
+ # One helper object can only capture CUDA Graphs once. Use this flag to check if the graphs
+ # have been created.
+ self._graphs_created = False
+
+ def graphs_created(self):
"""
- Create the CUDA Graph capturing input data.
- The data is organized per-chunk per-microbatch per-layer.
+ Returns whether the CUDA Graphs have been created.
"""
+ return self._graphs_created
- rotary_pos_emb_cache = {}
+ def _get_sample_arguments(self, order, chunk_id_list=None):
+ """
+ Generate sample arguments and keyword arguments for CUDA Graph capturing with
+ memory-optimized buffer reuse.
+
+ This method creates static input tensors for each (layer, microbatch) pair needed
+ by TE's make_graphed_callables(). It optimizes memory usage by reusing input buffers
+ across non-overlapping forward passes based on the pipeline parallel schedule.
+ This optimization is essential for reducing peak memory during CUDA Graph capturing with
+ many microbatches, as it allows buffers to be reused instead of allocating new ones for
+ later microbatches.
+
+ Memory Optimization Strategy:
+ The 1F1B (one-forward-one-backward) interleaved schedule in pipeline parallelism
+ means that once a microbatch's backward pass completes, its input buffers are no
+ longer needed. This method tracks buffer lifecycle and reuses "consumed" buffers
+ (those whose backward has completed) for new forward passes with matching tensor
+ signatures (shape, dtype, layout).
+
+ Example schedule: [1, 1, 1, 2, 2, 2, -2, 1, -2, 1, -2, 2, -1, 2, -1, -1, -2, -2, -1, -1]
+ - Positive values indicate forward passes (chunk_id = value)
+ - Negative values indicate backward passes (chunk_id = -value)
+ - When processing -2 (backward of chunk 2), its buffers become available for reuse
+ - The next forward with matching signature can reuse those buffers
- def get_rotary_pos_emb(transformer_module, transformer_input):
- if (
- transformer_module.position_embedding_type == 'rope'
- and not self.config.multi_latent_attention
- ):
- rotary_seq_len = transformer_module.rotary_pos_emb.get_rotary_seq_len(
- None, transformer_module.decoder, transformer_input, self.config, None
- )
- if rotary_seq_len not in rotary_pos_emb_cache:
- rotary_pos_emb_cache[rotary_seq_len] = transformer_module.rotary_pos_emb(
- rotary_seq_len
- )
- return rotary_pos_emb_cache[rotary_seq_len]
- else:
- return None
+ Args:
+ order (List[int]): The forward/backward execution order from
+ convert_schedule_table_to_order(). Positive integers represent forward passes
+ (1-indexed chunk ID), negative integers represent backward passes.
+ chunk_id_list (List[Tuple[int, int]]): The list of chunk IDs and layer IDs in the
+ order. This is useful only when overlap_moe_expert_parallel_comm is enabled,
+ the order maps each layers' idx to their original chunk id.
+
+ Returns:
+ Tuple[List[Tuple], List[Dict]]: A tuple containing:
+ - sample_args: List of positional argument tuples for each (layer, microbatch).
+ Length = num_layers * num_microbatches. Elements with the same tensor
+ signature may share references to reduce memory allocation.
+ - sample_kwargs: List of keyword argument dicts for each (layer, microbatch).
+ Length = num_layers * num_microbatches. Elements with the same tensor
+ signature may share references to reduce memory allocation.
+
+ Data Structures:
+ - fwd_sample_queues: Dict[chunk_id, List[Tuple[sample_keys, fwd_idx]]]
+ Queue of forward samples per chunk awaiting their backward pass.
+ - consumed_sample_queue: Dict[sample_keys, List[fwd_idx]]
+ Pool of buffer indices whose backward is complete, keyed by tensor signature.
+ - sample_keys: Tuple of (shape, dtype, layout) for args + (key, shape, dtype, layout)
+ for kwargs, used to match compatible buffers for reuse.
+ """
+ assert self.num_model_chunks == max(
+ order
+ ), "num_model_chunks must match the max chunk id in order."
+ if chunk_id_list is None:
+ # check only if 1f1b overlap is disabled.
+ assert (
+ self.num_microbatches == len(order) // self.num_model_chunks // 2
+ ), "num_microbatches must match the number of microbatches in order."
# Generate sample arguments and keyword arguments for capturing.
- sample_args = []
- sample_kwargs = []
- for chunk_number, chunk_with_decoder in enumerate(self.chunks_with_decoder):
- if chunk_with_decoder is None:
- continue
- layers = self.callables_per_chunk[chunk_number]
- for _ in range(get_num_microbatches()):
- for layer in layers:
- static_inputs = layer.get_layer_static_inputs(
- self.seq_length, self.micro_batch_size
+ sample_args = [None] * (len(self.flattened_callables) * self.num_microbatches)
+ sample_kwargs = [None] * (len(self.flattened_callables) * self.num_microbatches)
+
+ rotary_pos_emb_cache = {}
+
+ def _get_layer_static_inputs(layer, chunk_of_the_layer):
+ """
+ Get the static inputs for a layer.
+ """
+ assert layer in chunk_of_the_layer.decoder.layers or any(
+ layer is mtp_layer.transformer_layer for mtp_layer in chunk_of_the_layer.mtp.layers
+ ), "Layer is not in the chunk"
+
+ def get_rotary_pos_emb(transformer_module, transformer_input):
+ if (
+ transformer_module.position_embedding_type == 'rope'
+ and not self.config.multi_latent_attention
+ ):
+ rotary_seq_len = transformer_module.rotary_pos_emb.get_rotary_seq_len(
+ None, transformer_module.decoder, transformer_input, self.config, None
)
+ if rotary_seq_len not in rotary_pos_emb_cache:
+ rotary_pos_emb_cache[rotary_seq_len] = transformer_module.rotary_pos_emb(
+ rotary_seq_len
+ )
+ return rotary_pos_emb_cache[rotary_seq_len]
+ else:
+ return None
+
+ static_inputs = layer.get_layer_static_inputs(self.seq_length, self.micro_batch_size)
+
+ from megatron.core.transformer.identity_op import IdentityOp
+ from megatron.core.transformer.transformer_layer import TransformerLayer
- from megatron.core.transformer.identity_op import IdentityOp
- from megatron.core.transformer.transformer_layer import TransformerLayer
+ contains_self_attn = (
+ isinstance(layer, TransformerLayer)
+ and not isinstance(layer.self_attention, IdentityOp)
+ and (
+ not self.config.cuda_graph_scope
+ or CudaGraphScope.attn in self.config.cuda_graph_scope
+ )
+ )
- contains_self_attn = isinstance(layer, TransformerLayer) and not isinstance(
- layer.self_attention, IdentityOp
+ _sample_kwargs = {}
+ if is_te_min_version("1.10.0"):
+ # te.make_graphed_callables() accepts keyword arguments since 1.10.0.
+ hidden_states = static_inputs.pop("hidden_states")
+ _sample_args = (hidden_states,)
+ if contains_self_attn:
+ rotary_pos_emb = get_rotary_pos_emb(chunk_of_the_layer, hidden_states)
+ if rotary_pos_emb is not None:
+ static_inputs["rotary_pos_emb"] = rotary_pos_emb
+ _sample_kwargs = static_inputs
+ elif contains_self_attn:
+ _sample_args = (
+ static_inputs.pop("hidden_states"),
+ static_inputs.pop("attention_mask"),
+ )
+ else:
+ _sample_args = (static_inputs.pop("hidden_states"),)
+ return _sample_args, _sample_kwargs
+
+ # Calculate the starting index of each chunk in callables for future use.
+ prefix_num_layers = [0]
+ for model_chunk_idx in range(self.num_model_chunks):
+ num_layers = self.num_layers_per_chunk[model_chunk_idx]
+ prefix_num_layers.append(prefix_num_layers[-1] + num_layers)
+
+ # Reorganize args and kwargs for input tensor reuse.
+ # fwd_sample_queues is keyed by model chunk index. The value is a queue of tuples.
+ # Each tuple contains the sample key signature and its fwd_idx. When we finish a backward
+ # chunk, we pop the corresponding fwd_idx and push to the consumed_sample_queue.
+ # consumed_sample_queue is keyed by the sample key signature. The value is a queue of the
+ # fwd_idx whose backward has been called so that we can reuse the same static buffers.
+ # In this way, we can reuse the same static input buffers for the non-overlapping samples
+ # with the same input signature.
+ fwd_sample_queues = {}
+ consumed_sample_queue = {}
+ layer_sample_keys_cache = {}
+ fwd_idx = [0] * self.num_model_chunks
+ for idx, chunk_id in enumerate(order):
+ model_chunk_idx = abs(ceil(chunk_id)) - 1
+
+ if chunk_id > 0:
+ if model_chunk_idx not in fwd_sample_queues:
+ fwd_sample_queues[model_chunk_idx] = []
+
+ sample_start_idx = (prefix_num_layers[model_chunk_idx] * self.num_microbatches) + (
+ fwd_idx[model_chunk_idx] * self.num_layers_per_chunk[model_chunk_idx]
+ )
+ if chunk_id_list:
+ model_chunk_idx = chunk_id_list[idx][0]
+ callables_curr_chunk = [
+ self.callables_per_chunk[model_chunk_idx][chunk_id_list[idx][1]]
+ ]
+ else:
+ callables_curr_chunk = self.callables_per_chunk[model_chunk_idx]
+ for layer_idx, layer in enumerate(callables_curr_chunk):
+ per_callable_fwd_idx = sample_start_idx + layer_idx
+
+ # Get sample_args and sample_kwargs for index per_callable_fwd_idx.
+ assert (
+ sample_args[per_callable_fwd_idx] is None
+ and sample_kwargs[per_callable_fwd_idx] is None
+ ), (
+ f"sample_args and sample_kwargs must be None before assigning static data, "
+ f"but got sample_args[{per_callable_fwd_idx}] = "
+ f"{sample_args[per_callable_fwd_idx]} and "
+ f"sample_kwargs[{per_callable_fwd_idx}] = "
+ f"{sample_kwargs[per_callable_fwd_idx]}."
)
- if is_te_min_version("1.10.0"):
- # te.make_graphed_callables() accepts keyword arguments since 1.10.0.
- hidden_states = static_inputs.pop("hidden_states")
- sample_args.append((hidden_states,))
- if contains_self_attn:
- rotary_pos_emb = get_rotary_pos_emb(chunk_with_decoder, hidden_states)
- if rotary_pos_emb is not None:
- static_inputs["rotary_pos_emb"] = rotary_pos_emb
- sample_kwargs.append(static_inputs)
- elif contains_self_attn:
- sample_args.append(
- (
- static_inputs.pop("hidden_states"),
- static_inputs.pop("attention_mask"),
+ if id(layer) not in layer_sample_keys_cache:
+ # Have not generated the static inputs for this layer yet. So we don't
+ # know the input signature of this layer. Generate the static inputs, and
+ # cache the signature.
+ sample_args[per_callable_fwd_idx], sample_kwargs[per_callable_fwd_idx] = (
+ _get_layer_static_inputs(
+ layer, self.chunks_with_decoder[model_chunk_idx]
)
)
+ sample_args_keys = tuple(
+ (t.shape, t.dtype, t.layout) for t in sample_args[per_callable_fwd_idx]
+ )
+ sample_kwargs_keys = tuple(
+ (k, v.shape, v.dtype, v.layout)
+ for k, v in sorted(sample_kwargs[per_callable_fwd_idx].items())
+ )
+ sample_keys = sample_args_keys + sample_kwargs_keys
+ layer_sample_keys_cache[id(layer)] = sample_keys
else:
- sample_args.append((static_inputs.pop("hidden_states"),))
+ # Get signature from cache. This signature will be used to see if we can
+ # reuse the static inputs of a previous forward pass for this forward pass.
+ # If not, we still need to generate the new static inputs.
+ sample_keys = layer_sample_keys_cache[id(layer)]
+ model_chunk_idx = abs(chunk_id) - 1
+ fwd_sample_queues[model_chunk_idx].append((sample_keys, per_callable_fwd_idx))
+ if consumed_sample_queue.get(sample_keys, []):
+ # We can reuse the static inputs of a previous forward pass for this
+ # forward pass, because they are of the same input signature and the
+ # backward pass of the previous forward pass has completed.
+ reuse_fwd_idx = consumed_sample_queue[sample_keys].pop(0)
+ assert (
+ sample_args[reuse_fwd_idx] is not None
+ and sample_kwargs[reuse_fwd_idx] is not None
+ ), (
+ f"sample_args and sample_kwargs must not be None when reusing, but got "
+ f"sample_args[{reuse_fwd_idx}] = {sample_args[reuse_fwd_idx]} and "
+ f"sample_kwargs[{reuse_fwd_idx}] = {sample_kwargs[reuse_fwd_idx]}.",
+ )
+ sample_args[per_callable_fwd_idx] = sample_args[reuse_fwd_idx]
+ sample_kwargs[per_callable_fwd_idx] = sample_kwargs[reuse_fwd_idx]
+
+ if sample_args[per_callable_fwd_idx] is None:
+ # Unfortunately, no previous static inputs are available for reuse,
+ # sample_args is still None. Last attempt: generate the new static inputs
+ # for this forward pass.
+ if chunk_id_list:
+ model_chunk_idx = chunk_id_list[idx][0]
+ sample_args[per_callable_fwd_idx], sample_kwargs[per_callable_fwd_idx] = (
+ _get_layer_static_inputs(
+ layer, self.chunks_with_decoder[model_chunk_idx]
+ )
+ )
+ model_chunk_idx = abs(chunk_id) - 1
+ fwd_idx[model_chunk_idx] += 1
+ elif ceil(chunk_id) == chunk_id:
+ num_consumed_samples = min(
+ len(fwd_sample_queues[model_chunk_idx]),
+ self.num_layers_per_chunk[model_chunk_idx],
+ )
+ for sample_keys, per_callable_fwd_idx in fwd_sample_queues[model_chunk_idx][
+ :num_consumed_samples
+ ]:
+ if sample_keys not in consumed_sample_queue:
+ consumed_sample_queue[sample_keys] = []
+ consumed_sample_queue[sample_keys].append(per_callable_fwd_idx)
+ fwd_sample_queues[model_chunk_idx] = fwd_sample_queues[model_chunk_idx][
+ num_consumed_samples:
+ ]
+ else:
+ # skip register static inputs for wgrad backward graphs
+ continue
+
+ return sample_args, sample_kwargs
+
+ def _get_cuda_graph_input_data(self):
+ """
+ Create the CUDA Graph capturing input data.
+ The data is organized per-chunk per-microbatch per-layer.
+ """
# Get the PP and VPP scheduling order.
from megatron.core.pipeline_parallel.schedules import (
convert_schedule_table_to_order,
+ get_overlap_moe_expert_parallel_comm_order,
get_pp_rank_microbatches,
get_schedule_table,
)
+ # If PP is not enabled, we only need to capture one microbatch.
+ if (
+ parallel_state.get_pipeline_model_parallel_world_size() == 1
+ and not self.config.overlap_moe_expert_parallel_comm
+ ):
+ assert (
+ self.num_model_chunks == 1
+ ), "If PP is not enabled, there should be only one model chunk."
+ self.num_microbatches = 1
+ else:
+ self.num_microbatches = get_num_microbatches()
+
_, _, num_warmup_microbatches, _ = get_pp_rank_microbatches(
- get_num_microbatches(),
+ self.num_microbatches,
self.num_model_chunks,
self.config.microbatch_group_size_per_vp_stage,
False,
)
schedule_table = get_schedule_table(
- get_num_microbatches(),
+ self.num_microbatches,
self.num_model_chunks,
self.config.microbatch_group_size_per_vp_stage,
)
@@ -1566,9 +1803,57 @@ def get_rotary_pos_emb(transformer_module, transformer_input):
level=logging.DEBUG,
msg=f'Rank {torch.distributed.get_rank()}: ORDER {order}',
)
+ chunk_id_list = None
+ if self.config.overlap_moe_expert_parallel_comm:
+ wgrad_in_graph_scope = CudaGraphScope.attn in self.config.cuda_graph_scope or (
+ CudaGraphScope.moe_router in self.config.cuda_graph_scope
+ and self.config.moe_shared_expert_intermediate_size is not None
+ and not self.config.moe_shared_expert_overlap
+ )
+ capture_wgrad_graph = self.config.delay_wgrad_compute and wgrad_in_graph_scope
+ order, chunk_id_list = get_overlap_moe_expert_parallel_comm_order(
+ order, self.num_layers_per_chunk, capture_wgrad_graph
+ )
+ self.num_layers_per_chunk = [1] * sum(self.num_layers_per_chunk)
+ self.num_model_chunks = max(order)
+ _order_without_wgrad = []
+ for c_id in order:
+ if ceil(c_id) != c_id:
+ continue
+ _order_without_wgrad.append(c_id)
+ self.num_microbatches = len(_order_without_wgrad) // self.num_model_chunks // 2
+ log_on_each_pipeline_stage(
+ logger=logger,
+ tp_group=None,
+ dp_cp_group=None,
+ level=logging.DEBUG,
+ msg=f'Rank {torch.distributed.get_rank()}: '
+ f'ORDER after overlap_moe_expert_parallel_comm {order}',
+ )
+
+ # Generate sample arguments and keyword arguments for capturing.
+ sample_args, sample_kwargs = self._get_sample_arguments(order, chunk_id_list)
def get_make_graphed_callables_kwargs():
- kwargs = {'num_warmup_iters': 11, 'allow_unused_input': True, '_order': order}
+ kwargs = {
+ 'allow_unused_input': True,
+ '_order': order,
+ 'retain_graph_in_backward': self.config.cuda_graph_retain_backward_graph,
+ }
+
+ # Calculate the number of warmup iterations per layer per microbatch inside TE
+ # make_graphed_callables(). There are two rules:
+ # 1. There should be at least 1 warmup iteration per layer per microbatch inside TE
+ # make_graphed_callables().
+ # 2. There should be at least 10 warmup iterations per layer, counting the MCore warmup
+ # steps before going into this capture routine.
+ kwargs['num_warmup_iters'] = max(
+ 1,
+ math.ceil(
+ (10 - self.config.cuda_graph_warmup_steps * get_num_microbatches())
+ / self.num_microbatches
+ ),
+ )
if is_te_min_version("2.6.0"):
# Starting from TE 2.6.0, make_graphed_callables() accepts different number
@@ -1626,9 +1911,13 @@ def _start_capturing(self):
"""
Start capturing CUDA Graphs.
"""
+ assert not self._graphs_created, "CUDA Graphs have already been created."
+
torch.distributed.barrier()
gc.collect()
torch.cuda.empty_cache()
+ if FREEZE_GC:
+ gc.freeze()
_set_capture_start()
log_single_rank(logger, logging.INFO, f'Start CUDA Graphs capture...')
@@ -1656,9 +1945,14 @@ def _finish_capturing(self, start_time):
optimizer.zero_grad()
clear_aux_losses_tracker()
reset_model_temporary_tensors(self.config, self.model)
+
+ if FREEZE_GC:
+ gc.unfreeze()
gc.collect()
torch.cuda.empty_cache()
+ self._graphs_created = True
+
def create_cudagraphs(self):
"""
Capture CUDA Graphs per TransformerLayer per microbatch.
@@ -1667,21 +1961,30 @@ def create_cudagraphs(self):
# Prepare CUDA Graph capturing input data and call `make_graphed_callables`.
sample_args, kwargs = self._get_cuda_graph_input_data()
- graphs = make_graphed_callables(tuple(self.flattened_callables), sample_args, **kwargs)
+ if self.config.sequence_parallel:
+ rng_context = get_cuda_rng_tracker().fork()
+ else:
+ rng_context = nullcontext()
+ with rng_context:
+ graphs = make_graphed_callables(tuple(self.flattened_callables), sample_args, **kwargs)
# Push the captured graphs to the corresponding TransformerBlock.
num_layers_accumulated = 0
for layers in self.callables_per_chunk:
for layer_number, layer in enumerate(layers):
layer.cuda_graphs = []
- for batch_number in range(get_num_microbatches()):
- layer.cuda_graphs.append(
- graphs[
- num_layers_accumulated * get_num_microbatches()
+ for batch_number in range(self.num_microbatches):
+ if self.config.overlap_moe_expert_parallel_comm:
+ graph_idx = (
+ num_layers_accumulated + layer_number
+ ) * self.num_microbatches + batch_number
+ else:
+ graph_idx = (
+ num_layers_accumulated * self.num_microbatches
+ batch_number * len(layers)
+ layer_number
- ]
- )
+ )
+ layer.cuda_graphs.append(graphs[graph_idx])
num_layers_accumulated += len(layers)
self._finish_capturing(start_time)
@@ -1695,3 +1998,33 @@ def cuda_graph_set_manual_hooks(self):
model_chunk = self.model[chunk_number]
for layer in layers:
layer.setup_manual_hooks(model_chunk._make_forward_pre_hook)
+
+ def delete_cuda_graphs(self):
+ """
+ Delete all CUDA graphs.
+ """
+ assert self._graphs_created, "CUDA Graphs have not been created."
+
+ graph_resettable = is_te_min_version("2.10.0")
+ graphs_reset, graphs_not_reset = 0, 0
+ for layers in self.callables_per_chunk:
+ for layer in layers:
+ for graph in layer.cuda_graphs:
+ if graph_resettable:
+ graph.reset()
+ graphs_reset += 1
+ else:
+ graphs_not_reset += 1
+ layer.cuda_graphs = []
+ layer.cuda_graph_manual_hooks = []
+
+ log_on_each_pipeline_stage(
+ logger=logger,
+ tp_group=None,
+ dp_cp_group=None,
+ level=logging.INFO,
+ msg=f'Rank {torch.distributed.get_rank()}: '
+ f'{graphs_reset} graphs deleted with explicit reset, '
+ f'{graphs_not_reset} graphs deleted without explicit reset.',
+ )
+ self._graphs_created = False
diff --git a/megatron/core/transformer/dot_product_attention.py b/megatron/core/transformer/dot_product_attention.py
index 32f9a08cfa1..7102440552a 100644
--- a/megatron/core/transformer/dot_product_attention.py
+++ b/megatron/core/transformer/dot_product_attention.py
@@ -1,4 +1,4 @@
-# Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved.
+# Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
import math
@@ -12,6 +12,9 @@
from megatron.core.fusions.fused_softmax import FusedScaleMaskSoftmax
from megatron.core.packed_seq_params import PackedSeqParams
from megatron.core.process_groups_config import ProcessGroupCollection
+from megatron.core.transformer.dot_product_attention_context_parallel import (
+ AttentionFuncionWithContextParallel,
+)
from megatron.core.transformer.enums import AttnMaskType
from megatron.core.transformer.module import MegatronModule
from megatron.core.transformer.transformer_config import TransformerConfig
@@ -54,9 +57,12 @@ def __init__(
self.config: TransformerConfig = config
- assert (
- self.config.context_parallel_size == 1
- ), "Context parallelism is only supported by TEDotProductAttention!"
+ if self.config.context_parallel_size > 1:
+ assert attention_dropout is None and self.config.attention_dropout == 0.0, (
+ f'DotProductAttention with context parallelism does not support attention dropout,'
+ f' but got {self.config.context_parallel_size=},'
+ f' {attention_dropout=}, and {self.config.attention_dropout=}.'
+ )
self.layer_number = max(1, layer_number)
self.attn_mask_type = attn_mask_type
@@ -174,6 +180,19 @@ def forward(
self.num_attention_heads_per_partition // self.num_query_groups_per_partition, dim=2
)
+ if self.config.context_parallel_size > 1:
+ output = AttentionFuncionWithContextParallel.apply(
+ query,
+ key,
+ value,
+ attention_mask,
+ self.config.attention_dropout,
+ self.softmax_scale,
+ parallel_state.get_context_parallel_group(),
+ )
+ output = output.view(query.shape[0], query.shape[1], self.hidden_size_per_partition)
+ return output
+
# [b, np, sq, sk]
output_size = (query.size(1), query.size(2), query.size(0), key.size(0))
diff --git a/megatron/core/transformer/dot_product_attention_context_parallel.py b/megatron/core/transformer/dot_product_attention_context_parallel.py
new file mode 100644
index 00000000000..aaf08d40ade
--- /dev/null
+++ b/megatron/core/transformer/dot_product_attention_context_parallel.py
@@ -0,0 +1,345 @@
+# Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+
+# Some of this code was adopted from https://github.com/zhuzilin/ring-flash-attention/
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+
+import torch
+from torch.nn import functional as F
+
+try:
+ import einops
+
+ HAVE_EINOPS = True
+except ImportError:
+ HAVE_EINOPS = False
+
+
+@torch.no_grad
+def eager_attn_fwd(q, k, v, attn_bias, sinks, scale, dropout):
+ """Forward pass for eager attention"""
+
+ # Rearrange query, key, value to (b, h, s, d)
+ b, sq, h, d = q.shape
+ sk = k.shape[1]
+ _q = einops.rearrange(q, 'b s h d -> b h s d')
+ _k = einops.rearrange(k, 'b s h d -> b h d s')
+ _v = einops.rearrange(v, 'b s h d -> b h s d')
+
+ # Compute attention weights
+ attn_w = torch.matmul(_q, _k) * scale
+ attn_w = attn_w + attn_bias
+
+ # Add sinks to attention weights
+ if sinks is None:
+ logits = attn_w
+ else:
+ _sinks = sinks.reshape(1, h, 1, 1).expand(b, -1, sq, 1)
+ logits = torch.cat([attn_w, _sinks], dim=-1)
+
+ # Compute attention scores
+ probs = F.softmax(logits, dim=-1, dtype=logits.dtype)
+ if sinks is None:
+ attn_w = probs
+ else:
+ attn_w = probs[..., :-1] # Drop the sink
+
+ # Compute attention output
+ attn_output = torch.matmul(attn_w, _v)
+ attn_output = einops.rearrange(attn_output, 'b h s d -> b s h d')
+ attn_output = attn_output.contiguous()
+
+ return attn_output, probs
+
+
+@torch.no_grad
+def eager_attn_bwd(q, k, v, attn_bias, sinks, scale, dropout, attn_output, probs, grad_output):
+ """Backward pass for eager attention"""
+
+ # Rearrange query, key, value to (b, h, s, d)
+ b, sq, h, d = q.shape
+ sk = k.shape[1]
+ _q_T = einops.rearrange(q, 'b s h d -> b h d s')
+ _k_T = einops.rearrange(k, 'b s h d -> b h s d')
+ _v_T = einops.rearrange(v, ' b s h d -> b h d s')
+
+ # Backward pass for score @ value
+ if sinks is None:
+ attn_w = probs
+ else:
+ attn_w = probs[..., :-1] # Drop the sink
+ grad_output = einops.rearrange(grad_output, 'b s h d -> b h s d')
+ attn_w_T = einops.rearrange(attn_w, ' b h sq sk -> b h sk sq')
+ grad__v = torch.matmul(attn_w_T, grad_output)
+ grad_attn_w = torch.matmul(grad_output, _v_T)
+
+ # Backward pass for softmax
+ if sinks is None:
+ grad_probs = grad_attn_w
+ else:
+ dummy = torch.zeros((b, h, sq, 1), device=q.device, dtype=q.dtype)
+ grad_probs = torch.cat([grad_attn_w, dummy], dim=3)
+ del grad_attn_w
+ grad_logits = torch._softmax_backward_data(
+ grad_probs, probs, -1, probs.dtype
+ ) # [b, h, sq, sk+1]
+
+ # Backward pass for adding sinks
+ if sinks is None:
+ grad_sinks = None
+ grad_attn_w = grad_logits
+ else:
+ grad__sinks = grad_logits[:, :, :, -1] # [b, h, sq]
+ grad_sinks = einops.rearrange(grad__sinks, 'b h s -> h (b s)').sum(-1)
+ grad_attn_w = grad_logits[:, :, :, :-1].contiguous() # [b, h, sq, sk]
+
+ # Backward pass for q @ K^T
+ grad_attn_w *= scale
+ grad__q = torch.matmul(grad_attn_w, _k_T)
+ grad__k = torch.matmul(_q_T, grad_attn_w)
+
+ # Rearrange grads to (b, s, h, d)
+ grad_v = einops.rearrange(grad__v, 'b h s d -> b s h d')
+ grad_k = einops.rearrange(grad__k, 'b h d s -> b s h d')
+ grad_q = einops.rearrange(grad__q, 'b h s d -> b s h d')
+ return grad_q, grad_k, grad_v, grad_sinks
+
+
+class AllGatherComm:
+ """All gather communication with async operations"""
+
+ def __init__(self, group=None) -> None:
+ self.group = group
+ self.handles = []
+
+ def all_gather(self, output_tensor: torch.Tensor, input_tensor: torch.Tensor):
+ '''All gather the input tensor to the output tensor'''
+
+ if self.group is None:
+ output_tensor.copy_(input_tensor)
+ else:
+ handle = torch.distributed.all_gather_into_tensor(
+ output_tensor, input_tensor, group=self.group, async_op=True
+ )
+ self.handles.append(handle)
+
+ def wait(self):
+ '''Wait for all gather operations to complete'''
+
+ if self.group is not None:
+ for handle in self.handles:
+ handle.wait()
+ self.handles = []
+
+
+def to_zz_mask_attn_bias(attention_mask, cp_size, nheads, nheads_k, heads_k_stride, device, dtype):
+ '''Convert the attention mask to the attention bias'''
+
+ if cp_size == 1:
+ zz_mask = attention_mask
+ else:
+ chunked = attention_mask.chunk(dim=3, chunks=cp_size * 2)
+ zz_mask = [_x for _p in zip(chunked[:cp_size], reversed(chunked[cp_size:])) for _x in _p]
+ zz_mask = torch.cat(zz_mask, dim=3)
+ attn_bias = torch.zeros(zz_mask.shape, device=device, dtype=dtype)
+ attn_bias.masked_fill_(zz_mask, float('-inf'))
+ attn_bias = attn_bias.expand(-1, heads_k_stride * (nheads // nheads_k), -1, -1)
+ return attn_bias
+
+
+class AttentionFuncionWithContextParallel(torch.autograd.Function):
+ """Native attention function with context parallelism."""
+
+ @staticmethod
+ def forward(ctx, q, k, v, attention_mask, attention_dropout, softmax_scale, pg):
+ '''Forward pass for the native attention function with context parallelism'''
+
+ # Assert einops exists
+ if not HAVE_EINOPS:
+ raise ImportError("einops is required by the attention CP but cannot be imported.")
+
+ # Initialize communication group and constants
+ cp_size = 1
+ if pg is not None:
+ cp_size = torch.distributed.get_world_size(pg)
+ comm = AllGatherComm(group=pg)
+ nheads = q.shape[2]
+ nheads_k = k.shape[2]
+ heads_k_stride = 1
+ assert nheads % nheads_k == 0 and nheads_k % heads_k_stride == 0
+ outs = []
+ probs = []
+
+ # Initialize KV buffers
+ kv_buffer = torch.empty(
+ (2, k.shape[0] * cp_size, k.shape[1], heads_k_stride, k.shape[3]),
+ dtype=k.dtype,
+ device=k.device,
+ )
+ kv_buffer_copy = torch.empty_like(kv_buffer)
+
+ # All-gather first chunk of KV buffers
+ k_0 = k[:, :, :heads_k_stride].contiguous()
+ v_0 = v[:, :, :heads_k_stride].contiguous()
+ comm.all_gather(kv_buffer_copy[0], k_0)
+ comm.all_gather(kv_buffer_copy[1], v_0)
+
+ # Prepare attention bias
+ assert (
+ attention_mask is not None
+ ), "Attention mask is required for the native attention function with context parallelism"
+ attn_bias = to_zz_mask_attn_bias(
+ attention_mask, cp_size, nheads, nheads_k, heads_k_stride, q.device, q.dtype
+ )
+
+ # Iterate over heads
+ for i in range(0, nheads_k, heads_k_stride):
+ # Wait for previous all-gather to complete
+ comm.wait()
+ kv_buffer, kv_buffer_copy = kv_buffer_copy, kv_buffer
+ # All-gather the next portion of KV buffers if not the last iteration
+ if i < nheads_k - heads_k_stride:
+ kvsl = i + heads_k_stride
+ kvsr = kvsl + heads_k_stride
+ send_k = k[:, :, kvsl:kvsr].contiguous()
+ send_v = v[:, :, kvsl:kvsr].contiguous()
+ comm.all_gather(kv_buffer_copy[0], send_k)
+ comm.all_gather(kv_buffer_copy[1], send_v)
+
+ # Prepare query, key, value for attention
+ q_i = q[:, :, i * nheads // nheads_k : (i + heads_k_stride) * nheads // nheads_k]
+ k_i = kv_buffer[0]
+ v_i = kv_buffer[1]
+
+ # Rearrange query, key, value to (b, s, h, d)
+ q_i = einops.rearrange(q_i, 's b h d -> b s h d')
+ k_i = einops.rearrange(k_i, 's b h d -> b s h d')
+ v_i = einops.rearrange(v_i, 's b h d -> b s h d')
+
+ # Forward pass
+ out_i, probs_i = eager_attn_fwd(
+ q_i, k_i, v_i, attn_bias, None, softmax_scale, attention_dropout
+ )
+ outs.append(out_i)
+ probs.append(probs_i)
+
+ # Concatenate outputs and rearrange to (s, b, h, d)
+ out = torch.cat(outs, dim=2)
+ out = einops.rearrange(out, 'b s h d -> s b h d')
+
+ # Save contexts for backward pass
+ ctx.save_for_backward(q, k, v, attention_mask, *outs, *probs)
+ ctx.dropout = attention_dropout
+ ctx.scale = softmax_scale
+ ctx.heads_k_stride = heads_k_stride # TODO make it configurable
+ ctx.pg = pg
+
+ return out
+
+ @staticmethod
+ def backward(ctx, dout):
+ '''Backward pass for the native attention function with context parallelism'''
+
+ # Initialize or resume constants and communication group
+ q, k, v, attention_mask, *rest = ctx.saved_tensors
+ nheads = q.shape[2]
+ nheads_k = k.shape[2]
+ heads_k_stride = ctx.heads_k_stride
+ assert nheads_k % heads_k_stride == 0
+ outs = rest[: nheads_k // heads_k_stride]
+ probs = rest[nheads_k // heads_k_stride :]
+ pg = ctx.pg
+ cp_size = 1
+ if pg is not None:
+ cp_size = torch.distributed.get_world_size(pg)
+ comm = AllGatherComm(group=pg)
+
+ # Initialize KV buffers
+ kv_buffer = torch.empty(
+ (2, k.shape[0] * cp_size, k.shape[1], heads_k_stride, k.shape[3]),
+ dtype=k.dtype,
+ device=k.device,
+ )
+ kv_buffer_copy = torch.empty_like(kv_buffer)
+
+ # All-gather first chunk of KV buffers
+ dq = []
+ dk = []
+ dv = []
+ k_0 = k[:, :, :heads_k_stride].contiguous()
+ v_0 = v[:, :, :heads_k_stride].contiguous()
+ comm.all_gather(kv_buffer_copy[0], k_0)
+ comm.all_gather(kv_buffer_copy[1], v_0)
+
+ # Prepare attention bias
+ attn_bias = to_zz_mask_attn_bias(
+ attention_mask, cp_size, nheads, nheads_k, heads_k_stride, q.device, q.dtype
+ )
+
+ # Iterate over heads
+ for i in range(0, nheads_k, heads_k_stride):
+ # Slice query and output for this iteration
+ q_slice = slice(i * nheads // nheads_k, (i + heads_k_stride) * nheads // nheads_k)
+ q_i = q[:, :, q_slice]
+ dout_i = dout[:, :, q_slice]
+
+ # Wait for previous all-gather to complete
+ comm.wait()
+ kv_buffer, kv_buffer_copy = kv_buffer_copy, kv_buffer
+
+ # All-gather the next portion of KV buffers if not the last iteration
+ if i < nheads_k - heads_k_stride:
+ kvsl = i + heads_k_stride
+ kvsr = kvsl + heads_k_stride
+ send_k = k[:, :, kvsl:kvsr].contiguous()
+ send_v = v[:, :, kvsl:kvsr].contiguous()
+ comm.all_gather(kv_buffer_copy[0], send_k)
+ comm.all_gather(kv_buffer_copy[1], send_v)
+
+ # Prepare key, value for attention
+ k_i = kv_buffer[0]
+ v_i = kv_buffer[1]
+
+ # Rearrange query, key, value to (b, s, h, d)
+ q_i = einops.rearrange(q_i, 's b h d -> b s h d')
+ k_i = einops.rearrange(k_i, 's b h d -> b s h d')
+ v_i = einops.rearrange(v_i, 's b h d -> b s h d')
+ dout_i = einops.rearrange(dout_i, 's b h d -> b s h d')
+
+ # Backward pass
+ dq_i, _dk_i, _dv_i, _ = eager_attn_bwd(
+ q_i, k_i, v_i, attn_bias, None, ctx.scale, ctx.dropout, outs[i], probs[i], dout_i
+ )
+
+ # Rearrange gradients to (s, b, h, d)
+ dq_i = einops.rearrange(dq_i, 'b s h d -> s b h d')
+ _dk_i = einops.rearrange(_dk_i, 'b s h d -> s b h d')
+ _dv_i = einops.rearrange(_dv_i, 'b s h d -> s b h d')
+ if pg is None:
+ dk_i = _dk_i
+ dv_i = _dv_i
+ else:
+ # Reduce-scatter gradients if CP > 1
+ dk_i = torch.zeros(
+ (k_i.shape[1] // cp_size, k_i.shape[0], k_i.shape[2], k_i.shape[3]),
+ device=k_i.device,
+ dtype=k_i.dtype,
+ )
+ dv_i = torch.zeros(
+ (v_i.shape[1] // cp_size, v_i.shape[0], v_i.shape[2], v_i.shape[3]),
+ device=v_i.device,
+ dtype=v_i.dtype,
+ )
+ torch.distributed.reduce_scatter_tensor(dk_i, _dk_i, group=pg)
+ torch.distributed.reduce_scatter_tensor(dv_i, _dv_i, group=pg)
+
+ # Collect gradients
+ dq.append(dq_i)
+ dk.append(dk_i)
+ dv.append(dv_i)
+
+ # Concatenate gradients and return
+ dq = torch.cat(dq, dim=2)
+ dk = torch.cat(dk, dim=2)
+ dv = torch.cat(dv, dim=2)
+ return dq, dk, dv, None, None, None, None
diff --git a/megatron/core/transformer/enums.py b/megatron/core/transformer/enums.py
index 52b82029f90..d06d58d65f2 100644
--- a/megatron/core/transformer/enums.py
+++ b/megatron/core/transformer/enums.py
@@ -65,3 +65,15 @@ class AttnBackend(enum.Enum):
unfused = 3
local = 4
auto = 5
+
+
+class CudaGraphScope(enum.Enum):
+ """Cuda Graph Scope - defines which parts of the model to capture."""
+
+ full_iteration = 1 # Captures the entire training/inference iteration
+ attn = 2 # Captures attention layers
+ mlp = 3 # Captures MLP layers (dense layers only)
+ moe = 4 # Captures MoE layers (drop-and-pad MoE layers only)
+ moe_router = 5 # Captures MoE router part
+ moe_preprocess = 6 # Captures MoE preprocessing part (requires moe_router)
+ mamba = 7 # Captures Mamba layers
diff --git a/megatron/core/transformer/experimental_attention_variant/dsa.py b/megatron/core/transformer/experimental_attention_variant/dsa.py
new file mode 100644
index 00000000000..353b31e9bcd
--- /dev/null
+++ b/megatron/core/transformer/experimental_attention_variant/dsa.py
@@ -0,0 +1,826 @@
+# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved.
+
+import copy
+import math
+from dataclasses import dataclass
+from typing import Optional, Tuple, Union
+
+import torch
+
+from megatron.core import parallel_state
+from megatron.core.models.common.embeddings import (
+ RotaryEmbedding,
+ YarnRotaryEmbedding,
+ apply_rotary_pos_emb,
+)
+from megatron.core.packed_seq_params import PackedSeqParams
+from megatron.core.process_groups_config import ProcessGroupCollection
+from megatron.core.tensor_parallel.mappings import gather_from_sequence_parallel_region
+from megatron.core.transformer.enums import AttnMaskType
+from megatron.core.transformer.module import MegatronModule
+from megatron.core.transformer.spec_utils import ModuleSpec, build_module
+from megatron.core.transformer.transformer_config import TransformerConfig
+
+try:
+ from fast_hadamard_transform import hadamard_transform
+except ImportError:
+ hadamard_transform = None
+
+
+def rotate_activation(x: torch.Tensor) -> torch.Tensor:
+ """Apply Hadamard rotation activation.
+ Reference:
+ https://github.com/deepseek-ai/DeepSeek-V3.2-Exp/blob/main/inference/model.py#L424-L428
+
+ Args:
+ x: Input tensor (must be bfloat16).
+
+ Returns:
+ Rotated tensor.
+ """
+ assert (
+ x.dtype == torch.bfloat16
+ ), f"rotate_activation only support bf16 input, but got {x.dtype}"
+ assert hadamard_transform is not None, "fast_hadamard_transform is not installed."
+ hidden_size = x.size(-1)
+ return hadamard_transform(x, scale=hidden_size**-0.5)
+
+
+class DSAIndexerLossLoggingHelper:
+ """Helper class for logging sparse attention indexer losses."""
+
+ tracker = {}
+
+ @staticmethod
+ def save_loss_to_tracker(
+ loss: torch.Tensor,
+ layer_number: int,
+ num_layers: int,
+ reduce_group: torch.distributed.ProcessGroup = None,
+ avg_group: torch.distributed.ProcessGroup = None,
+ ):
+ """Save the indexer loss for logging.
+
+ Args:
+ loss: The loss tensor.
+ layer_number: Layer index of the loss, 1-indexed.
+ num_layers: The number of total layers.
+ reduce_group: The group for reducing the loss.
+ avg_group: The group for averaging the loss.
+ """
+ # Skip indexer loss logging if layer_number is None.
+ if layer_number is None:
+ return
+
+ tracker = DSAIndexerLossLoggingHelper.tracker
+ if "values" not in tracker:
+ tracker["values"] = torch.zeros(num_layers, device=torch.cuda.current_device())
+ tracker["values"][layer_number - 1] += loss.detach()
+ tracker["reduce_group"] = reduce_group
+ tracker["avg_group"] = avg_group
+
+ @staticmethod
+ def clean_loss_in_tracker():
+ """Clear the indexer losses."""
+ tracker = DSAIndexerLossLoggingHelper.tracker
+ if "values" in tracker:
+ tracker["values"].zero_()
+ tracker["reduce_group"] = None
+ tracker["avg_group"] = None
+
+ @staticmethod
+ def reduce_loss_in_tracker():
+ """Collect and reduce the indexer losses across ranks."""
+ tracker = DSAIndexerLossLoggingHelper.tracker
+ if "values" not in tracker:
+ return
+ values = tracker["values"]
+
+ torch.distributed.all_reduce(
+ values, group=parallel_state.get_pipeline_model_parallel_group()
+ )
+ # Reduce indexer losses across ranks.
+ if tracker.get('reduce_group') is not None:
+ torch.distributed.all_reduce(values, group=tracker.get('reduce_group'))
+ if tracker.get('avg_group') is not None:
+ torch.distributed.all_reduce(
+ values, group=tracker['avg_group'], op=torch.distributed.ReduceOp.AVG
+ )
+ torch.distributed.all_reduce(
+ values,
+ group=parallel_state.get_data_parallel_group(with_context_parallel=False),
+ op=torch.distributed.ReduceOp.AVG,
+ )
+
+ @staticmethod
+ def track_indexer_metrics(
+ loss_scale: float,
+ iteration: int,
+ writer,
+ wandb_writer=None,
+ total_loss_dict=None,
+ per_layer_logging: bool = False,
+ ):
+ """Track the sparse attention indexer metrics for logging.
+
+ Args:
+ loss_scale: Scale factor for the loss.
+ iteration: Current training iteration.
+ writer: TensorBoard writer.
+ wandb_writer: Weights & Biases writer.
+ total_loss_dict: Dictionary to accumulate total losses.
+ per_layer_logging: Whether to log per-layer losses.
+ """
+ DSAIndexerLossLoggingHelper.reduce_loss_in_tracker()
+ tracker = DSAIndexerLossLoggingHelper.tracker
+ if "values" not in tracker:
+ return
+
+ indexer_loss_values = tracker["values"] * loss_scale
+ num_layers = indexer_loss_values.shape[0]
+
+ # Average across all layers (assuming all layers have sparse attention)
+ avg_indexer_loss = indexer_loss_values.sum() / num_layers
+
+ # Log average loss
+ if total_loss_dict is not None:
+ if "indexer loss" in total_loss_dict:
+ total_loss_dict["indexer loss"] += avg_indexer_loss
+ else:
+ total_loss_dict["indexer loss"] = avg_indexer_loss
+
+ if writer is not None:
+ writer.add_scalar("indexer loss", avg_indexer_loss, iteration)
+
+ if wandb_writer is not None:
+ wandb_writer.log({"indexer loss": avg_indexer_loss}, iteration)
+
+ DSAIndexerLossLoggingHelper.clean_loss_in_tracker()
+
+
+def compute_dsa_indexer_loss(
+ index_scores: torch.Tensor,
+ topk_indices: torch.Tensor,
+ query: torch.Tensor,
+ key: torch.Tensor,
+ softmax_scale: float,
+ loss_coeff: float,
+ sparse_loss: bool,
+ pg_collection: ProcessGroupCollection,
+) -> torch.Tensor:
+ """
+ Compute KL divergence loss between index_scores and true attention_scores.
+
+ This loss trains the indexer to predict which tokens are important by matching the distribution
+ of true attention scores.
+
+ Reference: Section 2.1 of
+ https://github.com/deepseek-ai/DeepSeek-V3.2-Exp/blob/main/DeepSeek_V3_2.pdf
+
+ Args:
+ index_scores: Scores predicted by indexer [batch, seqlen_q, seqlen_k].
+ topk_indices: Top-k indices [batch, seqlen_q, index_topk].
+ query: Query tensor [seqlen_q, batch, heads, dim].
+ key: Key tensor [seqlen_k, batch, heads, dim].
+ softmax_scale: Scale coefficient after q @ k^T.
+ loss_coeff: Coefficient for the indexer KL divergence loss.
+ sparse_loss: bool, whether to use sparse indexer loss. If True, only the topk
+ indices will be used to compute the loss.
+ pg_collection: Process group collection, must have TP process group.
+
+ Returns:
+ index_loss: KL divergence loss (scalar).
+ """
+ sq, b, np, hn = query.size()
+ sk = key.size(0)
+
+ # [sq, b, np, hn] -> [b, np, sq, hn] -> [b * np, sq, hn]
+ query = query.permute(1, 2, 0, 3).reshape(b * np, sq, hn)
+ # [sk, b, np, hn] -> [b, np, hn, sk] -> [b * np, hn, sk]
+ key = key.permute(1, 2, 3, 0).reshape(b * np, hn, sk)
+ # Compute attention scores [b * np, sq, sk]
+ attention_scores = torch.bmm(query.float(), key.float()) * softmax_scale
+ # Reshape to [b, np, sq, sk]
+ attention_scores = attention_scores.reshape(b, np, sq, sk)
+
+ # causal_mask [sq, sk]
+ causal_mask = torch.triu(
+ torch.full((sq, sk), float('-inf'), dtype=torch.float32, device=attention_scores.device),
+ diagonal=1,
+ )
+ # index_mask [b, sq, sk]
+ index_mask = torch.full(
+ (b, sq, sk), float("-inf"), dtype=torch.float32, device=causal_mask.device
+ ).scatter_(-1, topk_indices, 0)
+
+ # [b, np, sq, skv] + [1, 1, sq, skv] -> [b, np, sq, skv]
+ attention_scores += causal_mask.view(1, 1, sq, sk)
+ if sparse_loss:
+ # [b, np, sq, sk] + [b, 1, sq, sk] -> [b, np, sq, sk]
+ attention_scores += index_mask.view(b, 1, sq, sk)
+ # [b, sq, sk] + [b, sq, sk] -> [b, sq, sk]
+ index_scores += index_mask
+
+ # [b, np, sq, sk] -> [b, np, sq, sk]
+ attention_scores = torch.nn.functional.softmax(attention_scores, dim=-1, dtype=torch.float32)
+ # [b, sq, sk] -> [b, sq, sk]
+ index_scores = torch.nn.functional.softmax(index_scores, dim=-1, dtype=torch.float32)
+
+ # Sum attention scores across heads.
+ # [batch, heads, seqlen_q, seqlen_k] -> [batch, seqlen_q, seqlen_k]
+ attention_scores = attention_scores.sum(dim=1)
+ if pg_collection.tp.size() > 1:
+ # attention scores are scattered to TP ranks in head dimension.
+ torch.distributed.all_reduce(attention_scores.contiguous(), group=pg_collection.tp)
+ # L1 normalize target on the last dimension. Doesn't use abs() because attention_scores are
+ # obtained from softmax so they are already non-negative.
+ attention_scores = attention_scores / attention_scores.sum(dim=-1, keepdim=True)
+
+ # Compute KL divergence: KL(target || index) = target(x) * log(target(x) / index(x))
+ # kl_per_element [b, sq, sk]
+ kl_per_element = attention_scores * (
+ torch.log(attention_scores + 1e-10) - torch.log(index_scores + 1e-10)
+ )
+
+ # [b, sq, sk] -> [b, sq] -> [1]
+ # Each token has same weight in the loss.
+ kl_div = kl_per_element.sum(dim=-1).mean()
+
+ # Scale by coefficient.
+ indexer_loss = kl_div * loss_coeff
+
+ return indexer_loss
+
+
+class DSAIndexerLossAutoScaler(torch.autograd.Function):
+ """An AutoScaler that triggers the backward pass and scales the grad for indexer loss.
+
+ This custom autograd function attaches a KL divergence loss to the activation
+ to train the indexer to predict attention scores without affecting the forward pass.
+ """
+
+ main_loss_backward_scale: torch.Tensor = None
+
+ @staticmethod
+ def forward(ctx, output: torch.Tensor, indexer_loss: torch.Tensor):
+ """Preserve the indexer_loss by storing it in the context to avoid garbage collection.
+
+ Args:
+ output: The output tensor (activation).
+ indexer_loss: The indexer KL divergence loss tensor.
+
+ Returns:
+ torch.Tensor: The output tensor unchanged.
+ """
+ ctx.save_for_backward(indexer_loss)
+ return output
+
+ @staticmethod
+ def backward(ctx, grad_output: torch.Tensor):
+ """Compute and scale the gradient for indexer loss.
+
+ Args:
+ grad_output: The gradient of the output.
+
+ Returns:
+ Tuple[torch.Tensor, torch.Tensor]: The gradient of the output, scaled indexer loss
+ gradient.
+ """
+ (indexer_loss,) = ctx.saved_tensors
+ if DSAIndexerLossAutoScaler.main_loss_backward_scale is None:
+ DSAIndexerLossAutoScaler.main_loss_backward_scale = torch.tensor(
+ 1.0, device=indexer_loss.device
+ )
+ indexer_loss_backward_scale = DSAIndexerLossAutoScaler.main_loss_backward_scale
+ scaled_indexer_loss_grad = torch.ones_like(indexer_loss) * indexer_loss_backward_scale
+ return grad_output, scaled_indexer_loss_grad
+
+ @staticmethod
+ def set_loss_scale(scale: torch.Tensor):
+ """Set the scale of the indexer loss.
+
+ Args:
+ scale: The scale value to set.
+ """
+ if DSAIndexerLossAutoScaler.main_loss_backward_scale is None:
+ DSAIndexerLossAutoScaler.main_loss_backward_scale = scale
+ else:
+ DSAIndexerLossAutoScaler.main_loss_backward_scale.copy_(scale)
+
+
+@dataclass
+class DSAIndexerSubmodules:
+ """
+ Configuration class for specifying the submodules of an DSA Indexer.
+
+ Args:
+ linear_wq_b: Linear projection for query bottleneck expansion.
+ linear_wk: Linear projection for key.
+ k_norm: Layer normalization for key.
+ linear_weights_proj: Linear projection for attention weights.
+ """
+
+ linear_wq_b: Union[ModuleSpec, type] = None
+ linear_wk: Union[ModuleSpec, type] = None
+ k_norm: Union[ModuleSpec, type] = None
+ linear_weights_proj: Union[ModuleSpec, type] = None
+
+
+@dataclass
+class DSAttentionSubmodules:
+ """
+ Configuration class for specifying the submodules of DSAttention.
+
+ Args:
+ indexer: DSA Indexer module for computing sparse attention indices.
+ """
+
+ indexer: Union[ModuleSpec, type] = None
+
+
+class DSAIndexer(MegatronModule):
+ """
+ DSA Lightning Indexer for DeepSeek Sparse Attention.
+
+ Computes index scores to identify the top-k most relevant key-value pairs for each query in
+ sparse attention.
+
+ Reference:
+ https://github.com/deepseek-ai/DeepSeek-V3.2-Exp/blob/main/inference/model.py#L431-L480
+ """
+
+ def __init__(
+ self,
+ config: TransformerConfig,
+ submodules: DSAIndexerSubmodules,
+ pg_collection: Optional[ProcessGroupCollection] = None,
+ ) -> None:
+ """Initialize the indexer.
+
+ Args:
+ config (TransformerConfig): The configuration for the transformer model.
+ submodules (DSAIndexerSubmodules): Indexer submodules specification.
+ pg_collection (ProcessGroupCollection, optional): Process groups for the indexer.
+ """
+ super().__init__(config=config)
+ self.hidden_size = self.config.hidden_size
+ self.qk_pos_emb_head_dim = self.config.qk_pos_emb_head_dim
+ self.q_lora_rank = (
+ self.config.q_lora_rank
+ if self.config.q_lora_rank is not None
+ else self.config.hidden_size
+ )
+
+ self.index_n_heads = self.config.dsa_indexer_n_heads
+ self.index_head_dim = self.config.dsa_indexer_head_dim
+ self.index_topk = self.config.dsa_indexer_topk
+
+ self.softmax_scale: float = self.index_head_dim**-0.5
+
+ if pg_collection is None:
+ pg_collection = ProcessGroupCollection.use_mpu_process_groups(required_pgs=['tp', 'cp'])
+ self.pg_collection = pg_collection
+
+ # Initialize Position Embedding.
+ if self.config.rope_type == 'rope':
+ self.rotary_pos_emb = RotaryEmbedding(
+ self.qk_pos_emb_head_dim,
+ rotary_percent=self.config.rotary_percent,
+ rotary_base=self.config.rotary_base,
+ cp_group=self.pg_collection.cp,
+ )
+ elif self.config.rope_type == 'yarn':
+ self.rotary_pos_emb = YarnRotaryEmbedding(
+ self.qk_pos_emb_head_dim,
+ rotary_base=self.config.rotary_base,
+ scaling_factor=self.config.rotary_scaling_factor,
+ original_max_position_embeddings=self.config.original_max_position_embeddings,
+ beta_fast=self.config.beta_fast,
+ beta_slow=self.config.beta_slow,
+ mscale=self.config.mscale,
+ mscale_all_dim=self.config.mscale_all_dim,
+ cp_group=self.pg_collection.cp,
+ )
+ else:
+ raise ValueError(
+ f'Unsupported RoPE type: {self.config.rope_type}, supported types are "rope" and '
+ f'"yarn"'
+ )
+
+ self.linear_wq_b = build_module(
+ submodules.linear_wq_b,
+ self.q_lora_rank,
+ self.index_n_heads * self.index_head_dim,
+ config=self.config,
+ init_method=self.config.init_method,
+ bias=False,
+ skip_bias_add=False,
+ skip_weight_param_allocation=False,
+ parallel_mode="duplicated",
+ )
+
+ self.linear_wk = build_module(
+ submodules.linear_wk,
+ self.hidden_size,
+ self.index_head_dim,
+ config=self.config,
+ init_method=self.config.init_method,
+ bias=False,
+ skip_bias_add=False,
+ skip_weight_param_allocation=False,
+ parallel_mode="duplicated",
+ )
+
+ k_norm_config = copy.copy(self.config)
+ k_norm_config.normalization = "LayerNorm"
+ self.k_norm = build_module(
+ submodules.k_norm,
+ config=k_norm_config,
+ hidden_size=self.index_head_dim,
+ eps=self.config.layernorm_epsilon,
+ )
+
+ self.linear_weights_proj = build_module(
+ submodules.linear_weights_proj,
+ self.hidden_size,
+ self.index_n_heads,
+ config=self.config,
+ init_method=self.config.init_method,
+ bias=False,
+ skip_bias_add=False,
+ skip_weight_param_allocation=False,
+ parallel_mode="duplicated",
+ )
+
+ def _apply_rope(self, x: torch.Tensor, rotary_pos_emb: torch.Tensor, mscale: float):
+ """Apply RoPE to the input tensor."""
+ # x_nope [seqlen, batch, *, index_head_dim - qk_pos_emb_head_dim]
+ # x_pe [seqlen, batch, *, qk_pos_emb_head_dim]
+ x_nope, x_pe = torch.split(
+ x, [self.index_head_dim - self.qk_pos_emb_head_dim, self.qk_pos_emb_head_dim], dim=-1
+ )
+ x_pe = apply_rotary_pos_emb(
+ x_pe,
+ rotary_pos_emb,
+ config=self.config,
+ cu_seqlens=None,
+ mscale=mscale,
+ cp_group=self.pg_collection.cp,
+ )
+ # [seqlen, batch, *, index_head_dim]
+ x = torch.cat([x_nope, x_pe], dim=-1)
+ return x
+
+ def _compute_index_scores(
+ self, q: torch.Tensor, weights: torch.Tensor, k: torch.Tensor
+ ) -> torch.Tensor:
+ """
+ Perform index score using BF16 precision.
+
+ Reference:
+ https://github.com/deepseek-ai/DeepSeek-V3.2-Exp/blob/main/inference/kernel.py#L254-L274
+ This is a BF16 implementation of the `fp8_index` logic:
+ 1. Compute attention scores: q @ k^T;
+ 2. Apply ReLU activation;
+ 3. Weight by attention weights;
+ 4. Sum across attention heads.
+
+ Args:
+ q: BF16 [seqlen_q, batch, index_n_heads, index_head_dim], the query tensor.
+ weights: BF16 [seqlen_q, batch, index_n_heads], the attention weights.
+ k: BF16 [seqlen_k, batch, index_head_dim], the key tensor.
+
+ Returns:
+ index_scores: FP32 [batch, seqlen_q, seqlen_k], the index scores.
+ """
+ # Compute attention scores: q @ k^T
+ # [seqlen_q, batch, index_n_heads, index_head_dim] @ [seqlen_k, batch, index_head_dim]^T
+ # -> [seqlen_q, batch, index_n_heads, seqlen_k]
+ index_scores = torch.einsum('sbhd,tbd->sbht', q.float(), k.float())
+
+ # Apply ReLU activation.
+ index_scores = torch.relu(index_scores)
+
+ # Weight each head by attention weights.
+ # [seqlen_q, batch, index_n_heads, seqlen_k] * [seqlen_q, batch, index_n_heads, 1]
+ # -> [seqlen_q, batch, index_n_heads, seqlen_k]
+ index_scores = index_scores * weights.unsqueeze(-1)
+
+ # Sum across attention heads.
+ # [seqlen_q, batch, index_n_heads, seqlen_k] -> [seqlen_q, batch, seqlen_k]
+ index_scores = index_scores.sum(dim=2)
+
+ # Transpose to [batch, seqlen_q, seqlen_k].
+ index_scores = index_scores.transpose(0, 1)
+
+ return index_scores
+
+ def forward_with_scores(
+ self,
+ x: torch.Tensor,
+ qr: torch.Tensor,
+ mask: Optional[torch.Tensor] = None,
+ packed_seq_params: Optional[PackedSeqParams] = None,
+ ) -> Tuple[torch.Tensor, torch.Tensor]:
+ """
+ Forward pass for DSA Indexer that returns both index scores and top-k indices.
+
+ This is used when KL loss is enabled to compare indexer scores with true attention scores.
+
+ Args:
+ x: hidden states [seqlen, batch, hidden_size].
+ qr: Low-rank query tensor [seqlen, batch, q_lora_rank].
+ mask: Attention mask [batch, seqlen, seqlen].
+ packed_seq_params: Packed sequence parameters for variable length sequences.
+
+ Returns:
+ index_scores: Index scores [batch, seqlen, seqlen].
+ topk_indices: Top-k indices [batch, seqlen, index_topk].
+ """
+ assert packed_seq_params is None, "Packed sequence is not supported for DSAttention"
+
+ # =========================================
+ # Prepare RoPE params
+ # =========================================
+ rotary_seq_len = self.rotary_pos_emb.get_rotary_seq_len(
+ None, None, x, self.config, packed_seq_params
+ )
+ if self.config.rope_type == "rope":
+ rotary_pos_emb = self.rotary_pos_emb(
+ rotary_seq_len, packed_seq_params=packed_seq_params
+ )
+ mscale = 1.0
+ else:
+ rotary_pos_emb, mscale = self.rotary_pos_emb(
+ rotary_seq_len, packed_seq_params=packed_seq_params
+ )
+
+ # =========================================
+ # Gather inputs if sp is enabled
+ # =========================================
+ if self.config.sequence_parallel and self.pg_collection.tp.size() > 1:
+ x = gather_from_sequence_parallel_region(x, group=self.pg_collection.tp)
+ qr = gather_from_sequence_parallel_region(qr, group=self.pg_collection.tp)
+
+ # =========================================
+ # Get sequence length and batch size
+ # =========================================
+ seqlen, bsz, _ = x.size()
+
+ # =========================================
+ # q linear and apply rope to q
+ # =========================================
+ # [seqlen, batch, q_lora_rank] -> [seqlen, batch, index_n_heads * index_head_dim]
+ q, _ = self.linear_wq_b(qr)
+ # [seqlen, batch, index_n_heads * index_head_dim]
+ # -> [seqlen, batch, index_n_heads, index_head_dim]
+ q = q.reshape(seqlen, bsz, self.index_n_heads, self.index_head_dim)
+ q = self._apply_rope(q, rotary_pos_emb, mscale)
+
+ # =========================================
+ # k linear and apply rope to k
+ # =========================================
+ # [seqlen, batch, hidden_size] -> [seqlen, batch, index_head_dim]
+ k, _ = self.linear_wk(x)
+ k = self.k_norm(k)
+ # [seqlen, batch, index_head_dim] -> [seqlen, batch, 1, index_head_dim]
+ k = k.reshape(seqlen, bsz, 1, self.index_head_dim)
+ k = self._apply_rope(k, rotary_pos_emb, mscale)
+ # [seqlen, batch, 1, index_head_dim] -> [seqlen, batch, index_head_dim]
+ k = k.reshape(seqlen, bsz, self.index_head_dim)
+
+ # =========================================
+ # Rotate activation
+ # =========================================
+ q = rotate_activation(q)
+ k = rotate_activation(k)
+
+ # =========================================
+ # Compute index scores
+ # =========================================
+ # [seqlen, batch, hidden_size] -> [seqlen, batch, index_n_heads]
+ weights, _ = self.linear_weights_proj(x)
+ weights = weights * (self.index_n_heads**-0.5) * self.softmax_scale
+ # [batch, seqlen, seqlen]
+ index_scores = self._compute_index_scores(q, weights, k)
+ if mask is not None:
+ assert mask.dtype == index_scores.dtype, "Mask dtype must match index scores dtype"
+ index_scores = index_scores + mask
+
+ # =========================================
+ # Select top-k indices
+ # =========================================
+ topk_k = min(self.index_topk, seqlen)
+ # [batch, seqlen, index_topk]
+ topk_indices = index_scores.topk(topk_k, dim=-1)[1]
+
+ return index_scores, topk_indices
+
+ def forward(
+ self,
+ x: torch.Tensor,
+ qr: torch.Tensor,
+ mask: Optional[torch.Tensor] = None,
+ packed_seq_params: Optional[PackedSeqParams] = None,
+ ):
+ """
+ Forward pass for DSA Indexer.
+
+ Args:
+ x: hidden states [seqlen, batch, hidden_size].
+ qr: Low-rank query tensor [seqlen, batch, q_lora_rank].
+ mask: Attention mask [batch, seqlen, seqlen].
+ packed_seq_params: Packed sequence parameters for variable length sequences.
+
+ Returns:
+ topk_indices: Top-k indices for sparse attention [batch, seqlen, index_topk].
+ """
+ _, topk_indices = self.forward_with_scores(x, qr, mask, packed_seq_params)
+ return topk_indices
+
+
+def unfused_dsa_fn(query, key, value, topk_indices, softmax_scale):
+ """
+ Unfused sparse attention implementation.
+ """
+ sq, b, np, hn = query.size()
+ skv = key.size(0)
+ hnv = value.size(3)
+
+ # ===================================
+ # Raw attention scores [b, np, sq, skv]
+ # ===================================
+ # [sq, b, np, hn] -> [b, np, sq, hn] -> [b * np, sq, hn]
+ query = query.permute(1, 2, 0, 3).reshape(b * np, sq, hn)
+ # [skv, b, np, hn] -> [b, np, hn, skv] -> [b * np, hn, skv]
+ key = key.permute(1, 2, 3, 0).reshape(b * np, hn, skv)
+ # Compute attention scores [b * np, sq, skv]
+ attention_scores = torch.bmm(query.float(), key.float()) * softmax_scale
+ # Reshape to [b, np, sq, skv]
+ attention_scores = attention_scores.reshape(b, np, sq, skv)
+
+ # ===================================
+ # Apply sparse mask from indexer
+ # ===================================
+ # index_mask [b, sq, skv]
+ index_mask = torch.full((b, sq, skv), float("-inf"), device=attention_scores.device)
+ index_mask.scatter_(-1, topk_indices, 0)
+ # causal_mask [sq, skv]
+ causal_mask = torch.triu(
+ torch.full((sq, skv), float('-inf'), dtype=torch.float32, device=index_mask.device),
+ diagonal=1,
+ )
+ # [b, sq, skv] + [1, sq, skv] -> [b, sq, skv]
+ index_mask += causal_mask.view(1, sq, skv)
+ # [b, np, sq, skv] + [b, 1, sq, skv] -> [b, np, sq, skv]
+ attention_scores += index_mask.unsqueeze(1)
+ attention_scores = torch.nn.functional.softmax(attention_scores, dim=-1, dtype=torch.float32)
+
+ # ===================================
+ # Output
+ # ===================================
+ # [skv, b, np, hnv] -> [b, np, skv, hnv] -> [b * np, skv, hnv]
+ value = value.permute(1, 2, 0, 3).reshape(b * np, skv, hnv)
+ # Reshape attention_scores: [b, np, sq, skv] -> [b * np, sq, skv]
+ attention_scores = attention_scores.reshape(b * np, sq, skv)
+ # Compute output: [b * np, sq, hnv]
+ output = torch.bmm(attention_scores.to(value.dtype), value)
+ # Reshape output: [b * np, sq, hnv] -> [b, np, sq, hnv] -> [sq, b, np, hnv]
+ output = output.reshape(b, np, sq, hnv).permute(2, 0, 1, 3).contiguous()
+ # Flatten: [sq, b, np, hnv] -> [sq, b, np * hnv]
+ output = output.reshape(sq, b, np * hnv)
+ return output
+
+
+class DSAttention(MegatronModule):
+ """
+ This module implements sparse attention mechanism using an DSA Indexer to compute top-k
+ attention indices for reducing computational complexity.
+
+ Reference:
+ https://github.com/deepseek-ai/DeepSeek-V3.2-Exp/blob/main/inference/model.py#L491-L597
+ """
+
+ def __init__(
+ self,
+ config: TransformerConfig,
+ submodules: DSAttentionSubmodules,
+ layer_number: int,
+ attn_mask_type: AttnMaskType,
+ attention_type: str,
+ attention_dropout: Optional[float] = None,
+ softmax_scale: Optional[float] = None,
+ k_channels: Optional[int] = None,
+ v_channels: Optional[int] = None,
+ cp_comm_type: str = "p2p",
+ pg_collection: ProcessGroupCollection = None,
+ ):
+ super().__init__(config=config)
+
+ self.layer_number = layer_number
+
+ self.indexer = build_module(
+ submodules.indexer, config=self.config, pg_collection=pg_collection
+ )
+
+ if softmax_scale is None:
+ softmax_scale = 1.0 / math.sqrt(
+ k_channels if k_channels is not None else config.kv_channels
+ )
+ self.softmax_scale = softmax_scale
+
+ def forward(
+ self,
+ query: torch.Tensor,
+ key: torch.Tensor,
+ value: torch.Tensor,
+ x: torch.Tensor,
+ qr: torch.Tensor,
+ attention_mask: torch.Tensor,
+ attn_mask_type: AttnMaskType = None,
+ attention_bias: torch.Tensor = None,
+ packed_seq_params: PackedSeqParams = None,
+ ):
+ """
+ Forward pass for Sparse Attention.
+
+ Args:
+ query: Query tensor [sq, b, np, hn].
+ key: Key tensor [skv, b, np, hn].
+ value: Value tensor [skv, b, np, hnv].
+ x: Original hidden states [sq, b, hidden_size].
+ qr: Low-rank query representation [sq, b, q_lora_rank].
+ attention_mask: Attention mask tensor [b, 1, sq, sk].
+ attn_mask_type: Type of attention mask.
+ attention_bias: Optional attention bias.
+ packed_seq_params: Packed sequence parameters.
+
+ Returns:
+ output: Output tensor [sq, b, hidden_size]
+ """
+ sq, b, np, hn = query.size()
+ skv = key.size(0)
+ hnv = value.size(3)
+
+ # Detach x and qr to prevent gradients of indexer from flowing back to the main model.
+ x = x.detach()
+ qr = qr.detach()
+
+ # Get a FP32 mask with -inf for masked positions.
+ if attn_mask_type is not None:
+ assert attn_mask_type == AttnMaskType.causal, 'Only causal mask is supported for now'
+ # Generate upper triangular mask with -inf above diagonal, 0 elsewhere
+ # torch.triu with diagonal=1 creates upper triangular matrix (excluding main diagonal)
+ # float_mask [sq, skv]
+ float_mask = torch.triu(
+ torch.full((sq, skv), float('-inf'), dtype=torch.float32, device=x.device),
+ diagonal=1,
+ )
+ else:
+ assert attention_mask.shape == (b, 1, sq, skv), 'attention_mask shape mismatch'
+ # [b, 1, sq, skv] -> [b, sq, skv]
+ mask = attention_mask.squeeze()
+ # float_mask [b, sq, skv]
+ float_mask = torch.zeros_like(mask, dtype=torch.float32).masked_fill(
+ mask, float('-inf')
+ )
+
+ # ===================================
+ # Get index scores and top-k indices
+ # ===================================
+ index_scores, topk_indices = self.indexer.forward_with_scores(
+ x, qr, mask=float_mask, packed_seq_params=packed_seq_params
+ )
+
+ # ===================================
+ # Run sparse attention kernel
+ # ===================================
+ output = unfused_dsa_fn(query, key, value, topk_indices, self.softmax_scale)
+
+ # ===================================
+ # Attach indexer loss
+ # ===================================
+ if self.training and torch.is_grad_enabled():
+ # Compute KL divergence loss between indexer scores and true attention scores
+ indexer_loss_coeff = getattr(self.config, 'dsa_indexer_loss_coeff', 0.0)
+ indexer_loss = compute_dsa_indexer_loss(
+ index_scores,
+ topk_indices,
+ query.detach(),
+ key.detach(),
+ self.softmax_scale,
+ indexer_loss_coeff,
+ getattr(self.config, "dsa_indexer_use_sparse_loss", False),
+ self.indexer.pg_collection,
+ )
+ # Save indexer loss for logging
+ if indexer_loss_coeff > 0:
+ DSAIndexerLossLoggingHelper.save_loss_to_tracker(
+ loss=indexer_loss,
+ layer_number=self.layer_number,
+ num_layers=self.config.num_layers,
+ )
+ # Attach loss to output
+ output = DSAIndexerLossAutoScaler.apply(output, indexer_loss)
+
+ return output
diff --git a/megatron/core/transformer/fsdp_dtensor_checkpoint.py b/megatron/core/transformer/fsdp_dtensor_checkpoint.py
index f7a938aff2a..04ec982e6ff 100644
--- a/megatron/core/transformer/fsdp_dtensor_checkpoint.py
+++ b/megatron/core/transformer/fsdp_dtensor_checkpoint.py
@@ -65,6 +65,19 @@ def get_ep_layer_offset(num_experts: int | None = None) -> int:
return local_expert_offset
+def get_total_num_experts(num_experts: int | None = None) -> int:
+ """
+ Get the total number of experts for the current model.
+
+ Args:
+ num_experts: Total number of experts in the model. If None, returns 0.
+
+ Returns:
+ The total number of experts.
+ """
+ return num_experts if num_experts else 0
+
+
def get_expert_index_from_key(key):
"""Extract expert index from various expert key formats.
@@ -101,7 +114,7 @@ def handle_experts_in_state_dict(state_dict, num_experts: int | None = None):
The processed state dictionary with rewritten expert keys.
"""
local_expert_start = get_ep_layer_offset(num_experts)
- local_expert_end = num_experts if num_experts else 0
+ local_expert_end = get_total_num_experts(num_experts)
def should_keep_expert_key(expert_index):
"""Determine if this rank should keep this expert key based on expert index"""
diff --git a/megatron/core/transformer/mlp.py b/megatron/core/transformer/mlp.py
index 5d765484709..98e30887e7b 100644
--- a/megatron/core/transformer/mlp.py
+++ b/megatron/core/transformer/mlp.py
@@ -142,7 +142,7 @@ def __init__(
tp_group=tp_group,
)
- def forward(self, hidden_states, per_token_scale=None):
+ def forward(self, hidden_states, per_token_scale=None, **kwargs):
"""Perform the forward pass through the MLP block."""
# [s, b, 4 * h/p]
nvtx_range_push(suffix="linear_fc1")
diff --git a/megatron/core/transformer/module.py b/megatron/core/transformer/module.py
index 1058a207b12..d68f34ffd0b 100644
--- a/megatron/core/transformer/module.py
+++ b/megatron/core/transformer/module.py
@@ -9,6 +9,7 @@
from megatron.core import parallel_state
from megatron.core.dist_checkpointing.mapping import ShardedStateDict
+from megatron.core.transformer.enums import CudaGraphScope
from megatron.core.transformer.transformer_config import TransformerConfig
from megatron.core.transformer.utils import (
ensure_metadata_has_dp_cp_group,
@@ -167,7 +168,10 @@ def __init__(self, config: TransformerConfig, vp_stage: Optional[int] = None):
assert isinstance(config, TransformerConfig), "config must be a TransformerConfig"
# Enable cuda graphs.
- if config.cuda_graph_impl == "local":
+ if (
+ config.cuda_graph_impl == "local"
+ and CudaGraphScope.full_iteration not in config.cuda_graph_scope
+ ):
from megatron.core.transformer.cuda_graphs import CudaGraphManager
self.cudagraph_manager = CudaGraphManager(config, vp_stage=vp_stage)
@@ -393,7 +397,9 @@ def __init__(self, config: TransformerConfig, module: torch.nn.Module):
self.config = config
self.fp16 = config.fp16
self.bf16 = config.bf16
+ self.vp_size = config.virtual_pipeline_model_parallel_size
self.vp_stage = getattr(module, 'vp_stage', None)
+ self.pg_collection = getattr(module, 'pg_collection', None)
if self.fp16:
self.add_module('module', module.half())
@@ -438,11 +444,23 @@ def forward(self, *inputs, fp32_output=True, **kwargs):
The wrapped module's outputs, potentially upcast to fp32 depending on pipeline stage
and ``fp32_output``.
"""
- if parallel_state.is_pipeline_first_stage(ignore_virtual=False, vp_stage=self.vp_stage):
+ from megatron.core.pipeline_parallel.utils import (
+ is_pp_first_stage,
+ is_pp_last_stage,
+ is_vp_first_stage,
+ is_vp_last_stage,
+ )
+
+ if self.pg_collection is None:
+ pp_group = parallel_state.get_pipeline_model_parallel_group()
+ else:
+ pp_group = self.pg_collection.pp
+ if is_vp_first_stage(self.vp_stage, self.vp_size) and is_pp_first_stage(pp_group):
inputs = fp32_to_float16(inputs, self.float16_convertor)
outputs = self.module(*inputs, **kwargs)
if (
- parallel_state.is_pipeline_last_stage(ignore_virtual=False, vp_stage=self.vp_stage)
+ is_vp_last_stage(self.vp_stage, self.vp_size)
+ and is_pp_last_stage(pp_group)
and fp32_output is True
):
outputs = float16_to_fp32(outputs)
diff --git a/megatron/core/transformer/moe/README.md b/megatron/core/transformer/moe/README.md
index 0a933aed0df..71dfa17fda0 100644
--- a/megatron/core/transformer/moe/README.md
+++ b/megatron/core/transformer/moe/README.md
@@ -1,159 +1,396 @@
# Megatron Core MoE
-Megatron-Core MoE provides comprehensive parallelism strategies, seamlessly integrating Expert Parallelism with tensor, data, sequence, and pipeline parallelism. With MCore v0.9, we've achieved remarkable performance of **468 TFLOPS** for Mixtral 8X7B bf16 training. Additionally, we support state-of-the-art MoE model architectures including DeepSeek-V3 and Qwen-MoE.
-
-### What's New
-- **Support for DeepSeek-V3 architecture**
- - Enable TP for MLA and DeepSeek-V3
- - Enable CP for MLA and DeepSeek-V3
- - Requires TransformerEngine >= 2.5.0
- - Many thanks to [SuperCB](https://github.com/SuperCB) from Xiaohongshu Inc. and [RandMist](https://github.com/RandMist) from WeChat Infra Department, Tencent Inc. for their contributions.
- - Support aux-loss-free load balancing strategy
- - Support node-limited routing
- - Support Multi-Token Prediction (MTP)
- - Batch-level overlapping to hide EP-A2A communication
-- **Support DeepSeek's DeepEP for efficient token dispatching and combining**
-- Support HybridEP for efficient token dispatching and combining within intra-node and MNNVL scenarios.
-- Add fusion for token permutation and unpermutation
-- Support Uneven virtual pipeline parallel split
-- Support output-discarding checkpointing on some submodules
-
-### Parallelism
-- **Expert Parallelism**
- - A specific method of parallelism for MoE models, where experts are partitioned onto different workers and each worker processes a different batch of training samples, each worker process one or more experts for each MoE layer.
-- **3D Parallelism**: Data Parallelism, Tensor Parallelism, Pipeline Parallelism
- - Note: When using MoE with expert parallelism and tensor parallelism, sequence parallelism must be enabled.
-- **Context Parallelism**:
- - Split the sequence dimension to support long context training.
-- **Richer parallel mappings**: EP can be combined with DP/TP/PP/CP for handling larger MoE variants.
-- **MoE Parallel Folding**: Support for setting different parallelism strategies for Attention and MoE components, enabling more flexible and efficient model sharding. See detailed documentation below.
-- **Full distributed optimizer support.**
-
-### Router and Load Balancing
-- Router type:
- - Top-K MLP router
-- Load Balancing algorithms:
- - Sinkhorn (S-BASE)
- - Aux loss / Load balancing loss
- - Aux-loss-free load balancing strategy
-- CUDA fused routing and load balancing kernels
+Megatron Core MoE is a production-ready framework for training large-scale Mixture-of-Experts models, providing the foundational architecture, performance optimizations, and best practices that guide MoE framework development across the industry.
+
+## Table of Contents
+
+- [What's New](#whats-new)
+- [Overview of MCore MoE Supported Features and Architectures](#overview-of-mcore-moe-supported-features-and-architectures)
+- [Quick Start Guide](#quick-start-guide)
+ - [Basic MoE Training](#basic-moe-training-in-megatron-lm)
+ - [Pre-defined Configs for Popular Models](#use-the-pre-defined-config-to-train-the-popular-moe-models)
+ - [General Performance Tips](#general-performance-tips)
+- [Best Practices for High Performance MoE Training](#best-practices-to-achieve-high-performance-on-moe-training)
+ - [Step 1: Find Feasible Parallel Mapping](#step-1-find-the-feasible-parallel-mapping-under-the-memory-capacity-of-the-gpu)
+ - [Step 2: Select Optimal Parallelism Strategy](#step-2-select-optimal-parallelism-strategy)
+ - [Step 3: Enable Performance Features](#step-3-enable-performance-features-based-on-profiling-bottlenecks)
+- [Feature Documentation](#feature-documentation)
+ - [Router and Load Balancing](#router-and-load-balancing)
+ - [Token Dispatching](#token-dispatching)
+ - [Upcycling](#upcycling)
+- [Training Optimizations](#training-optimizations)
+ - [MoE Parallel Folding](#moe-parallel-folding)
+ - [Memory Optimization](#memory-optimization)
+ - [Communication Optimization](#communication-optimization)
+ - [Compute Optimization](#compute-optimization)
+ - [FP8 Training](#fp8-training)
+ - [CUDA Graph](#cuda-graph)
+- [MoE Arguments Reference](#moe-arguments-reference)
+- [Examples](#examples)
+- [Contributing](#contributing)
+- [Citation](#citation)
+
+## What's New
+For latest features and architectures, please refer to the [MCore dev roadmap](https://github.com/NVIDIA/Megatron-LM/issues/1729).
+
+### 🔥 [MCore dev] (2026/01)
+- 🚀 Pipeline-aware fine-grained activation offloading
+- 🚀 Qwen3-Next model support
+- 🚀 Muon and Layer-wise distributed optimizer
+
+### 🔥 [MCore v0.15] (2025/11)
+- 🚀 Add HybridEP backend to Flex Dispatcher(GB200, B200, H100 supported)
+- 🚀 Support FSDP with EP for MoE models
+
+### 🔥 [MCore v0.14] (2025/09)
+- 🚀 Batch-level overlapping to hide EP-A2A communication (--overlap-moe-expert-parallel-comm --delay-wgrad-compute)
+- 🚀 FP8 support for Fine-grained Recomputations
+- Router fusion kernels for MoE models (--moe-router-fusion)
+- Context Parallelism (CP) support for MTP and MLA
+
+### 🔥 [MCore v0.13] (2025/07)
+- Support bf16 dtype for optimizer states to use precision-aware optimizer in TransformerEngine (--use-precision-aware-optimizer)
+- Flexible Asymmetric Virtual Pipeline Parallelism with Custom Pipeline Layout (--pipeline-model-parallel-layout)
+- Add Hybrid Shard Data-Parallel support for MoE models (--num-distributed-optimizer-instances)
+- Fine-grained recomputation to reduce activation memory. (--recompute-modules with --recompute-granularity selective)
+- Memory efficient token permutation by moving the probs multiplication from unpermutation to activation function of GroupedMLP.
+
+### 🔥 [MCore v0.12] (2025/05)
+- Support DeepSeek's DeepEP for efficient token dispatching (--moe-token-dispatcher-type flex --moe-enable-deepep)
+- Support Multi-Token Prediction (MTP) (--mtp-num-layers 1)
+- CUDA Graph support for dropless MoE models with attention only capture (--te-rng-track --external-cuda-graph --cuda-graph-scope attn)
+
+## Overview of MCore MoE Supported Features and Architectures
+
+### Model Support
+- ✅ **DeepSeek**
+ - ✅ DeepSeek-V2
+ - ✅ DeepSeek-V3, including MTP
+- ✅ **Qwen**
+ - ✅ Qwen2-57B-A14B
+ - ✅ Qwen3-30B-A3B
+ - ✅ Qwen3-235B-A22B
+- ✅ **Mixtral**
+ - ✅ Mixtral-8x7B
+ - ✅ Mixtral-8x22B
+
+### Core MoE Functionality
+- ✅ Token dropless MoE (dMoE) - Advanced routing without token dropping
+- ✅ Top-K Router with flexible K selection
+- ✅ Load balancing losses for expert utilization optimization
+
+### Advanced Parallelism
+- ✅ Expert Parallel (EP) with 3D parallelism integration
+- ✅ Full parallelism combo: EP + DP + TP + PP + SP support
+- ✅ Context Parallel (CP) for long sequence MoE training
+- ✅ Parallel Folding Heterogeneous Parallelism Mappings for Efficient Large-Scale MoE Model Training
+- ✅ Distributed Optimizer for MoE (ZeRO-1 equivalent)
### Performance Optimizations
-- (Experimental) **DeepEP** is integrated for efficient token communication in large-scale MoE training.
-- GroupedGEMM when num local experts > 1
- - Supported dtype: bf16
- - Performance improvements for larger MoE models
-- Enable `--tp-comm-overlap` for MoE
-- FP8 training support
-
-### Token Dispatch Mechanism
-- Dropless / No token drop
-- Token drop, with or without padding to capacity
-- Token permutation / Unpermutation fusion
+- ✅ Memory Efficient token permutation
+- ✅ Fine-grained Recomputations (mla, moe, mlp, moe_act, norm)
+- ✅ MLA TP Support for Mixture of Linear Attention
+- ✅ GroupedGEMM and GA Fusion
+- ✅ DP/PP/TP Communication Overlapping
+- ✅ Overlapped Shared Expert execution
+- ✅ Router Fusion optimizations
+- ✅ Token (un)permutation Fusion kernels
+- ✅ cuDNN fused Attention integration
+
+### Hardware & Precision Support
+- ✅ DeepEP support for H100 and B200
+- ✅ GroupedGEMM including FP8/MXFP8 support
+- ✅ FP8 weights with BF16 optimizer states
+- ✅ FP8 training full support
+
+### Developer Experience
+- ✅ MoE Model Zoo with pre-training best practices
+- ✅ Distributed Checkpointing for MoE models
+- ✅ Upcycling Support for model scaling
+- ✅ MCore2HF Converter for ecosystem compatibility
+- ✅ Layer-wise logging for detailed monitoring
+- ✅ Runtime Upcycling capabilities
+
+## Quick Start Guide
+
+### Basic MoE Training in Megatron-LM
+
+To train a top-2 MoE model with 8 experts and auxiliary loss, add the following arguments to your megatron training script:
-### Ease of use
-- Checkpoint converter for Mixtral models, see the [example](https://github.com/NVIDIA/Megatron-LM/tree/main/examples/mixtral) for details.
-- MoE Layer Frequency to customize the hybrid MoE/Dense layer architecture
-- Distributed checkpoining
-- Per-layer logging
-- Upcycling Support
+```bash
+## Set MoE Hidden site
+--num-experts 8
+--moe-shared-expert-intermediate-size: 2048
+## Set router config
+--moe-router-load-balancing-type aux_loss
+--moe-router-topk 2
+--moe-aux-loss-coeff 1e-2
+## Set token dispatcher
+--moe-token-dispatcher-type alltoall
+```
-# User Guide
+Detailed documentation for each feature is available in the [Feature Documentation](#feature-documentation) section.
-## Usage
+### Use the pre-defined config to train the popular MoE models
+We have provided some pre-defined config to train the popular MoE models in the [Megatron-MoE-Model-Zoo](https://github.com/yanring/Megatron-MoE-ModelZoo/tree/main) repository. You can use them as a reference to configure your training script. Currently we have added the config for Mixtral 8x7B, Mixtral 8x22B, DeepSeek-V3, Qwen3-30B-A3B, Qwen3-235B-A22B.
-### Quick Start
-To train a top-2 MoE model with 8 experts and auxiliary loss, include the following arguments:
+### General Performance Tips
+#### Training arguments
+The following flags are general performance flags that can help to achieve higher performance on almost all workloads. Check if you have enabled all of them in your training script.
```bash
---num-experts 8
---expert-model-parallel-size 8
+## Enable DeepEP token dispatcher
+--moe-token-dispatcher-type flex
+--moe-flex-dispatcher-backend deepep
+## Enable GroupedGEMM
--moe-grouped-gemm
+## Enable fusion kernels
+--moe-router-fusion
--moe-permute-fusion
---moe-router-load-balancing-type aux_loss # options: aux_loss, sinkhorn, none. Default is aux_loss.
---moe-router-topk 2
---moe-aux-loss-coeff 1e-2
+--cross-entropy-loss-fusion
+--cross-entropy-fusion-impl te
+
+## Communication optimization
--use-distributed-optimizer
---moe-token-dispatcher-type alltoall
-```
+--overlap-param-gather
+--overlap-grad-reduce
+--tp-comm-overlap
-To enable the token drop mechanism, such as GShard and SwitchTransformer, include the following arguments:
+## Enable manual gc to prevent python jitter
+--manual-gc: true
+--manual-gc-interval: 10
+```
+#### Environment variables
+Below are some environment variables that can be useful.
```bash
---moe-expert-capacity-factor 1.0
---moe-pad-expert-input-to-capacity # Optional
+export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True # Enable expandable segments to prevent memory fragmentation
+export NCCL_NVLS_ENABLE=0 # Disable NVLS to prevent memory overhead
```
+#### Dependencies
+- Use the latest version of [TransformerEngine](https://github.com/NVIDIA/TransformerEngine).
+- Use the latest [NGC PyTorch Docker Image](https://catalog.ngc.nvidia.com/orgs/nvidia/containers/pytorch)
-The following figure illustrates differenting dropping strategies in MCore:
-
-
-
-1. The default dropless strategy will not drop or pad any token.
-2. By setting `--moe-expert-capacity-factor`, the tokens exceed the capacity of expert will be dropped based on their selected probabilities.
- The dropping is performed before the token exchange operation between EP ranks when EP > 1.
- The formula of capacity is `capacity = num_tokens_per_rank * topk * capacity_factor / num_experts`.
-3. By setting `--moe-pad-expert-input-to-capacity`, the experts with tokens less than capacity will be padded to the capacity.
-
-### Fine-tuning Mixtral Models
-Megatron-Core has full support for Mixtral MoE models, and we provide the checkpoint converter for Mixtral models from huggingface format to MCore format.
-
-
-### Distributed Checkpointing
-MCore v0.7 introduced fully parallel and asynchronous saving capabilities to distributed checkpointing,
-which addresses the issues of low efficiency in the traditional checkpoint saving methods.
-It also solved the problem of incompatibility between checkpoints of different parallel mappings in the traditional format.
-With the new distributed checkpointing solution, MCore can achieve flexible parallelism configurations by saving and loading the unified format checkpoints.
-Compared to native PyTorch solution, MCore achieves up to 50x reduction in checkpointing overhead.
-
-From MCore v0.8, MoE supports Distributed Checkpointing, which means users can save and load with any combination of parallelism and it is currently available, including expert parallel.
-1. Loading weight and distributed optimizer states with TPxCPxEPxPP resharding with SequentialMLP is supported in version 0.8.
-2. GroupedMLP weight resharding is supported in version 0.8.0 and optimizer state resharding is supported in version 0.10.0. Switching between GroupedMLP/SequentialMLP when loading and saving is partially supported.
-3. TEGroupedMLP has fully support on distributed checkpointing and is fully exchangable with SequentialMLP in version 0.9.0.
-4. Optimizer state resharding cannot do across EP=1 with EP>1 due to the different optimizer type.
-
-Usage
-- `--ckpt-format torch_dist` The main argument, it will attempt to save and load using distributed checkpointing.
-- `--auto-detect-ckpt-format` With this, it can load both distributed checkpointing and legacy checkpointing.
-
-Checkpoint compatibility across SequentialMLP, GroupedMLP, and TEGroupedMLP:
-```text
- ┌───────────────┐ ┌───────────────┐ ┌───────────────┐
- │ GroupedMLP │ │ SequentialMLP │ │ TEGroupedMLP │
- │ │ │ │ │ │
- │ │ │ │ │ │
- │ ┌───────────┐ │ │ ┌───────────┐ │ │ ┌───────────┐ │
- │ │legacy ckpt│ │ │ │legacy ckpt│ │ │ │legacy ckpt│ │
- │ └─────┬─────┘ │ │ └─────┬─────┘ │ │ └─────┬─────┘ │
- │ ▼ │ │ ▼ │ │ ▼ │
- │ ┌─────────┐ │ │ ┌─────────┐ │ │ ┌─────────┐ │
- │ │dist ckpt│ │ │ │dist ckpt│ │ │ │dist ckpt│ │
-┌──►│ │ weight │ │◄────────►│ │ weight │ │◄────────►│ │ weight │ │◄──┐
-│ │ └─────────┘ │ │ └─────────┘ │ │ └─────────┘ │ │
-└───┼───────────────┼──────────┼───────────────┼──────────┼───────────────┼───┘
- │┌─────────────┐│ │┌─────────────┐│ │┌─────────────┐│
- ││ dist ckpt ││ ││ dist ckpt ││ ││ dist ckpt ││
- ││optim states ││ ││optim states ││◄────────►││optim states ││
- │└─────────────┘│ │└─────────────┘│ │└─────────────┘│
- └───────────────┘ └───────────────┘ └───────────────┘
-```
+## Best Practices to achieve high performance on MoE training
+
+Distributed training involves complex trade-offs between **communication**, **memory**, and **computation**, making it challenging to find an optimal parallelism configuration. This section provides a systematic workflow to help you identify the best parallel mapping for your model and hardware.
+
+### Step 1: Find the feasible parallel mapping under the memory capacity of the GPU
+To find the best parallel mapping, we need to first know the feasible parallel mapping for the model under the memory capacity of the GPU.
+The consumption of memory consists of three parts:
+- Activation memory
+- Weight and gradient memory
+- Optimizer states memory
+Different parallel strategies will shard these tensor memory in different ways.
+
+| Parallel Strategy | Peak Activation Memory | Weight Memory | Optimizer states | Communication (Per-Layer) |
+|:-----------------:|:-------------------------------:|:--------------:|:---------------------------------:|:-------------------------:|
+| TP | 1/N (with SP on) | 1/N | 1/N | High |
+| EP | ~1 (varies with EP balancing) | 1/N in MoELayer| 1/N | Medium |
+| PP | 1 (>1 with virtual pipeline) | 1/N | 1/N | Medium |
+| CP | 1/N | 1 | 1/N (with distributed optimizer) | Medium |
+| DP | 1 | 1 | 1/N (with distributed optimizer) | Low |
+
+We provide the argument of `--fake-init-process-group` to emulate distributed training on one GPU. This is useful to find the feasible parallel mapping under the memory capacity of the GPU. See https://github.com/NVIDIA/Megatron-LM/pull/2254 for detailed usage.
+
+### Step 2: Select Optimal Parallelism Strategy
+
+The optimal parallelism configuration varies based on **model architecture**, **sequence length**, and **hardware platform**. Below are general guidelines to help you achieve high throughput.
+
+#### Guideline 1: Minimize Model Parallelism, Maximize Data Parallelism
+
+| Aspect | Recommendation |
+|--------|----------------|
+| **Goal** | Keep TP/EP/PP as small as possible while avoiding OOM |
+| **Why** | Model parallelism introduces communication overhead that hurts performance |
+| **How** | Use distributed optimizer (`--use-distributed-optimizer`) to shard optimizer states across DP ranks, freeing memory for larger DP size |
+
+#### Guideline 2: Keep EP and TP Communication Within NVLink Domain
+
+| Aspect | Recommendation |
+|--------|----------------|
+| **Goal** | Ensure EP×TP fits within a single node (typically 8 GPUs) |
+| **Why** | EP and TP are communication-intensive; NVLink provides much higher bandwidth than cross-node interconnects |
+| **Scaling** | When scaling beyond one node, prefer PP over expanding TP/EP across nodes |
+
+**Note:**
+For very large MoE models like DeepSeek-V3, the EP communication may exceed the NVLink bandwidth. In this case, consider using 1F1B A2A Overlap to overlap the EP communication.
+
+#### Guideline 3: Use Pipeline Parallelism (PP) for Multi-Node Scaling
+
+| Aspect | Recommendation |
+|--------|----------------|
+| **Goal** | Use PP to distribute layers across nodes while keeping EP×TP within NVLink |
+| **VPP** | Enable Virtual Pipeline Parallelism to reduce pipeline bubbles when `PP ≥ 2` |
+| **Config** | Set `--num-layers-per-virtual-pipeline-stage` to control VPP size |
+
+**VPP Size Tuning:**
+- Valid values: all divisors of `num_layers / PP_size`
+- Example: `num_layers=24, PP=4` → valid VPP sizes: `{1, 2, 3, 6}`
+- Trade-off: Larger VPP = fewer bubbles but more P2P communications
+- Recommendation: A middle value often gives the best balance
+
+#### Guideline 4: Prefer EP over TP for Expert Layers
+
+| EP Advantages | Details |
+|---------------|---------|
+| **Better GEMM efficiency** | Larger local matrix sizes improve GPU utilization |
+| **Lower communication** | EP has less communication overhead than TP for MoE layers |
+| **Simpler computation graph** | Easier to overlap communication with computation |
+| **Token permutation** | When `EP = num_experts`, local token permutation is eliminated |
+
+**Example:** For Mixtral 8x7B, `EP8×TP1` outperforms `EP4×TP2`.
+
+#### Guideline 5: Enable Context Parallelism (CP) for Long Sequences
+
+| Aspect | Recommendation |
+|--------|----------------|
+| **When to use** | Sequence length ≥ 8K tokens |
+| **Key factor** | CP efficiency depends on overlapping communication with computation |
+| **Config** | Set `--context-parallel-size` to partition sequences across GPUs |
+
+### Step 3: Enable Performance Features Based on Profiling Bottlenecks
+
+After establishing a working parallel configuration, profile your training to identify bottlenecks and apply targeted optimizations.
+
+#### Memory Bottleneck
+
+**Symptom**: Forced to use full recomputation or excessively large parallelism degrees to avoid OOM.
+
+**Solutions**:
+| Optimization | Overhead | Config | Reference |
+|--------------|----------|--------|---------|
+| Selective Recomputation | Low | `--recompute-granularity selective --recompute-modules ...` | [Fine-grained Recomputation](#fine-grained-recomputation) |
+| Activation Offloading | Medium | `--fine-grained-activation-offloading --offload-modules ...` | [Fine-grained Activation Offloading](#fine-grained-activation-offloading) |
+| Optimizer Offloading | Medium | `--optimizer-cpu-offload` | --- |
+
+#### Communication Bottleneck
+
+**Symptom**: Profiling shows significant time spent in collective operations.
+
+**Solutions**: Identify which communication is the bottleneck and enable corresponding overlap:
+| Communication Type | Overlap Config |
+|--------------------|----------------|
+| DP gradient reduce | `--overlap-grad-reduce` |
+| DP param gather | `--overlap-param-gather` |
+| TP communication | `--tp-comm-overlap` |
+| EP All-to-All | `--overlap-moe-expert-parallel-comm --delay-wgrad-compute` |
+| PP send/recv | Enable VPP with `--num-layers-per-virtual-pipeline-stage` |
+
+#### CPU Overhead Bottleneck
+
+**Symptom**: Nsight Systems timeline shows gaps between GPU kernels where CPU cannot launch kernels fast enough.
+
+**Solutions**:
+| Optimization | Config |
+|--------------|--------|
+| Disable Python GC | `--manual-gc --manual-gc-interval 100` |
+| Enable CUDA Graphs | `--cuda-graph-impl transformer_engine --cuda-graph-scope attn moe_router moe_preprocess` |
+| Reduce kernel launches | Decrease TP size or increase micro-batch size |
+
+#### Computation Bottleneck
+
+**Symptom**: GPU utilization is low despite no communication or CPU bottlenecks.
+
+**Solutions**:
+| Optimization | Config |
+|--------------|--------|
+| Enable kernel fusions | `--moe-router-fusion --moe-grouped-gemm --moe-permute-fusion` |
+| Use FP8 precision | `--fp8-format e4m3 --fp8-recipe blockwise` |
+
+
+## Feature Documentation
+
+### Router and Load Balancing
+
+Routers determine which expert(s) handle each token. A lightweight MLP scores every token and applies `softmax` or `sigmoid` to compute routing probabilities. The router then selects the top-K experts for each token.
+
+> **Note**: The router logits is better to remain in **FP32** or **FP64** rather than BF16 by --moe-router-dtype fp32. At high expert counts, FP32 precision yields better accuracy because output hidden states of experts are multiplied by router scores and accumulated to get the final output.
+
+#### Router Types
+
+| Router Types | Description | Config |
+|-------------|-------------|----------|
+| **Top-K Router** | Standard routing with configurable K, uses softmax for probability computation | --moe-router-topk 8 |
+| **Group Top-K Router** | Selects top-K expert groups, then routes experts in selected groups | --moe-router-num-groups 8 --moe-router-group-topk 4 |
+| **Router score function** | Score function to calculate the probs from output logits of router | --moe-router-score-function softmax/sigmoid |
+
+#### Load Balancing Strategies
+
+| Strategy | Description | Config |
+|----------|-------------|--------|
+| **aux_loss** | Auxiliary loss for balancing expert usage on a micro-batch | `--moe-router-load-balancing-type aux_loss` |
+| **seq_aux_loss** | Sequence-level auxiliary loss for balancing expert usage on each sequence| `--moe-router-load-balancing-type seq_aux_loss` |
+| **global_aux_loss** | Global auxiliary loss for balancing expert usage on a global batch across all ranks | `--moe-router-load-balancing-type global_aux_loss` |
+| **sinkhorn** | Optimal transport formulation for balancing expert usage | `--moe-router-load-balancing-type sinkhorn` |
+| **aux loss free** | Dynamic bias-based load balancing strategy without auxiliary loss | `--moe-router-enable-expert-bias --moe-router-bias-update-rate 1e-3`|
+| **none** | No load balancing | `--moe-router-load-balancing-type none` |
+
+### Token Dispatching
+
+After routing, tokens are **dispatched** to the GPU hosting the assigned expert. After expert computation, tokens are sent back and **combined** to restore the original sequence.
+
+| Dispatcher | Description | Best For | Config |
+|------------|-------------|----------|--------|
+| **alltoall** | NCCL-based All-to-All communication for token exchange | Standard EP > 1 setups | `--moe-token-dispatcher-type alltoall` |
+| **FlexDispatcher with [DeepEP](https://github.com/deepseek-ai/DeepEP) backend** | Removes redundant tokens during cross-node communication, fuses intra/inter-node communication into single kernel | Cross-node EP, fine-grained MoE (DeepSeek-V3) | `--moe-token-dispatcher-type flex --moe-flex-dispatcher-backend deepep` |
+| **FlexDispatcher with [HybridEP](https://github.com/deepseek-ai/DeepEP/tree/hybrid-ep) backend** | NVIDIA's optimized dispatcher using TMA and IBGDA, fewer SMs, native MNNVL support | GB200 NVL72, Multi-Node NVLink | `--moe-token-dispatcher-type flex --moe-flex-dispatcher-backend hybridep` |
+| **allgather** | Gathers all tokens to each GPU, no inter-GPU token movement | TP-only setups, small EP, large Top-K | `--moe-token-dispatcher-type allgather` |
+
+### Upcycling
+Use `--moe-use-upcycling` to enable upcycling, which loads the dense model from the `--load` directory, converts it to an MoE model at runtime, and starts training. The converted model is saved to the `--save` path before training begins. Upcycling is built on distributed checkpointing, supporting parallel modes different from existing dense checkpoints, such as arbitrary expert parallelism during upcycling.
+
+In addition to the default upcycling strategy, we also support granular upcycling strategy which is a more state-of-the-art upcycling strategy from [our recent research work](https://arxiv.org/abs/2410.07524). For the default upcycling strategy, we duplicate the existing MLP to multiple experts, with each expert starting from a copy of the MLP. For the granular upcycling strategy, we use `--moe-upcycling-granularity` to specify how many times smaller is the expert hidden size compared with the original dense FFN hidden size. For using granular upcycling strategy, please set `--moe-upcycling-granularity` as a positive integer. If this param is set to 1, it means using the default upcycling strategy.
+
+Note: The MoE model structure is defined through script arguments. All MoE-related arguments (such as `--num-experts`) can be customized; however, other model structure arguments must be consistent with those of the dense model. For granular upcycling strategy, the moe's FFN hidden size should be set as dense FFN hidden size divided by `--moe-upcycling-granularity`.
+
+## Training Optimizations
+MoE training faces three fundamental performance bottlenecks: **Memory Wall**, **Communication Wall**, and **Compute Efficiency Wall**. The following optimizations address each of these challenges.
+
+### MoE Parallel Folding
+**The Problem with Traditional Approaches:**
+- Prior MoE frameworks constrain **EP ≤ DP** (Expert Parallelism must be a sub-group of Data Parallelism), which severely limits scalability.
+- Applying the same TP/CP to both attention and MoE is suboptimal:
+ - High TP benefits attention but hurts MoE (small per-expert dims make TP overhead prohibitive)
+ - High CP benefits long-context attention but is unnecessary for MoE (tokens processed independently)
+
+**MoE Parallel Folding** is Megatron Core's solution that **decouples attention and MoE parallelism**:
-Best practices for distributed checkpointing:
-1. Convert a legacy checkpoint to a distributed checkpoint. To achieve this, we can add both `--ckpt-format torch_dist --auto-detect-ckpt-format`, then it will load the legacy one and save as the distributed checkpoint format later when the training progress tries to save checkpoints.
-2. Convert checkpoint of the legacy GroupedMLP to TEGroupedMLP. This is only supported for the weight parts. To achieve this, we can use the above method to convert the legacy checkpoint to a distributed checkpoint of the legacy GroupedMLP. After updating the libraries and using TEGroupedMLP, we can directly load the previously saved checkpoint by adding argument `--no-load-optim`.
+| Parallelism Group | Attention Layers | MoE Layers |
+|-------------------|------------------|------------|
+| **Dimensions** | TP × CP × DP × PP | ETP × EP × EDP × PP |
-### Shared Experts
-MCore v0.9 introduced the shared expert feature. We can enable this feature by setting suitable `--moe-shared-expert-intermediate-size`.
+#### Key Benefits
-The parallelism patterns of the shared experts follow the settings of the dense part, i.e., the attention module. The shared experts are not distributed but replicated in EP ranks.
+1. **Breaks the EP ≤ DP Constraint**
+ - Traditional: TP=4, CP=2, DP=8, PP=4 → max EP=8
+ - With Folding: Same attention config, but MoE uses ETP=1, EP=64, EDP=1 → 8× more expert parallelism
-We also have an experimental feature that tries to overlap the communications and computations in the shared experts and the dispatcher.
-We can set `--moe-shared-expert-overlap` and use `alltoall` dispatcher to enable it.
-The overlapping relies on the envirionment setting `CUDA_DEVICE_MAX_CONNECTIONS=1`.
-The `AllGather` and `ReduceScatter` communications in the shared experts are overlapped with `permute`/`unpermute` in the dispatcher.
-The `MLP` computation part in the shared experts are overlapped with the `AlltoAll` communications in the dispatcher.
-Both the forward and the backward pass can overlap. But to get the overlapping in the backward pass, the PyTorch version should `>= 2.2.0`.
+2. **Reduces Minimum GPU Requirements**
+ - Traditional CP=8, EP=8 requires at least 64 GPUs
+ - With Folding: CP and EP are folded together, only 8 GPUs needed
-### Checkpointing
+3. **Enables Independent Optimization**
+ - Use high TP for attention (memory efficiency)
+ - Use ETP=1 for MoE (better GEMM efficiency, less communication)
+
+4. **Keeps High-Bandwidth Communication in NVLink Domain**
+ - Both CP and EP communication can remain within NVLink domain
+
+> **Reference**: [MoE Parallel Folding: Heterogeneous Parallelism Mappings for Efficient Large-Scale MoE Model Training](https://arxiv.org/abs/2504.14960)
+
+### Memory Optimization
+
+Memory optimization is critical for large-scale MoE training, as MoE models maintain all expert parameters even though only a subset is activated per token.
+
+| Optimization | Description | Config |
+|--------------|-------------|--------|
+| **Fine-grained Recomputation** | Selectively recomputes specific modules (e.g., `mla_up_proj`, `layernorm`, `moe_act`) instead of full layers | `--recompute-granularity selective --recompute-modules mla_up_proj layernorm moe_act` |
+| **Fine-grained Activation Offloading** | Offloads activations to CPU memory, overlapping D2H/H2D transfers with computation | See `docs/source/api-guide/fine_grained_activation_offloading.md` |
+| **Precision-aware Optimizer** | Stores optimizer states (exp_avg, exp_avg_sq) in BF16 instead of FP32, reducing optimizer memory by 50% | `--use-precision-aware-optimizer --exp-avg-dtype bf16 --exp-avg-sq-dtype bf16` |
+| **Optimizer Offloading** | Offloads optimizer states to CPU memory. | `--optimizer-cpu-offload` |
+
+#### Fine-grained Recomputation
A new output-discarding checkpointing method is also supported. This method discards the output memory of certain submodules during the forward pass and recomputes them during the backward pass, which can save memory compared to standard checkpointing. This can be enabled for specific submodules using the `--recompute-granularity selective --recompute-modules [submodule1, submodule2, ...]` argument. The supported submodules are:
* `moe_act`: Recompute the GroupedMLP activation function.
@@ -163,123 +400,214 @@ A new output-discarding checkpointing method is also supported. This method disc
* `mlp`: Recompute the dense MLP submodule (uses standard checkpointing rather than output-discarding) which is useful for hybrid-models like DeepSeek-V3.
* `moe`: Recompute the MoE layer submodule (uses standard checkpointing rather than output-discarding).
-### Upcycling
-Use `--moe-use-upcycling` to enable upcycling, which loads the dense model from the `--load` directory, converts it to an MoE model at runtime, and starts training. The converted model is saved to the `--save` path before training begins. Upcycling is built on distributed checkpointing, supporting parallel modes different from existing dense checkpoints, such as arbitrary expert parallelism during upcycling.
+#### Fine-grained Activation Offloading
-In addition to the default upcycling strategy, we also support granular upcycling strategy which is a more state-of-the-art upcycling strategy from [our recent research work](https://arxiv.org/abs/2410.07524). For the default upcycling strategy, we duplicate the existing MLP to multiple experts, with each expert starting from a copy of the MLP. For the granular upcycling strategy, we use `--moe-upcycling-granularity` to specify how many times smaller is the expert hidden size compared with the original dense FFN hidden size. For using granular upcycling strategy, please set `--moe-upcycling-granularity` as a positive integer. If this param is set to 1, it means using the default upcycling strategy.
+Unlike recomputation (which trades compute for memory), offloading trades **GPU-CPU bandwidth for memory**: activations are transferred to CPU during forward pass and retrieved during backward pass. The key is hiding transfer latency behind computation using asynchronous D2H/H2D transfers.
-Note: The MoE model structure is defined through script arguments. All MoE-related arguments (such as `--num-experts`) can be customized; however, other model structure arguments must be consistent with those of the dense model. For granular upcycling strategy, the moe's FFN hidden size should be set as dense FFN hidden size divided by `--moe-upcycling-granularity`.
+**Key Features:**
+- **Module-level granularity**: Target specific modules rather than entire layers
+- **Computation-offloading overlap**: Asynchronous transfers via independent CUDA streams
+- **Compatible with PP/VPP**: Works with pipeline parallelism and fine-grained recomputation
-### Leverage DeepSeek's DeepEP for High-Performance Cross-Node Token Dispatching
-- [DeepSeek-DeepEP](https://github.com/deepseek-ai/deepep) provides a highly optimized implementation for MoE token dispatching and combining operations, specifically designed for large-scale MoE training scenarios.
-- DeepEP is particularly recommended for training large-scale, fine-grained MoE architectures such as DeepSeek-V3 and other advanced MoE models.
-- To enable DeepEP in your training configuration, simply set `--moe-token-dispatcher-type=flex` and `--moe-flex-dispatcher-backend=deepep` in your command line arguments.
+**Usage**
+```bash
+--fine-grained-activation-offloading
+--offload-modules expert_fc1 moe_act # Choices: attn_norm, core_attn, attn_proj, mlp_norm, expert_fc1, moe_act
+```
-### Integrate HybridEP for High-Performance Intra-Node Token Dispatching
-- [HybridEP](https://github.com/deepseek-ai/DeepEP/tree/hybrid-ep) is developed by NVIDIA as an optimized solution for large-scale MoE (Mixture of Experts) all-to-all communication. It is designed to leverage NVIDIA GPU hardware capabilities, significantly reducing Streaming Multiprocessor (SM) resource usage.
-- HybridEP currently supports intra-node and multi-node NVLink scenarios.
-- To enable HybridEP, set `--moe-token-dispatcher-type=flex` and
- `--moe-flex-dispatcher-backend=hybridep` in your command line arguments.
+For more details, see `docs/source/api-guide/fine_grained_activation_offloading.md`
-### CUDA Graph Support
-CUDA Graph functionality can be enabled through the `--cuda-graph-impl` option. There are two implementations:
+### Communication Optimization
-1. `--cuda-graph-impl=local`: Captures cuda graphs using the MCore-internal cuda graph manager.
-2. `--cuda-graph-impl=transformer_engine`: Captures cuda graphs using the TE `make_graphed_callables()` interface.
+Distributed training introduces communication overhead from various parallelism strategies. Megatron Core supports overlapping communication with computation to hide latency and improve throughput.
-To use `--cuda-graph-impl=transformer_engine`, the user should call related methods `TECudaGraphHelper.create_cudagraphs()` and `TECudaGraphHelper.cuda_graph_set_manual_hooks()` in the training script. Please refer to the usage in `megatron/training/training.py`.
+#### Data Parallel (DP) Communication Overlap
-For MoE models, certain configurations may prevent CUDA Graph capture of MoE layers. Specifically, when `--moe-expert-capacity-factor` and `--moe-pad-expert-input-to-capacity` are not set, the resulting dynamic shapes make MoE layers uncapturable. In such cases, you can still leverage CUDA Graphs for the attention layers (operations in `TransformerLayer._forward_attention()`) by setting `--cuda-graph-scope=attn`, while leaving the MoE layers (operations in `TransformerLayer._forward_mlp()`) unmodified. See the argument description for more usage of `--cuda-graph-scope`.
+With distributed optimizer, DP introduces **reduce-scatter** (gradients) and **all-gather** (parameters) communications, chunked by Transformer layer granularity.
+
+| Optimization | Description | Config |
+|--------------|-------------|--------|
+| **Gradient Reduce Overlap** | Overlaps gradient reduce-scatter with backward computation | `--overlap-grad-reduce` |
+| **Param Gather Overlap** | Overlaps parameter all-gather with forward computation | `--overlap-param-gather` |
+| **BF16 Gradient Reduce** | Reduces gradients in BF16 instead of FP32 for better performance | `--grad-reduce-in-fp32 false` (via mixed precision config) |
+| **FP8 Param Gather** | Conducts parameter all-gather in FP8, reducing overhead by 50% | `--fp8-param-gather` |
+
+#### Tensor Parallel (TP) Communication Overlap
+
+TP with sequence parallelism introduces activation all-gather and reduce-scatter operations. Communications are overlapped in **bulk** (no dependency) or **pipelined** (with dependency) fashion.
+
+| Optimization | Description | Config |
+|--------------|-------------|--------|
+| **TP Comm Overlap** | Enables bulk and pipelined TP communication overlap | `--tp-comm-overlap` |
+
+> **Requirements**: `tensor_model_parallel_size >= 2` and `--sequence-parallel`
+
+#### Pipeline Parallel (PP) Communication Overlap
+
+PP introduces P2P activation sends/receives between pipeline stages. Overlap is automatic in the 1F1B pipelining phase when VPP is enabled.
+
+| Optimization | Description | Config |
+|--------------|-------------|--------|
+| **P2P Comm Overlap** | Overlaps PP P2P communications with non-dependent computations | `--overlap-p2p-comm` (auto-enabled with VPP) |
+| **VPP for Better Overlap** | Increases overlap opportunities by reducing layers per virtual stage | `--num-layers-per-virtual-pipeline-stage` |
+
+#### Expert Parallel (EP) Communication Overlap
+
+EP All-to-All can consume 30-40% of training time without optimization. These features hide or reduce EP communication overhead.
+| Optimization | Description | Config |
+|--------------|-------------|--------|
+| **EP A2A Overlap** | Overlaps All-to-All with computation by merging FWD-BWD passes of adjacent microbatches | `--overlap-moe-expert-parallel-comm --delay-wgrad-compute` |
+| **Shared Expert Overlap** | Runs shared expert computation concurrently with EP token transfer | `--moe-shared-expert-overlap` |
-### Batch-Level EP-A2A hidding
-Enable A2A overlap across different batches inspired by the DSv3 DualPipe implmentation. \
-**Features**
-- Hide ep a2a communication by batch-level overlapping
-- Split weight gradient and activation gradient computations for better overlap with communications
-- Support interleaved pipelined parallelism
-- Support FP8 training
-- Support MTP (`-mtp-num-layers 1` only, multiple MTP layers are not supported yet.)
+> **Requirements for EP A2A Overlap**: `expert_model_parallel_size > 1`, CUDA_DEVICE_MAX_CONNECTIONS > 1.
+### Compute Optimization
+
+Fine-grained MoE produces many small operations that can underutilize GPU resources. These optimizations reduce kernel launch overhead and improve GPU utilization.
+
+| Optimization | Description | Config |
+|--------------|-------------|--------|
+| **Grouped GEMM** | Batches multiple expert GEMM operations into a single kernel call, improving GPU utilization | `--moe-grouped-gemm` |
+| **Router Fusion** | Fuses router projection, top-k selection, softmax, and auxiliary loss into fewer kernels | `--moe-router-fusion` |
+| **Permute Fusion** | Fuses token permutation/unpermutation operations into optimized single kernels | `--moe-permute-fusion` |
+| **FP8 Training** | Uses FP8 Tensor Core operations for faster GEMMs on Hopper/Blackwell GPUs | `--fp8 --fp8-recipe blockwise` |
+
+
+### FP8 Training
+
+FP8 training provides benefits across all three performance walls:
+
+| Wall | FP8 Benefit | Impact |
+|------|-------------|--------|
+| **Compute** | Faster Tensor Core GEMMs | FP8 ops on Hopper/Blackwell are faster than BF16 |
+| **Memory** | 50% activation reduction | Stores linear layer inputs in FP8 instead of BF16 |
+| **Communication** | 50% parameter all-gather | With FP8 primary weights (except MXFP8) |
+
+#### FP8 Recipes
+
+| Recipe | Scaling Granularity | Format | Platform | Use Case |
+|--------|---------------------|--------|----------|----------|
+| **Per-tensor** | Whole tensor | E4M3/E5M2 hybrid | Hopper, Blackwell | Conservative, initial experimentation |
+| **Blockwise** | 1×128 (activations), 128×128 (weights) | E4M3 | Hopper | **Production-proven** (DeepSeek-V3, Minimax-M2) |
+| **MXFP8** | 1×32 | E4M3 + E8M0 scaling | Blackwell | Native hardware support on GB200 |
+
+> **Recommendation**: Use **blockwise FP8** on Hopper for production training. It has been validated at scale on DeepSeek-V3 class models.
+
+#### MoE-Specific FP8 Optimizations
+
+| Optimization | Description | Config |
+|--------------|-------------|--------|
+| **Routing Map Padding** | Pads routing map (not tokens) to align M dimension to 16/32, avoiding per-tensor padding overhead | `--moe-router-padding-for-fp8` |
+| **FP8 Primary Weights** | Casts FP32 master weights directly to FP8, eliminating BF16 intermediate copy | `--fp8-param-gather` (Need additional `--reuse-grad-buf-for-mxfp8-param-ag` for MXFP8) |
+
+
+#### Example Configuration
-**Usage**
```bash
-# Add the following flags to your training scripts
---overlap-moe-expert-parallel-comm
-# [optional] only works with specific TE version
---delay-wgrad-compute
+# Blockwise FP8 on Hopper (recommended for production)
+--fp8-format e4m3
+--fp8-recipe blockwise
+--fp8-param-gather
+--moe-router-padding-for-fp8
+
+# MXFP8 on Blackwell
+--fp8-format e4m3
+--fp8-recipe mxfp8
+--moe-router-padding-for-fp8
+--fp8-param-gather
+--reuse-grad-buf-for-mxfp8-param-ag
```
-### MoE Related Arguments
-| Item | Description |
-| --- | --- |
-| --num-experts | Number of Experts in MoE (None means no MoE) |
-| --expert-model-parallel-size | Degree of expert model parallelism. Default is 1. |
-| --moe-ffn-hidden-size | MoE Feed-Forward Network hidden size. Default is None. |
-
-
- View all MoE related arguments.
-
-| Item | Description |
-| --- | --- |
-| --num-experts | Number of Experts in MoE (None means no MoE) |
-| --expert-model-parallel-size | Degree of expert model parallelism. Default is 1. |
-| --moe-ffn-hidden-size | MoE Feed-Forward Network hidden size. Default is None. |
-| --expert-tensor-parallel-size | Degree of tensor model parallelism of expert layer. Default is same to --tensor-model-parallel-size. |
-| --moe-layer-freq | Frequency between MoE layers and Dense layers. Accepts either: 1) An integer N for 1:N ratio (one expert layer for every N-1 dense layers), 2) A string "N" for the same ratio, or 3) A string with Python list expression for custom patterns like `([1]*3+[0]*1)*3` which gives [1,1,1,0,1,1,1,0,1,1,1,0] where 1=expert layer and 0=dense layer. Examples: `([0]+[1]*23)` for 1 dense layer followed by 23 experts layers, `([1]*3+[0]*2)*2` for three expert layers followed by two dense layers, repeated twice. Default is 1. |
-| --moe-grouped-gemm | When there are multiple experts per rank, launch multiple local GEMM kernels in multiple streams to improve the utilization and performance with GroupedLinear in TransformerEngine. |
-| --moe-router-load-balancing-type | Determines the load balancing strategy for the router. "aux_loss" corresponds to the load balancing loss used in GShard and SwitchTransformer; "seq_aux_loss" corresponds to the load balancing loss used in DeepSeekV2 and DeepSeekV3, which computes the loss for each individual sample; "sinkhorn" corresponds to the balancing algorithm used in S-BASE, and "none" implies no load balancing. The default is "aux_loss". |
-| --moe-router-dtype | Data type for routing computation and expert output weighted averaging. Options are 'fp32' and 'fp64'. This can improve numerical stability, particularly when using a large number of experts. The throughput/memory impact should be negligible when used with --moe-permute-fusion. Default is None (no dtype promotion). |
-| --moe-router-topk | Number of experts to route to for each token. The default is 2. |
-| --moe-router-score-function | Score function for MoE routing. Can be "softmax" or "sigmoid". Default is "softmax". |
-| --moe-router-pre-softmax | Enable pre-softmax routing for MoE, which means softmax is before the top-k selection. By default, softmax is done after top-k. |
-| --moe-router-num-groups | Number of groups to divide experts into for group-limited routing. When using group-limited routing: 1) Experts are divided into equal-sized groups, 2) For each token, a subset of groups are selected based on routing scores (sum of top-2 expert scores within each group), 3) From these selected groups, moe_router_topk experts are chosen. Two common use cases: 1) Device-limited routing: Set equal to expert parallel size (EP) to limit each token to experts on a subset of devices (See DeepSeek-V2: https://arxiv.org/pdf/2405.04434) 2) Node-limited routing: Set equal to number of nodes in EP group to limit each token to experts on a subset of nodes (See DeepSeek-V3: https://arxiv.org/pdf/2412.19437)) |
-| --moe-router-group-topk | Number of selected groups for group-limited routing. |
-| --moe-router-topk-scaling-factor | Scaling factor for routing score in top-k selection, only works when --moe-router-pre-softmax enabled. Defaults to None, which means no scaling. |
-| --moe-router-enable-expert-bias | TopK routing with dynamic per-expert bias in the aux-loss-free load balancing strategy. The routing decision is based on the sum of the routing scores and the expert bias. See https://arxiv.org/abs/2408.15664 for details. |
-| --moe-router-fusion | Enable fusion for MoE TopK routing and aux-loss computation. This is only supported in TransformerEngine 2.7.0 and above. |
-| --moe-router-bias-update-rate | The expert bias is updated based on the number of assigned tokens to each expert in a global batch, where the bias is increased for experts with less assigned tokens and decreased for experts with more assigned tokens. Default is 1e-3 same as that used in DeepSeekV3. |
-| --moe-router-force-load-balancing | (Experimental) Force override routing to balance token distribution using random logits for MoE routers, supporting naive top-k and group-limited top-k. This experimental feature is for benchmarking purposes only! |
-| --moe-router-padding-for-quantization | Pad the routing_map to make sure the number of tokens each expert received is a multiple of 16/32 for FP8/FP4 precision. It is suggested to enable this for dropless training with FP8 precision when num_local_experts > 1. This is a more efficient way to pad for FP8 which eliminates the explicit padding in the GroupedMLP layer. |
-| --moe-aux-loss-coeff | Scaling coefficient for the aux loss: a starting value of 1e-2 is recommended. Default is 0.0. |
-| --moe-z-loss-coeff | Scaling coefficient for the z-loss: a starting value of 1e-3 is recommended. Default is None. |
-| --moe-input-jitter-eps | Add noise to the input tensor by applying jitter with a specified epsilon value. Default is None. |
-| --moe-token-dispatcher-type | Determines the token dispatcher type. Choices are "allgather", "alltoall". Default is "allgather". We recommend using 'alltoall' if expert parallelism is applied. We have upgraded the "alltoall" dispatcher in place during MCore v0.9, while the original implementation renamed as "alltoall_seq" is retained until MCore v0.13.|
-| --moe-flex-dispatcher-backend | (Experimental) Select the backend for the flex token dispatcher. Supported options: "deepep", "hybridep". Enables efficient token dispatching and combining for MoE models. |
-| --moe-per-layer-logging | Enable per-layer logging for MoE, currently supports auxiliary loss and z loss. |
-| --moe-expert-capacity-factor | The capacity factor for each expert, None means no token will be dropped. Default is None. |
-| --moe-pad-expert-input-to-capacity | Pads the input for each expert to match the expert capacity length, effective only after the --moe-expert-capacity-factor is set. |
-| --moe-token-drop-policy | The policy to drop tokens. Can be either "probs" or "position". If "probs", the tokens with the lowest probabilities will be dropped. If "position", tokens at the end of each batch will be dropped. |
-| --moe-layer-recompute | Enable activation checkpointing for moe_layer, should be used when memory is not sufficient. |
-| --moe-permute-fusion | Fuse token rearrangement ops during token dispatching. |
-| --moe-shared-expert-intermediate-size | Set shared expert total ffn hidden size. It should be equal to `num_shared_experts * ffn_size_of_each_shared_expert` if there are multiple shared experts. None means no shared expert. |
-| --moe-shared-expert-overlap | (Experimental, may change) If this is set, the communications/computations in the shared experts and the dispatcher will overlap (The `alltoall` dispatcher is needed.) Otherwise, the shared expert runs after the routed experts. |
-| --moe-use-upcycling | Load the dense model checkpoint, convert it into an MoE model at runtime and start training. The converted model will be saved to the path specified by `--save` before training begins. Upcycling is implemented on the top of distributed checkpointing, so it supports parallel modes different from the dense model.|
-| --overlap-moe-expert-parallel-comm | Enable batch-level overlapping in 1f1b stage. |
-| --delay-wgrad-compute | Enable split dgrad and wgrad for `overlap-moe-expert-parallel-comm` execution. Increasing room to hide communication latency by more finegrained control. |
-| --pipeline-model-parallel-layout | (Experimental, may change) A string containing a Python list expression that defines a custom pipeline model parallel layout. |
-| --moe-upcycling-granularity | This param sepecifics how many times smaller is the expert hidden size compared with the original dense FFN hidden size. For using granular upcycling strategy, please set this param as a positive integer. If this param is set to 1, it means using the default upcycling strategy.|
+> **Note**: For blockwise and MXFP8 recipes with current scaling, training loss curves show negligible difference compared to BF16 baselines.
-
-## MoE training example:
-
-Click here.
+### CUDA Graph
+CUDA Graph functionality can be enabled through the `--cuda-graph-impl` option. There are two implementations:
+
+1. `--cuda-graph-impl=local`: Captures cuda graphs using the MCore-internal cuda graph manager.
+2. `--cuda-graph-impl=transformer_engine`: Captures cuda graphs using the TE `make_graphed_callables()` interface.
+To use `--cuda-graph-impl=transformer_engine`, the user should call related methods `TECudaGraphHelper.create_cudagraphs()` and `TECudaGraphHelper.cuda_graph_set_manual_hooks()` in the training script. Please refer to the usage in `megatron/training/training.py`.
+
+For MoE models, certain configurations may prevent CUDA Graph capture of MoE layers. Specifically, when `--moe-expert-capacity-factor` and `--moe-pad-expert-input-to-capacity` are not set, the resulting dynamic shapes make MoE layers uncapturable. In such cases, you can still leverage CUDA Graphs for the attention layers (operations in `TransformerLayer._forward_attention()`) by setting `--cuda-graph-scope=attn`, while leaving the MoE layers (operations in `TransformerLayer._forward_mlp()`) unmodified. See the argument description for more usage of `--cuda-graph-scope`.
+
+## MoE Arguments Reference
+### Core Arguments
+| Argument | Description | Default |
+|----------|-------------|---------|
+| --num-experts | Number of Experts in MoE | None |
+| --expert-model-parallel-size | Degree of expert model parallelism | 1 |
+| --moe-ffn-hidden-size | MoE FFN hidden size | FFN hidden size of the dense model |
+| --expert-tensor-parallel-size | Expert layer tensor parallelism | Same as TP(Recommeded to set to 1 for fine-grained MoE models) |
+| --moe-layer-freq | MoE layer frequency pattern | 1 |
+
+### Router Arguments
+| Argument | Description | Default |
+|----------|-------------|---------|
+| --moe-router-load-balancing-type | Load balancing: aux_loss, sinkhorn, seq_aux_loss, none | aux_loss |
+| --moe-router-topk | Number of experts per token | 2 |
+| --moe-router-score-function | Score function: softmax, sigmoid | softmax |
+| --moe-router-pre-softmax | Softmax before top-k | False |
+| --moe-router-num-groups | Groups for group-limited routing | None |
+| --moe-router-group-topk | Selected groups in group-limited routing | None |
+| --moe-router-enable-expert-bias | Dynamic per-expert bias | False |
+| --moe-router-bias-update-rate | Bias update rate | 1e-3 |
+| --moe-router-fusion | Enable router fusion | False |
+| --moe-router-dtype | Router precision: fp32, fp64 | None |
+| --moe-router-padding-for-fp8 | Pad for FP8 alignment | False |
+
+### Loss and Regularization
+| Argument | Description | Default |
+|----------|-------------|---------|
+| --moe-aux-loss-coeff | Auxiliary loss coefficient | 0.0 |
+| --moe-z-loss-coeff | Z-loss coefficient | None |
+| --moe-input-jitter-eps | Input jitter epsilon | None |
+
+### Token Dispatching
+| Argument | Description | Default |
+|----------|-------------|---------|
+| --moe-token-dispatcher-type | Dispatcher: allgather, alltoall, flex | allgather |
+| --moe-enable-deepep | Enable DeepEP (with flex) | False |
+| --moe-expert-capacity-factor | Capacity factor | None |
+| --moe-pad-expert-input-to-capacity | Pad to capacity | False |
+| --moe-token-drop-policy | Drop policy: probs, position | probs |
+| --moe-permute-fusion | Fuse permutation ops | False |
+
+### Performance Optimization
+| Argument | Description | Default |
+|----------|-------------|---------|
+| --moe-grouped-gemm | Use GroupedGEMM | False |
+| --overlap-moe-expert-parallel-comm | Batch-level EP overlap | False |
+| --delay-wgrad-compute | Split dgrad/wgrad compute | False |
+| --moe-shared-expert-intermediate-size | Shared expert FFN size | None |
+| --moe-shared-expert-overlap | Overlap shared expert | False |
+
+### Memory and Checkpointing
+| Argument | Description | Default |
+|----------|-------------|---------|
+| --moe-layer-recompute | Recompute MoE layer | False |
+| --moe-use-upcycling | Enable upcycling | False |
+| --moe-upcycling-granularity | Upcycling granularity | 1 |
+
+### Miscellaneous
+| Argument | Description | Default |
+|----------|-------------|---------|
+| --moe-per-layer-logging | Per-layer logging | False |
+| --moe-router-force-load-balancing | Force load balancing (experimental) | False |
+
+## Examples
```bash
#!/bin/bash
# Runs Mixtral 8x7B model on 32 H100/A100 GPUs
-# The Dropless MoE suffers from an imbalanced token distribution at the early stage of training (the first few hundred iterations), which may lead to poor performance and out-of-memory (OOM) issues.
-# To check the performance of a Dropless MoE model, we should run the model for at least 500 iterations or resume from trained checkpoints.
export CUDA_DEVICE_MAX_CONNECTIONS=1
GPUS_PER_NODE=8
-# Change for multinode config
MASTER_ADDR=${MASTER_ADDR:-"localhost"}
MASTER_PORT=${MASTER_PORT:-"6000"}
-NNODES=${NNODES:-"1"}
+NNODES=${NNODES:-"4"}
NODE_RANK=${RANK:-"0"}
WORLD_SIZE=$(($GPUS_PER_NODE*$NNODES))
@@ -319,11 +647,12 @@ MODEL_ARGS=(
MOE_ARGS=(
--num-experts 8
--expert-model-parallel-size 8
- --moe-router-load-balancing-type aux_loss # options: aux_loss, sinkhorn, None. Default is aux_loss.
+ --moe-router-load-balancing-type aux_loss
--moe-router-topk 2
--moe-aux-loss-coeff 1e-2
--moe-grouped-gemm
--moe-permute-fusion
+ --moe-token-dispatcher-type alltoall
)
DATA_ARGS=(
@@ -358,24 +687,17 @@ MODEL_PARALLEL_ARGS=(
)
LOGGING_ARGS=(
- --log-interval 1 \
- --save-interval 10000 \
- --eval-interval 1000 \
- --eval-iters 10 \
- --save $CHECKPOINT_PATH \
- --load $CHECKPOINT_PATH \
- --tensorboard-dir "${CHECKPOINT_PATH}/tensorboard" \
- --no-load-optim \
- --no-load-rng
+ --log-interval 1
+ --save-interval 10000
+ --eval-interval 1000
+ --eval-iters 10
+ --save $CHECKPOINT_PATH
+ --load $CHECKPOINT_PATH
+ --tensorboard-dir "${CHECKPOINT_PATH}/tensorboard"
+ --ckpt-format torch_dist
+ --auto-detect-ckpt-format
)
-if [ -n "${WANDB_API_KEY}" ]; then
- LOGGING_ARGS+=(
- --wandb-project ${WANDB_PROJECT:-"Mixtral-Finetuning"}
- --wandb-exp-name ${WANDB_NAME:-"Mixtral_8x7B"}
- )
-fi
-
torchrun ${DISTRIBUTED_ARGS[@]} pretrain_gpt.py \
${MODEL_ARGS[@]} \
${MOE_ARGS[@]} \
@@ -384,107 +706,36 @@ torchrun ${DISTRIBUTED_ARGS[@]} pretrain_gpt.py \
${MODEL_PARALLEL_ARGS[@]} \
${LOGGING_ARGS[@]}
```
+
-# Performance Best Practice
+## Contributing
-### Tuning Guide of Parallel Mappings
+We welcome contributions! Please see [CONTRIBUTING.md](../../../../CONTRIBUTING.md) for guidelines.
-To find a good parallel mapping that help you achieve a high throughput of a new model, there are some general rule that could help. Here is an overview of properties in different aspects for each parallel strategy.
+## Support
-| Parallel Strategy | Peak Activation Memory | Weight Memory | Optimizer states | Communication (Per-Layer) |
-|:-----------------:|:-------------------------------:|:--------------:|:---------------------------------:|:-------------------------:|
-| TP | 1/N (with SP on) | 1/N | 1/N | High |
-| EP | 1 | 1/N in MoELayer| 1/N | Medium |
-| PP | 1 (>1 with virtual pipeline) | 1/N | 1/N | Medium |
-| CP | 1/N | 1 | 1/N (with distributed optimizer) | Medium |
-| DP | 1 | 1 | 1/N (with distributed optimizer) | Low |
+- GitHub Issues: [Report bugs or request features](https://github.com/NVIDIA/Megatron-LM/issues)
+- Documentation: [Full documentation](https://docs.nvidia.com/megatron-core/developer-guide/latest/index.html)
-For a specific model, the best parallel mapping varies based on the model architecture, trained sequence length and the hardware platform.
-Here we provide some general rules to get better performance:
-1. Keep the model parallism size as small as possible.
- - For the large language models, model parallism is often required to prevent OOM, but it will bring communication overhead and hurt performance.
- - With distributed optimizer, master weights and optimizer states will be sharded across all DP ranks with slight communication overhead.
- So try to reduce the model parallism size and increase data parallism size when there are lots of free GPU memory during training.
-2. Ensure the EPxTP communication winthin the NVLink domain.
- - Communications of EP and TP should remain within the NVLink domain as much as possible, as both are communication-intensive.
- - If the model is too large and requires scaling across multiple nodes, consider PP before TP and EP. See item 3 for details.
-3. Use Pipeline Parallelism to scale the model further.
- - Enable Virtual Pipeline Parallelism(VPP) to reduce pp bubbles when PP_size >= 2 by setting `num_layers_per_virtual_pipeline_stage`.
- - VPP_size tuning: the legal values of vpp_size are all common divisors of num_layers/pp_size, E.g., num_layers=24, pp_size=4, then we can pick vpp_size from {1, 2, 3, 6}. The larger the vpp_size, the lower the pipeline bubbles, while the larger number of P2P communications between each PP stages. Empirically a value in the middle often gives the best trade-off. `VPP_size=num_layers / PP_size / num_layers_per_virtual_pipeline_stage`
-4. Prefer EP over TP for the expert layer when possible:
- - TP saves more memory than EP, but EP can achieve better GEMM efficiency and less communication overhead than TP.
- - If EP size increased to the number of expert, the local token permutation/un-permutation for experts computation are omitted.
- - Simplify the computation graph of MoE layers, more convenient for performing potential comm-computation overlapping.
- - In practice, EP8TP1 is better than EP4TP2 for 8x7B.
-5. Enable Context Parallelism for long context training.
- - The efficiency of CP largely depends on whether its communication can be overlapped with computation.
- - Empirically, use CP when sequence length >= 8K.
-### MoE Parallel Folding
+## Citation
-MoE Parallel Folding separates the MoE related parallel groups from Dense groups.
-1. Traditional MoE parallel groups are entangled with dense by using a 5-dimension parallel group generator with default order `tp-cp-ep-dp-pp`. The EP group in MoE is a sub-group of DP in Attention.
-2. With MoE Parallel Folding, we use a parallel group generator with `tp-cp-dp-pp` for Attention, and another with `tp-ep-dp-pp` for MoE. The EPxTP group in MoE is a sub-group of DPxCPxTP in Attention.
-
-By setting `--expert-tensor-parallel-size`, we can set MoE-specific TP size.
-
-#### Advantages of MoE Parallel Folding
-1. The CP and EP group are folded together by defualt, such that:
- 1. It reduces the minimal required GPUs to turn on both CP and EP. For example, the traditional way with (CP=8, EP=8) needs at least 64 GPUs, for now it only requires 8 GPUs.
- 2. The CP and EP communication can be both put in the NVLink domain.
-2. We can set different TP sizes for Attention and MoE part.
- 1. For MoE, EP is often more efficient than TP. But in the traditional way, only using EP can get OOM for most models.
- 2. With MoE parallel folding, we can turn on TP for Attention part and setting TP=1 for MoE models, which often gets better MFU.
-
-### End-to-End Training Practice
-**Use the latest NVIDIA PyTorch or NeMo Docker Image**
-- [NGC PyTorch Image](https://catalog.ngc.nvidia.com/orgs/nvidia/containers/pytorch)
-- [NGC NeMo Image](https://catalog.ngc.nvidia.com/orgs/nvidia/containers/nemo)
-
-**Token Dispatcher Choices**
-- Token Dispatcher sends tokens to the designated expert, involves tensor rearangement and communications.
-- Dispatcher `allgather` is the default option. It achieves better performance and efficiency when only tensor parallelism is used or when the Top-k value is very large.
-- Dispatcher `alltoall` is recommended if expert parallelism is applied.
-- Dispatcher `flex` is a new dispatcher decouples communication group from model parallelism. It supports two backends(DeepEP and HybridEP) selectable via `--moe-flex-dispatcher-backend`.
-
-**Enable Communication Overlap**
-- Enable `--overlap-param-gather` and `--overlap-grad-reduce` with distributed optimizer.
-- Enable `--tp-comm-overlap` when TP>1.
-- Enable p2p comm overlap when PP > 1 by setting `num_layers_per_virtual_pipeline_stage`.
-
-**Enable GroupedGEMM when num_local_experts>1 with `--moe-grouped-gemm`**
-- GroupedGEMM has higher efficiency than vanilla sequential GEMMs for each expert.
-- Recommend to use the TE version of Grouped GEMM (by upgrading to MCore v0.8 and TE v1.9), which support Gradient Accumulation Fusion and FP8 Training.
-
-**OOM Caused by Token Distribution Imbalance when Training From Scratch**
-MoE suffers from a severe load imbalance issue when the router is under-trained, leading to the model easily running out of memory (OOM), which typically occurs in the first 100~300 steps when training from scratch.
-Therefore, there are two recommended ways during the first 200 steps to avoid the OOM problem, which can be removed after the token distribution is more stable:
-1. Increase the `expert-tensor-parallel-size` and decrease `expert-model-parallel-size` to replace EP with TP in MoELayer, this can prevent the load imbalancing between EP ranks. Since current ETP implementation has some memeory overhead, you can further enable activation recomputation only for MoE Layer by adding `--moe-layer-recompute`.
-2. Setting capacity factor to a relatively small number like 1.0 by adding `--moe-token-capacity-factor 1.0`.
-
-**Leverage DeepSeek's DeepEP for High-Performance Cross-Node Token Dispatching**
-- The primary advantage of DeepEP is its cross-node token communication efficiency, which delivers substantial performance improvements when deploying expert parallelism across multiple nodes with large TopK values.
-- To enable DeepEP in your training configuration, simply set `--moe-token-dispatcher-type=flex` and `--moe-enable-deepep` in your command line arguments.
-
-**FP8 Training Best Practice**
-- Using latest version of [TransformerEngine](https://github.com/NVIDIA/TransformerEngine).
-- Enable router padding with `--moe-router-padding-for-quantization` to reduce padding overhead.
-- Enable native FP8 weights with `--fp8-param-gather` to reduce weights memory cost.
-
-### Reference Best Parallel Mapping
-
-Here are the reference parallel mappings of MCore v0.8 for Mixtral 8x7B and 8x22B models:
-| Model | Vocab Size| Dispatcher | Precision | #GPUs | SEQ LEN | TP | EP | PP | VP | MBS | GBS |
-|:-----------------------:|:---------:|:----------:|:---------:|:-----:|:-------:|:--:|:--:|:--:|:--:|:---:|:---:|
-| Mixtral 8x7B(Dropless) | 32K | All-to-All | BF16 | 64 | 4096 | 1 | 8 | 4 | 8 | 1 | 256 |
-| Mixtral 8x22B(Dropless) | 32K | All-to-All | BF16 | 128 | 4096 | 4 | 2 | 8 | 7 | 1 | 256 |
-
-Detailed Benchmark Information:
-Server:
-- 8xH100 80GB HBM3
-- NVLink 4th Generation
-- InfiniBand 8x400 Gbit/s
-
-Docker Image:
-- PyTorch 24.09 with TransformerEngine v1.11
+If you use Megatron-Core MoE in your research, please cite:
+
+```bibtex
+
+@article{megatron-lm,
+ title={Megatron-LM: Training Multi-Billion Parameter Language Models Using Model Parallelism},
+ author={Shoeybi, Mohammad and Patwary, Mostofa and Puri, Raul and LeGresley, Patrick and Casper, Jared and Catanzaro, Bryan},
+ journal={arXiv preprint arXiv:1909.08053},
+ year={2019}
+}
+
+@article{moe-parallel-folding,
+ title={MoE Parallel Folding: Heterogeneous Parallelism Mappings for Efficient Large-Scale MoE Model Training with Megatron Core},
+ author={Liu, Dennis and Yan, Zijie and Yao, Xin and Liu, Tong and Korthikanti, Vijay and Wu, Evan and Fan, Shiqing and Deng, Gao and Bai, Hongxiao and Chang, Jianbin and Aithal, Ashwath and Andersch, Michael and Shoeybi, Mohammad and Yao, Jiajie and Zhou, Chandler and Wu, David and Li, Xipeng and Yang, June},
+ year={2025},
+ journal={arXiv preprint arXiv:2504.14960},
+}
+```
diff --git a/megatron/core/transformer/moe/experts.py b/megatron/core/transformer/moe/experts.py
index 3f6fdffdd87..615e12e09d6 100644
--- a/megatron/core/transformer/moe/experts.py
+++ b/megatron/core/transformer/moe/experts.py
@@ -1,9 +1,9 @@
-# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved.
+# Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
import copy
import logging
from copy import deepcopy
-from functools import partial, wraps
+from functools import partial
from math import ceil
from typing import Optional, Tuple
@@ -11,7 +11,7 @@
import torch.nn.functional as F
from torch.nn.parameter import Parameter
-from megatron.core import parallel_state, tensor_parallel
+from megatron.core import tensor_parallel
from megatron.core.activations import squared_relu
from megatron.core.dist_checkpointing import ShardedTensor
from megatron.core.dist_checkpointing.mapping import (
@@ -25,9 +25,13 @@
from megatron.core.fusions.fused_bias_swiglu import weighted_bias_swiglu_impl
from megatron.core.fusions.fused_weighted_squared_relu import weighted_squared_relu_impl
from megatron.core.jit import jit_fuser
+from megatron.core.pipeline_parallel.fine_grained_activation_offload import (
+ FineGrainedActivationOffloadingInterface as off_interface,
+)
from megatron.core.tensor_parallel.layers import (
_initialize_affine_weight_cpu,
_initialize_affine_weight_gpu,
+ set_tensor_model_parallel_attributes,
)
from megatron.core.tensor_parallel.utils import divide
from megatron.core.transformer.mlp import MLP, MLPSubmodules, apply_swiglu_sharded_factory
@@ -44,7 +48,7 @@
make_sharded_object_for_checkpoint,
sharded_state_dict_default,
)
-from megatron.core.utils import deprecated
+from megatron.core.utils import deprecated, internal_api
try:
import transformer_engine as te # pylint: disable=unused-import
@@ -111,6 +115,8 @@ class GroupedMLP(MegatronModule):
Executes multiple experts in parallel to maximize computational efficiency.
"""
+ # TODO(M4): breaking api, switched from pass in tp_group to pass in pg_collection.
+ @internal_api
def __init__(
self,
num_local_experts: int,
@@ -218,6 +224,14 @@ def activation_func_with_probs(x, probs):
rank=tp_rank,
world_size=tp_size,
)
+ else:
+ # Ensure TP attrs are set even when not initializing
+ set_tensor_model_parallel_attributes(
+ tensor=self.weight1, is_parallel=True, dim=1, stride=1
+ )
+ set_tensor_model_parallel_attributes(
+ tensor=self.weight2, is_parallel=True, dim=0, stride=1
+ )
else:
self.weight1 = Parameter(
torch.empty(
@@ -242,6 +256,14 @@ def activation_func_with_probs(x, probs):
_initialize_affine_weight_gpu(
self.weight2, config.output_layer_init_method, partition_dim=0, is_expert=True
)
+ else:
+ # Ensure TP attrs are set even when not initializing
+ set_tensor_model_parallel_attributes(
+ tensor=self.weight1, is_parallel=True, dim=1, stride=1
+ )
+ set_tensor_model_parallel_attributes(
+ tensor=self.weight2, is_parallel=True, dim=0, stride=1
+ )
setattr(self.weight1, 'allreduce', not self.expert_parallel)
setattr(self.weight2, 'allreduce', not self.expert_parallel)
@@ -264,6 +286,7 @@ def forward(
permuted_probs: torch.Tensor,
):
"""Forward step of the GroupedMLP."""
+ assert self.config.bf16, "Currently GroupedGEMM for MoE only supports bf16."
if self.activation_recompute:
self.activation_checkpoint = tensor_parallel.CheckpointWithoutOutput()
@@ -556,6 +579,8 @@ class TEGroupedMLP(MegatronModule):
Executes multiple experts in parallel to maximize computational efficiency.
"""
+ # TODO(M4): breaking api, switched from pass in tp_group to pass in pg_collection.
+ @internal_api
def __init__(
self,
num_local_experts,
@@ -578,7 +603,6 @@ def __init__(
if self.config.gated_linear_unit:
ffn_hidden_size *= 2
- # TODO(Hepteract): pass pg_collection to submodule after refactoring Linear modules
self.linear_fc1 = build_module(
submodules.linear_fc1,
self.num_local_experts,
@@ -590,7 +614,7 @@ def __init__(
skip_bias_add=False,
is_expert=True,
tp_comm_buffer_name='fc1',
- tp_group=pg_collection.expt_tp,
+ pg_collection=pg_collection,
)
if self.config.use_te_activation_func and not (submodules.activation_func is None):
@@ -598,7 +622,6 @@ def __init__(
else:
self.activation_func = self.config.activation_func
- # TODO(Hepteract): pass pg_collection to submodule after refactoring Linear modules
self.linear_fc2 = build_module(
submodules.linear_fc2,
self.num_local_experts,
@@ -614,7 +637,17 @@ def __init__(
skip_bias_add=True,
is_expert=True,
tp_comm_buffer_name='fc2',
- tp_group=pg_collection.expt_tp,
+ pg_collection=pg_collection,
+ )
+
+ self.offload_expert_fc1 = (
+ self.config.fine_grained_activation_offloading
+ and "expert_fc1" in self.config.offload_modules
+ )
+
+ self.offload_moe_act = (
+ self.config.fine_grained_activation_offloading
+ and "moe_act" in self.config.offload_modules
)
self.activation_recompute = (
@@ -626,6 +659,12 @@ def __init__(
set_save_original_input(self.linear_fc2)
+ # This is to avoid the CPU overhead of multiple d2h copies
+ if self.offload_expert_fc1:
+ from megatron.core.extensions.transformer_engine import set_save_original_input
+
+ set_save_original_input(self.linear_fc1)
+
if self.config.fp8 or self.config.fp4:
assert HAVE_TE, "FP8 and FP4 requires TE."
self.quantization_padding = Fp8Padding(self.num_local_experts)
@@ -690,9 +729,18 @@ def forward(
# Probs already applied, so reset to 1.
permuted_probs = torch.ones_like(permuted_probs)
- intermediate_parallel, bias_parallel = self.linear_fc1(
- permuted_local_hidden_states, tokens_per_expert
- )
+ with off_interface(
+ self.offload_expert_fc1, permuted_local_hidden_states, "expert_fc1"
+ ) as permuted_local_hidden_states:
+ fc1_output, bias_parallel = self.linear_fc1(
+ permuted_local_hidden_states, tokens_per_expert
+ )
+ if self.offload_expert_fc1:
+ fc1_output = off_interface.group_commit(
+ fc1_output,
+ name="expert_fc1",
+ forced_released_tensors=[permuted_local_hidden_states],
+ )
def bias_act_func(intermediate_parallel, bias_parallel, permuted_probs):
if self.config.use_te_activation_func:
@@ -754,16 +802,24 @@ def glu(x):
if self.activation_recompute:
self.activation_checkpoint = tensor_parallel.CheckpointWithoutOutput()
- intermediate_parallel = self.activation_checkpoint.checkpoint(
- bias_act_func, intermediate_parallel, bias_parallel, permuted_probs
- )
- output, output_bias = self.linear_fc2(intermediate_parallel, tokens_per_expert)
- self.activation_checkpoint.discard_output_and_register_recompute(output)
+ with off_interface(self.offload_moe_act, fc1_output, "moe_act") as fc1_output:
+ bias_act_output = self.activation_checkpoint.checkpoint(
+ bias_act_func, fc1_output, bias_parallel, permuted_probs
+ )
else:
- intermediate_parallel = bias_act_func(
- intermediate_parallel, bias_parallel, permuted_probs
+ with off_interface(self.offload_moe_act, fc1_output, "moe_act") as fc1_output:
+ bias_act_output = bias_act_func(fc1_output, bias_parallel, permuted_probs)
+
+ output, output_bias = self.linear_fc2(bias_act_output, tokens_per_expert)
+ if self.activation_recompute:
+ self.activation_checkpoint.discard_output_and_register_recompute(output)
+
+ # Delay the offload of the moe act until after the linear_fc2 has been computed
+ # to make sure the fc1_output is reloaded to GPU before recomputing moe_act.
+ if self.offload_moe_act:
+ output = off_interface.group_commit(
+ output, name="moe_act", forced_released_tensors=[fc1_output]
)
- output, output_bias = self.linear_fc2(intermediate_parallel, tokens_per_expert)
# upad and concat the output
if self.config.fp8 or self.config.fp4:
@@ -832,6 +888,8 @@ class SequentialMLP(MegatronModule):
This class executes each expert sequentially.
"""
+ # TODO(M4): breaking api, switched from pass in tp_group to pass in pg_collection.
+ @internal_api
def __init__(
self,
num_local_experts,
diff --git a/megatron/core/transformer/moe/fused_a2a.py b/megatron/core/transformer/moe/fused_a2a.py
index a2e2c321e36..aa13b9b5b5b 100644
--- a/megatron/core/transformer/moe/fused_a2a.py
+++ b/megatron/core/transformer/moe/fused_a2a.py
@@ -321,6 +321,14 @@ def init_hybrid_ep_buffer(
)
+def reset_hybrid_ep_buffer():
+ '''
+ Reset the HybridEP buffer
+ '''
+ global _hybrid_ep_buffer
+ _hybrid_ep_buffer = None
+
+
@internal_api
class HybridEPDispatch(torch.autograd.Function):
'''
diff --git a/megatron/core/transformer/moe/moe_layer.py b/megatron/core/transformer/moe/moe_layer.py
index 0b0ee9b6850..e17cebcf1f9 100644
--- a/megatron/core/transformer/moe/moe_layer.py
+++ b/megatron/core/transformer/moe/moe_layer.py
@@ -1,4 +1,4 @@
-# Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved.
+# Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
from abc import ABC, abstractmethod
from dataclasses import dataclass
@@ -9,7 +9,12 @@
from megatron.core import parallel_state, tensor_parallel, utils
from megatron.core.process_groups_config import ProcessGroupCollection
from megatron.core.transformer.module import MegatronModule
-from megatron.core.transformer.moe.moe_utils import get_default_pg_collection
+from megatron.core.transformer.moe.moe_utils import (
+ MoECudaGraphPartialCaptureSignal,
+ MoECudaGraphTensorStore,
+ get_default_pg_collection,
+ maybe_skip_or_early_return_by_cudagraph,
+)
from megatron.core.transformer.moe.router import TopKRouter
from megatron.core.transformer.moe.token_dispatcher import (
MoEAllGatherTokenDispatcher,
@@ -19,6 +24,7 @@
)
from megatron.core.transformer.spec_utils import ModuleSpec, build_module
from megatron.core.transformer.transformer_config import TransformerConfig
+from megatron.core.utils import internal_api
try:
import transformer_engine as te # pylint: disable=unused-import
@@ -189,21 +195,36 @@ def __init__(
# Initialize shared experts
if self.use_shared_expert:
self.shared_experts = build_module(
- self.submodules.shared_experts, config=self.config, pg_collection=pg_collection
+ self.submodules.shared_experts,
+ config=self.config,
+ pg_collection=pg_collection,
+ gate=self.config.moe_shared_expert_gate,
)
if self.shared_expert_overlap:
self.token_dispatcher.set_shared_experts(self.shared_experts)
- def router_and_preprocess(self, hidden_states: torch.Tensor):
- """Compute and preprocess token routing for dispatch.
+ # Cudagraph tensor store for resuming the forward pass from the end of the cudagraph.
+ self.cudagraph_tensor_store = MoECudaGraphTensorStore()
+
+ @maybe_skip_or_early_return_by_cudagraph("route")
+ def route(self, hidden_states: torch.Tensor, padding_mask: Optional[torch.Tensor] = None):
+ """Compute token routing for preprocessing.
This method uses the router to determine which experts to send each token to,
- producing routing probabilities and a mapping. It then preprocesses the
- hidden states and probabilities for the token dispatcher. The original
- hidden states are returned as a residual connection.
+ producing routing probabilities and a mapping.
+ """
+ probs, routing_map = self.router(hidden_states, padding_mask=padding_mask)
+ return probs, routing_map
+
+ @maybe_skip_or_early_return_by_cudagraph("preprocess")
+ def preprocess(
+ self, hidden_states: torch.Tensor, probs: torch.Tensor, routing_map: torch.Tensor
+ ):
+ """Preprocess token routing for dispatch.
+
+ This method preprocesses the hidden states and routing probabilities for the token
+ dispatcher.
"""
- residual = hidden_states
- probs, routing_map = self.router(hidden_states)
# Project the hidden_states from hidden dimension down to latent dimenion.
if self.config.moe_latent_size:
assert (
@@ -213,16 +234,18 @@ def router_and_preprocess(self, hidden_states: torch.Tensor):
hidden_states, probs = self.token_dispatcher.dispatch_preprocess(
hidden_states, routing_map, probs
)
- return hidden_states, probs, residual
+ return hidden_states, probs
def dispatch(self, hidden_states: torch.Tensor, probs: torch.Tensor):
"""Dispatches tokens to assigned expert ranks via communication.
+
This method performs the actual communication (e.g., All-to-All) to distribute
tokens and their associated probabilities to the devices hosting their assigned
experts.
"""
return self.token_dispatcher.token_dispatch(hidden_states, probs)
+ @maybe_skip_or_early_return_by_cudagraph("shared_experts_compute")
def shared_experts_compute(self, hidden_states: torch.Tensor):
"""Computes the output of the shared experts.
@@ -250,9 +273,8 @@ def shared_experts_compute(self, hidden_states: torch.Tensor):
return shared_expert_output
- def routed_experts_compute(
- self, hidden_states: torch.Tensor, probs: torch.Tensor, residual: torch.Tensor
- ):
+ @internal_api
+ def routed_experts_compute(self, hidden_states: torch.Tensor, probs: torch.Tensor):
"""Computes the output of the routed experts on the dispatched tokens.
This method first post-processes the dispatched input to get permuted tokens
@@ -285,7 +307,7 @@ def combine(self, output: torch.Tensor, shared_expert_output: Optional[torch.Ten
output = output + shared_expert_output
return output
- def forward(self, hidden_states: torch.Tensor):
+ def forward(self, hidden_states: torch.Tensor, padding_mask: Optional[torch.Tensor] = None):
"""Forward pass for the MoE layer.
The forward pass comprises four main steps:
@@ -295,7 +317,11 @@ def forward(self, hidden_states: torch.Tensor):
4. Combine: The outputs from the experts are combined and returned.
Args:
- hidden_states (torch.Tensor): The input tensor to the MoE layer.
+ hidden_states (torch.Tensor): The input tensor shape [seq_length, bsz, hidden_size].
+ padding_mask (torch.Tensor, optional): Boolean mask indicating padding positions.
+ used for correct auxiliary loss computation for packed sequence.
+ Shape = [bsz, seq_length]. True = padding (exclude), False = valid (include).
+ Defaults to None (all tokens are valid).
Returns:
A tuple containing the output tensor and the MLP bias, if any.
@@ -306,12 +332,26 @@ def forward(self, hidden_states: torch.Tensor):
"are enabled without also enabling sequence parallelism."
)
+ # Transpose from [bsz, seq_length] to [seq_length, bsz] to align with hidden_states
+ if padding_mask is not None:
+ padding_mask = padding_mask.transpose(0, 1).bool()
+
# MoE forward: route -> dispatch -> compute -> combine
- def custom_forward(hidden_states):
- shared_expert_output = self.shared_experts_compute(hidden_states)
- hidden_states, probs, residual = self.router_and_preprocess(hidden_states)
+ def custom_forward(hidden_states, padding_mask=None):
+ try:
+ shared_expert_output = self.shared_experts_compute(hidden_states)
+ probs, routing_map = self.route(hidden_states, padding_mask=padding_mask)
+ hidden_states, probs = self.preprocess(hidden_states, probs, routing_map)
+ except MoECudaGraphPartialCaptureSignal as e:
+ # This signal is raised from the maybe_skip_or_early_return_by_cudagraph decorator.
+ # It means we should early-return from the MoE layer forward pass.
+ # This happens when we are partially capturing the CUDA graph of the MoE layer,
+ # like cuda_graph_scope=["moe_router", "moe_preprocess"].
+ # We need to return the intermediate tensors as CUDA graph outputs.
+ return e.get_early_return_outputs(hidden_states, shared_expert_output)
+
dispatched_input, probs = self.dispatch(hidden_states, probs)
- output, mlp_bias = self.routed_experts_compute(dispatched_input, probs, residual)
+ output, mlp_bias = self.routed_experts_compute(dispatched_input, probs)
assert mlp_bias is None, f"mlp_bias is not supported for {type(self.token_dispatcher)}"
output = self.combine(output, shared_expert_output)
@@ -319,24 +359,28 @@ def custom_forward(hidden_states):
if self.moe_layer_recompute:
if self.config.fp8 or self.config.fp4:
- output, mlp_bias = te_checkpoint(
+ outputs = te_checkpoint(
custom_forward,
False,
tensor_parallel.random.get_cuda_rng_tracker,
parallel_state.get_tensor_model_parallel_group(),
hidden_states,
+ padding_mask,
)
else:
- output, mlp_bias = tensor_parallel.checkpoint(custom_forward, False, hidden_states)
+ outputs = tensor_parallel.checkpoint(
+ custom_forward, False, hidden_states, padding_mask
+ )
else:
- output, mlp_bias = custom_forward(hidden_states)
+ outputs = custom_forward(hidden_states, padding_mask)
- return output, mlp_bias
+ return outputs
- def backward_dw(self):
+ def backward_dw(self, routed_experts: bool = True, shared_experts: bool = False):
"""Compute weight gradients for experts and shared experts."""
- self.experts.backward_dw()
- if self.use_shared_expert and not self.shared_expert_overlap:
+ if routed_experts:
+ self.experts.backward_dw()
+ if shared_experts and self.use_shared_expert and not self.shared_expert_overlap:
self.shared_experts.backward_dw()
def set_for_recompute_pre_mlp_layernorm(self):
diff --git a/megatron/core/transformer/moe/moe_utils.py b/megatron/core/transformer/moe/moe_utils.py
index f8b7d234fff..e5e06f05758 100644
--- a/megatron/core/transformer/moe/moe_utils.py
+++ b/megatron/core/transformer/moe/moe_utils.py
@@ -1,6 +1,7 @@
# Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
-
+import functools
import math
+from dataclasses import dataclass
from typing import List, Optional, Union
import torch
@@ -9,8 +10,17 @@
from megatron.core.fp4_utils import get_fp4_align_size
from megatron.core.fp8_utils import get_fp8_align_size
from megatron.core.process_groups_config import ProcessGroupCollection
+from megatron.core.tensor_parallel import (
+ get_cuda_rng_tracker,
+ get_data_parallel_rng_tracker_name,
+ get_expert_parallel_rng_tracker_name,
+)
+from megatron.core.tensor_parallel.mappings import reduce_from_tensor_model_parallel_region
from megatron.core.transformer.cuda_graphs import is_graph_capturing
+from megatron.core.transformer.enums import CudaGraphScope
+from megatron.core.transformer.moe.router_replay import RouterReplay
from megatron.core.transformer.transformer_config import TransformerConfig
+from megatron.core.utils import internal_api
try:
import transformer_engine as te # pylint: disable=unused-import
@@ -116,18 +126,34 @@ def switch_load_balancing_loss_func(
return aux_loss
-def z_loss_func(logits, z_loss_coeff):
+def z_loss_func(logits, z_loss_coeff, padding_mask: Optional[torch.Tensor] = None):
"""Encourages the router's logits to remain small to enhance stability.
Please refer to the ST-MoE paper (https://arxiv.org/pdf/2202.08906.pdf) for details.
Args:
logits (torch.Tensor): The logits of the router.
+ z_loss_coeff (float): The coefficient for the z-loss.
+ padding_mask (torch.Tensor, optional): Boolean mask indicating padding positions.
+ Shape [num_tokens]. True = padding (exclude),
+ False = valid (include). Defaults to None.
Returns:
torch.Tensor: The logits after applying the z-loss.
"""
+ logsum = torch.logsumexp(logits, dim=-1)
+ z_loss_values = torch.square(logsum)
+
+ if padding_mask is not None:
+ # Invert padding_mask: True (padding) -> 0, False (valid) -> 1
+ valid_mask = ~padding_mask
+ # Only compute z_loss for valid (non-padding) tokens
+ z_loss_values = z_loss_values * valid_mask
+ # Compute mean over valid tokens only
+ num_valid_tokens = valid_mask.sum()
+ z_loss = z_loss_values.sum() / torch.clamp(num_valid_tokens, min=1.0) * z_loss_coeff
+ else:
+ z_loss = torch.mean(z_loss_values) * z_loss_coeff
- z_loss = torch.mean(torch.square(torch.logsumexp(logits, dim=-1))) * z_loss_coeff
return z_loss
@@ -167,6 +193,28 @@ def get_capacity(num_tokens: int, num_experts: int, capacity_factor: float, min_
return capacity
+def get_tokens_per_expert_and_token_count(
+ routing_map: torch.Tensor,
+ reduce_group: torch.distributed.ProcessGroup,
+ topk: int = None,
+ with_padding_mask: bool = False,
+) -> torch.Tensor:
+ """
+ Compute global_tokens_per_expert, local_num_tokens and total_num_tokens with padding mask.
+ """
+ local_tokens_per_expert = routing_map.sum(dim=0)
+ global_tokens_per_expert = reduce_from_tensor_model_parallel_region(
+ local_tokens_per_expert, reduce_group
+ )
+ if with_padding_mask:
+ local_num_tokens = local_tokens_per_expert.sum() / topk
+ total_num_tokens = global_tokens_per_expert.sum() / topk
+ else:
+ local_num_tokens = routing_map.shape[0]
+ total_num_tokens = local_num_tokens * reduce_group.size()
+ return global_tokens_per_expert, local_num_tokens, total_num_tokens
+
+
class MoEAuxLossAutoScaler(torch.autograd.Function):
"""An AutoScaler that triggers the backward pass and scales the grad for auxiliary loss."""
@@ -533,6 +581,7 @@ def topk_routing_with_score_function(
score_function: str = "softmax",
expert_bias: Optional[torch.Tensor] = None,
fused: bool = False,
+ router_replay: Optional['RouterReplay'] = None,
):
"""Compute the routing probabilities and map for top-k selection with score function.
Args:
@@ -544,6 +593,9 @@ def topk_routing_with_score_function(
scaling_factor (float): Scaling factor of routing score in top-k selection.
score_function (str): The score function to use. Can be either "softmax" or "sigmoid".
expert_bias (torch.Tensor): The bias added to logits for expert routing.
+ router_replay (Optional['RouterReplay']): For debugging and development, allows for
+ deterministic routing by replaying a previously
+ recorded routing sequence.
Returns:
Tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
- routing_probs (torch.Tensor): A tensor of shape [num_tokens, num_experts] containing
@@ -570,7 +622,7 @@ def topk_routing_with_score_function(
expert_bias=expert_bias,
)
- def compute_topk(scores, topk, num_groups=None, group_topk=None):
+ def _compute_topk(scores, topk, num_groups=None, group_topk=None):
if group_topk:
return group_limited_topk(
scores=scores,
@@ -583,6 +635,15 @@ def compute_topk(scores, topk, num_groups=None, group_topk=None):
else:
return torch.topk(scores, k=topk, dim=1)
+ def compute_topk(scores, topk, num_groups=None, group_topk=None):
+ # Default behavior if no replay is active
+ if router_replay is None:
+ return _compute_topk(scores, topk, num_groups=num_groups, group_topk=group_topk)
+ else:
+ return router_replay.get_replay_topk(
+ scores, topk, num_groups, group_topk, _compute_topk
+ )
+
if score_function == "softmax":
if use_pre_softmax:
scores = torch.softmax(logits, dim=-1, dtype=torch.float32).type_as(logits)
@@ -625,35 +686,48 @@ def compute_topk(scores, topk, num_groups=None, group_topk=None):
def compute_routing_scores_for_aux_loss(
- logits: torch.Tensor, topk: int, score_function: str, fused: bool = False
+ logits: torch.Tensor,
+ topk: int,
+ score_function: str,
+ fused: bool = False,
+ padding_mask: Optional[torch.Tensor] = None,
):
"""Compute routing scores based on the score function.
Args:
logits (torch.Tensor): The logits tensor after gating, shape: [num_tokens, num_experts].
-
+ padding_mask (torch.Tensor, optional): Boolean mask indicating padding positions.
+ Shape [num_tokens]. True = padding (exclude),
+ False = valid (include). Defaults to None.
Returns:
- torch.Tensor: The normalized routing scores.
+ Tuple[torch.Tensor, torch.Tensor]: routing_map and scores.
"""
if fused:
if not HAVE_TE or fused_compute_score_for_moe_aux_loss is None:
raise ValueError(
"fused_compute_score_for_moe_aux_loss is not available. Please install TE >= 2.6.0."
)
- return fused_compute_score_for_moe_aux_loss(
+ routing_map, scores = fused_compute_score_for_moe_aux_loss(
logits=logits, topk=topk, score_function=score_function
)
-
- if score_function == "softmax":
- scores = torch.softmax(logits, dim=-1, dtype=torch.float32)
- elif score_function == "sigmoid":
- scores = torch.sigmoid(logits)
- scores = scores / (scores.sum(dim=-1, keepdim=True) + 1e-20)
else:
- raise ValueError(f"Invalid score_function: {score_function}")
+ if score_function == "softmax":
+ scores = torch.softmax(logits, dim=-1, dtype=torch.float32)
+ elif score_function == "sigmoid":
+ scores = torch.sigmoid(logits)
+ scores = scores / (scores.sum(dim=-1, keepdim=True) + 1e-20)
+ else:
+ raise ValueError(f"Invalid score_function: {score_function}")
+
+ _, top_indices = torch.topk(scores, k=topk, dim=1)
+ routing_map = torch.zeros_like(logits).int().scatter(1, top_indices, 1).bool()
- _, top_indices = torch.topk(scores, k=topk, dim=1)
- routing_map = torch.zeros_like(logits).int().scatter(1, top_indices, 1).bool()
+ # Apply padding mask to scores if provided
+ if padding_mask is not None:
+ # Invert padding_mask and make True indicates valid tokens
+ valid_mask = (~padding_mask).unsqueeze(-1)
+ routing_map = routing_map * valid_mask
+ scores = scores * valid_mask
return routing_map, scores
@@ -764,18 +838,29 @@ def clear_aux_losses_tracker():
tracker[name]["values"].zero_()
-def reduce_aux_losses_tracker_across_ranks(track_names: Optional[List[str]] = None):
+def reduce_aux_losses_tracker_across_ranks(
+ track_names: Optional[List[str]] = None, pg_collection: Optional[ProcessGroupCollection] = None
+):
"""Collect and reduce the auxiliary losses across ranks."""
tracker = get_moe_layer_wise_logging_tracker()
if track_names is None:
track_names = tracker.keys()
+
+ if pg_collection is None:
+ # Use parallel_state groups
+ pp_group = parallel_state.get_pipeline_model_parallel_group()
+ dp_group = parallel_state.get_data_parallel_group(
+ with_context_parallel=False, partial_data_parallel=False
+ )
+ else:
+ pp_group = pg_collection.pp
+ dp_group = pg_collection.dp
+
for name in track_names:
values = tracker[name]["values"]
# TODO(Hepteract): delete the usage of the global parallel_state.
# Collect aux losses across PP.
- torch.distributed.all_reduce(
- values, group=parallel_state.get_pipeline_model_parallel_group()
- )
+ torch.distributed.all_reduce(values, group=pp_group)
# Reduce aux losses across ranks.
if tracker[name].get('reduce_group') is not None:
torch.distributed.all_reduce(values, group=tracker[name].get('reduce_group'))
@@ -791,6 +876,11 @@ def reduce_aux_losses_tracker_across_ranks(track_names: Optional[List[str]] = No
torch.distributed.all_reduce(
values, group=tracker[name]['avg_group'], op=torch.distributed.ReduceOp.AVG
)
+ # Average aux losses across data parallel ranks.
+ # The `global_load_balancing_loss` already uses `tp_dp_cp_group` in `reduce_group`,
+ # so we don't need to reduce it again. Others use `tp_cp_group` in `reduce_group`.
+ if name != "global_load_balancing_loss":
+ torch.distributed.all_reduce(values, group=dp_group, op=torch.distributed.ReduceOp.AVG)
def track_moe_metrics(
@@ -805,6 +895,7 @@ def track_moe_metrics(
num_layers: Optional[int] = None,
moe_layer_freq: Optional[Union[int, List[int]]] = None,
mtp_num_layers: Optional[int] = None,
+ pg_collection: Optional[ProcessGroupCollection] = None,
):
"""Track the MoE metrics for logging."""
# Aux loss logging
@@ -819,7 +910,8 @@ def track_moe_metrics(
tracker[key]["reduce_group"] = None
tracker[key]["avg_group"] = None
tracker[key]["reduce_group_has_dp"] = False
- reduce_aux_losses_tracker_across_ranks(track_names)
+
+ reduce_aux_losses_tracker_across_ranks(track_names, pg_collection=pg_collection)
# Get number of MoE layers
if moe_layer_freq is None:
@@ -914,6 +1006,7 @@ def get_moe_layer_wise_logging_tracker():
return _MOE_LAYER_WISE_LOGGING_TRACKER
+@internal_api
class RandomSTE(torch.autograd.Function):
"""
Straight-Through Estimator(STE) function that returns random values
@@ -922,26 +1015,14 @@ class RandomSTE(torch.autograd.Function):
This is used to generate random logits of router for load-balanced benchmark.
"""
- generator = None
- random_logits = None
-
@staticmethod
def forward(ctx, logits):
"""
Forward pass returns random logits with rank-specific seed.
"""
- if is_graph_capturing() and RandomSTE.random_logits is not None:
- return RandomSTE.random_logits
-
- if RandomSTE.generator is None:
- global_rank = torch.distributed.get_rank()
- base_seed = 42
- seed = base_seed + global_rank
- RandomSTE.generator = torch.Generator(device=logits.device)
- RandomSTE.generator.manual_seed(seed)
-
- RandomSTE.random_logits = logits.clone().normal_(generator=RandomSTE.generator)
- return RandomSTE.random_logits
+ with get_cuda_rng_tracker().fork(get_expert_parallel_rng_tracker_name()):
+ random_logits = logits.clone().normal_()
+ return random_logits
@staticmethod
def backward(ctx, grad_output):
@@ -958,6 +1039,54 @@ def apply_random_logits(logits):
return RandomSTE.apply(logits)
+@internal_api
+class RandomSTEShared(torch.autograd.Function):
+ """
+ STE that generates random values with shared seed across all ranks.
+ When std < 0, caches and reuses values per layer.
+ """
+
+ _cache = {}
+
+ @staticmethod
+ def forward(ctx, logits, std, layer_number):
+ """Forward pass: apply random bias to logits."""
+ # Check cache if reuse mode (negative std)
+ if std < 0 and layer_number in RandomSTEShared._cache:
+ return logits + RandomSTEShared._cache[layer_number]
+
+ # Generate random bias with shared seed across all ranks
+ with get_cuda_rng_tracker().fork(get_data_parallel_rng_tracker_name()):
+ bias = torch.empty(logits.shape[-1], device=logits.device, dtype=logits.dtype).normal_(
+ std=abs(std)
+ )
+
+ # Cache if reuse mode
+ if std < 0 and layer_number is not None:
+ RandomSTEShared._cache[layer_number] = bias
+
+ return logits + bias
+
+ @staticmethod
+ def backward(ctx, grad_output):
+ """Backward pass: pass through gradients."""
+ return grad_output, None, None
+
+
+def apply_biased_logits(logits, std, layer_number=None):
+ """
+ Apply random bias to logits. All ranks get the same random values.
+
+ Args:
+ logits: Input logits tensor [num_tokens, num_experts]
+ std: Standard deviation for random bias. If negative, generate once
+ per layer and reuse (using abs(std) as actual std).
+ layer_number: Layer number for caching when std is negative.
+ """
+ logits = apply_random_logits(logits)
+ return RandomSTEShared.apply(logits, std, layer_number)
+
+
class RouterGatingLinearFunction(torch.autograd.Function):
"""
Autograd function for router gating linear.
@@ -1059,3 +1188,229 @@ def get_default_pg_collection():
with_context_parallel=True
)
return pg_collection
+
+
+class MoECudaGraphPartialCaptureSignal(Exception):
+ """
+ Used to early-return from a MoE layer forward pass in CUDA graph capture.
+ This signal is raised when we are partially capturing the CUDA graph of the MoE layer,
+ and the related intermediate tensors are recorded in self.kwargs.
+ Call self.get_early_return_outputs() to collect the CUDA graph outputs.
+ """
+
+ def __init__(self, moe_layer, return_step: str, **kwargs):
+ self.moe_layer = moe_layer
+ self.return_step = return_step
+ self.kwargs = kwargs
+
+ def get_early_return_outputs(
+ self, hidden_states: torch.Tensor, shared_expert_output: torch.Tensor
+ ):
+ """
+ Get the CUDA graph early return outputs for the MoE layer, including the intermediate
+ tensors and the intermediate attributes of the token dispatcher.
+
+ The returned output tensors are in the order of:
+ - routed experts path outputs
+ - hidden states, probs, and routing map for capturing router
+ - hidden states and probs for capturing router and preprocess
+ - intermediate attributes of the token dispatcher (if capturing the preprocess step)
+ - shared expert path output (if exists)
+ """
+ if self.return_step == "route":
+ # Capturing the router step returns three intermediate tensors:
+ # hidden states, routing probabilities, and routing map.
+ outputs = [hidden_states, self.kwargs['probs'], self.kwargs['routing_map']]
+ elif self.return_step == "preprocess":
+ # Capturing the preprocess step returns two intermediate tensors:
+ # hidden states and routing probabilities.
+ # It also returns the intermediate attributes of the token dispatcher, recorded in
+ # "token_dispatcher.cudagraph_attrs".
+ outputs = [self.kwargs['hidden_states'], self.kwargs['probs']]
+ valid_cudagraph_attrs = []
+ for attr_name in self.moe_layer.token_dispatcher.cudagraph_attrs:
+ hier_attr_name = attr_name.split('.')
+ attr = self.moe_layer.token_dispatcher
+ for name in hier_attr_name:
+ attr = getattr(attr, name, None)
+ if attr is None:
+ break
+ if isinstance(attr, torch.Tensor):
+ outputs.append(attr)
+ valid_cudagraph_attrs.append(attr_name)
+ if self.moe_layer.token_dispatcher.valid_cudagraph_attrs is None:
+ self.moe_layer.token_dispatcher.valid_cudagraph_attrs = valid_cudagraph_attrs
+ else:
+ assert (
+ self.moe_layer.token_dispatcher.valid_cudagraph_attrs == valid_cudagraph_attrs
+ ), (
+ "valid_cudagraph_attrs mismatch: "
+ f"{self.moe_layer.token_dispatcher.valid_cudagraph_attrs} != "
+ f"{valid_cudagraph_attrs}"
+ )
+ # Also return the shared expert output, if it is not None.
+ if shared_expert_output is not None:
+ outputs.append(shared_expert_output)
+ return outputs
+
+
+@internal_api
+@dataclass
+class MoECudaGraphTensorStore:
+ """Storage for tensors used in CUDA graph replay for MoE layers.
+
+ This dataclass stores intermediate tensors computed during CUDA graph replay
+ that need to be resumed from the end of the CUDA graph scope to skip redundant computations.
+
+ Attributes:
+ hidden_states (Optional[torch.Tensor]): The hidden states output from the CUDA graph replay.
+ probs (Optional[torch.Tensor]): The routing probabilities for each token-expert pair.
+ routing_map (Optional[torch.Tensor]): The sparse mapping indicating which experts
+ were selected for each token. Used to skip the normal router step.
+ shared_expert_output (Optional[torch.Tensor]): The output from shared experts
+ computation. Used to skip the normal shared expert computation step.
+ """
+
+ hidden_states: Optional[torch.Tensor] = None
+ probs: Optional[torch.Tensor] = None
+ routing_map: Optional[torch.Tensor] = None
+ shared_expert_output: Optional[torch.Tensor] = None
+
+ def is_empty(self) -> bool:
+ """Check if the store has any non-None tensors.
+
+ Returns:
+ bool: True if all fields are None, False otherwise.
+ """
+ return all(
+ getattr(self, field_name) is None
+ for field_name in ['hidden_states', 'probs', 'routing_map', 'shared_expert_output']
+ )
+
+ def set(self, **kwargs):
+ """Set the tensors in the store from keyword arguments."""
+ for field_name, value in kwargs.items():
+ assert field_name in [
+ 'hidden_states',
+ 'probs',
+ 'routing_map',
+ 'shared_expert_output',
+ ], f"Invalid field name: {field_name}"
+ if value is not None:
+ assert isinstance(
+ value, torch.Tensor
+ ), f"Value must be a torch.Tensor, got {type(value)} for field {field_name}"
+ setattr(self, field_name, value)
+
+ def clear(self):
+ """Reset all stored tensors to None."""
+ for field_name in ['hidden_states', 'probs', 'routing_map', 'shared_expert_output']:
+ setattr(self, field_name, None)
+
+
+def maybe_skip_or_early_return_by_cudagraph(step_condition):
+ """
+ Decorator to skip certain codepaths in the MoE layer forward pass in CUDA graph replay,
+ or early return from the MoE layer forward pass in CUDA graph capture.
+
+ Args:
+ step_condition: The step condition to check. Can be "shared_experts_compute", "route",
+ or "preprocess". If "shared_experts_compute", the shared experts computation will be
+ skipped in replay if it is in the CUDA graph scope. If "route" or "preprocess", the
+ router or preprocess will be skipped in replay if it is in the CUDA graph scope, or
+ early return from the MoE layer forward pass if it is in CUDA graph capturing mode.
+
+ Returns:
+ A decorator function that wraps the MoE layer forward pass.
+ """
+
+ def maybe_raise_signal(moe_layer, **kwargs):
+ """
+ Check if the MoE layer should early return for CUDA graph capture.
+ If so, raise a MoECudaGraphPartialCaptureSignal.
+ """
+ if (
+ moe_layer.config.cuda_graph_impl == "transformer_engine"
+ and moe_layer.training
+ and is_graph_capturing()
+ ):
+ if (
+ step_condition == "route"
+ and CudaGraphScope.moe_router in moe_layer.config.cuda_graph_scope
+ and CudaGraphScope.moe_preprocess not in moe_layer.config.cuda_graph_scope
+ ):
+ raise MoECudaGraphPartialCaptureSignal(moe_layer, "route", **kwargs)
+ elif (
+ step_condition == "preprocess"
+ and CudaGraphScope.moe_preprocess in moe_layer.config.cuda_graph_scope
+ ):
+ raise MoECudaGraphPartialCaptureSignal(moe_layer, "preprocess", **kwargs)
+
+ def decorator(func):
+
+ @functools.wraps(func)
+ def wrapped_func(moe_layer, *args, **kwargs):
+ """
+ Check if we should skip executing the original function based on the current
+ step condition and the tensor store status. If the tensor can be found in the store,
+ it indicates that it is already computed by the CUDA graph replay, so we can skip it.
+ Otherwise, we execute the original function and check if we should raise a signal to
+ early return in CUDA graph capture.
+ """
+ # The non-cudagraph codepath just calls the original function.
+ if not is_graph_capturing() and moe_layer.cudagraph_tensor_store.is_empty():
+ return func(moe_layer, *args, **kwargs)
+
+ assert (
+ not is_graph_capturing() or moe_layer.cudagraph_tensor_store.is_empty()
+ ), "cudagraph_tensor_store cannot be used when it is capturing cuda graph."
+ if step_condition == "shared_experts_compute":
+ if moe_layer.cudagraph_tensor_store.shared_expert_output is None:
+ # Don't skip the shared expert computation.
+ shared_expert_output = func(moe_layer, *args, **kwargs)
+ else:
+ # Skip the shared expert computation and get value from store.
+ shared_expert_output = moe_layer.cudagraph_tensor_store.shared_expert_output
+ return shared_expert_output
+ elif step_condition == "route":
+ if moe_layer.cudagraph_tensor_store.probs is None:
+ # Don't skip the router.
+ assert (
+ moe_layer.cudagraph_tensor_store.routing_map is None
+ ), "routing_map must be None if probs is None"
+ probs, routing_map = func(moe_layer, *args, **kwargs)
+
+ # Maybe early return after the router.
+ maybe_raise_signal(moe_layer, probs=probs, routing_map=routing_map)
+ else:
+ # Skip the router and get value from store.
+ probs, routing_map = (
+ moe_layer.cudagraph_tensor_store.probs,
+ moe_layer.cudagraph_tensor_store.routing_map,
+ )
+ return probs, routing_map
+ elif step_condition == "preprocess":
+ if (
+ moe_layer.cudagraph_tensor_store.is_empty()
+ or moe_layer.cudagraph_tensor_store.routing_map is not None
+ ):
+ # Don't skip the preprocess.
+ hidden_states, probs = func(moe_layer, *args, **kwargs)
+
+ # Maybe early return after the preprocess.
+ maybe_raise_signal(moe_layer, hidden_states=hidden_states, probs=probs)
+ else:
+ # Skip the preprocess and get value from store.
+ assert (
+ moe_layer.cudagraph_tensor_store.hidden_states is not None
+ and moe_layer.cudagraph_tensor_store.probs is not None
+ ), "hidden_states and probs must be given in moe_preprocess cudagraph replay"
+ hidden_states, probs = (
+ moe_layer.cudagraph_tensor_store.hidden_states,
+ moe_layer.cudagraph_tensor_store.probs,
+ )
+ return hidden_states, probs
+
+ return wrapped_func
+
+ return decorator
diff --git a/megatron/core/transformer/moe/router.py b/megatron/core/transformer/moe/router.py
index 34d81a21ffa..01238e425d9 100644
--- a/megatron/core/transformer/moe/router.py
+++ b/megatron/core/transformer/moe/router.py
@@ -1,19 +1,20 @@
# Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
from abc import ABC, abstractmethod
-from typing import Optional
+from typing import Optional, Union
import torch
from megatron.core.jit import jit_fuser
-from megatron.core.tensor_parallel import reduce_from_tensor_model_parallel_region
from megatron.core.transformer.module import MegatronModule
from megatron.core.transformer.moe.moe_utils import (
MoEAuxLossAutoScaler,
ProcessGroupCollection,
+ apply_biased_logits,
apply_random_logits,
apply_router_token_dropping,
compute_routing_scores_for_aux_loss,
+ get_tokens_per_expert_and_token_count,
router_gating_linear,
save_to_aux_losses_tracker,
sinkhorn,
@@ -21,6 +22,7 @@
topk_routing_with_score_function,
z_loss_func,
)
+from megatron.core.transformer.moe.router_replay import RouterReplay
from megatron.core.transformer.transformer_config import TransformerConfig
@@ -201,6 +203,10 @@ def __init__(
self.global_tokens_per_expert = None
self.ga_steps = None
+ self.router_replay = None
+ if self.config.enable_routing_replay:
+ self.router_replay = RouterReplay()
+
def _maintain_float32_expert_bias(self):
"""
Maintain the expert bias in float32.
@@ -268,22 +274,28 @@ def is_aux_loss_enabled(self) -> bool:
return False
def _apply_aux_loss(
- self, probs: torch.Tensor, scores_for_aux_loss: torch.Tensor, routing_map: torch.Tensor
+ self,
+ probs: torch.Tensor,
+ scores_for_aux_loss: torch.Tensor,
+ routing_map: torch.Tensor,
+ with_padding_mask: bool = False,
):
"""Apply the auxiliary loss for the given scores and routing map."""
aux_loss_coeff = self.get_aux_loss_coeff("aux_loss")
if aux_loss_coeff == 0:
return probs
- tokens_per_expert = routing_map.sum(dim=0)
- tokens_per_expert = reduce_from_tensor_model_parallel_region(
- tokens_per_expert, self.tp_cp_group
- )
- num_tokens = routing_map.shape[0]
- total_num_tokens = num_tokens * self.tp_cp_group.size()
+ global_tokens_per_expert, local_num_tokens, total_num_tokens = (
+ get_tokens_per_expert_and_token_count(
+ routing_map=routing_map,
+ reduce_group=self.tp_cp_group,
+ topk=self.topk,
+ with_padding_mask=with_padding_mask,
+ )
+ )
aux_loss = switch_load_balancing_loss_func(
probs=scores_for_aux_loss,
- tokens_per_expert=tokens_per_expert,
+ tokens_per_expert=global_tokens_per_expert,
total_num_tokens=total_num_tokens,
topk=self.topk,
num_experts=self.config.num_moe_experts,
@@ -291,7 +303,12 @@ def _apply_aux_loss(
fused=self.config.moe_router_fusion,
)
probs = self.attach_and_log_load_balancing_loss(
- probs, aux_loss_coeff, aux_loss, "load_balancing_loss", self.tp_cp_group
+ probs,
+ aux_loss_coeff,
+ aux_loss,
+ "load_balancing_loss",
+ self.tp_cp_group,
+ valid_token_count=local_num_tokens,
)
return probs
@@ -302,6 +319,7 @@ def _apply_seq_aux_loss(
routing_map: torch.Tensor,
seq_length: int,
bsz: int,
+ with_padding_mask: bool = False,
):
"""Apply the sequence-level auxiliary loss for the given scores and routing map.
@@ -315,17 +333,21 @@ def _apply_seq_aux_loss(
return probs
scores_for_aux_loss = scores_for_aux_loss.reshape(seq_length, -1)
- tokens_per_expert = routing_map.reshape(seq_length, -1).sum(dim=0)
- tokens_per_expert = reduce_from_tensor_model_parallel_region(
- tokens_per_expert, self.tp_cp_group
+ routing_map = routing_map.reshape(seq_length, -1)
+
+ global_tokens_per_expert, local_num_tokens, total_num_tokens = (
+ get_tokens_per_expert_and_token_count(
+ routing_map=routing_map,
+ reduce_group=self.tp_cp_group,
+ with_padding_mask=with_padding_mask,
+ topk=self.topk * bsz,
+ )
)
- total_num_tokens = seq_length * self.tp_cp_group.size()
-
aux_loss = (
switch_load_balancing_loss_func(
probs=scores_for_aux_loss,
- tokens_per_expert=tokens_per_expert,
+ tokens_per_expert=global_tokens_per_expert,
total_num_tokens=total_num_tokens,
topk=self.topk,
num_experts=self.config.num_moe_experts,
@@ -334,31 +356,42 @@ def _apply_seq_aux_loss(
)
/ bsz
)
+
probs = self.attach_and_log_load_balancing_loss(
- probs, seq_aux_loss_coeff, aux_loss, "seq_load_balancing_loss", self.tp_cp_group
+ probs,
+ seq_aux_loss_coeff,
+ aux_loss,
+ "seq_load_balancing_loss",
+ self.tp_cp_group,
+ valid_token_count=local_num_tokens,
)
return probs
def _apply_global_aux_loss(
- self, probs: torch.Tensor, scores_for_aux_loss: torch.Tensor, routing_map: torch.Tensor
+ self,
+ probs: torch.Tensor,
+ scores_for_aux_loss: torch.Tensor,
+ routing_map: torch.Tensor,
+ with_padding_mask: bool = False,
):
"""Apply the global auxiliary loss for the given scores and routing map."""
global_aux_loss_coeff = self.get_aux_loss_coeff("global_aux_loss")
if global_aux_loss_coeff == 0:
return probs
- tokens_per_expert = routing_map.sum(dim=0)
- tokens_per_expert = reduce_from_tensor_model_parallel_region(
- tokens_per_expert, self.tp_dp_cp_group
+ # Use unified function to compute tokens_per_expert and num_tokens
+ global_tokens_per_expert, local_num_tokens, total_num_tokens = (
+ get_tokens_per_expert_and_token_count(
+ routing_map=routing_map,
+ reduce_group=self.tp_dp_cp_group,
+ with_padding_mask=with_padding_mask,
+ topk=self.topk,
+ )
)
-
- self.global_tokens_per_expert += tokens_per_expert
+ self.global_tokens_per_expert += global_tokens_per_expert
self.ga_steps += 1
averated_tokens_per_expert = self.global_tokens_per_expert / self.ga_steps
- num_tokens = scores_for_aux_loss.shape[0]
- total_num_tokens = num_tokens * self.tp_dp_cp_group.size()
-
global_aux_loss = switch_load_balancing_loss_func(
probs=scores_for_aux_loss,
tokens_per_expert=averated_tokens_per_expert,
@@ -374,6 +407,7 @@ def _apply_global_aux_loss(
global_aux_loss,
"global_load_balancing_loss",
self.tp_dp_cp_group,
+ valid_token_count=local_num_tokens,
reduce_group_has_dp=True,
)
return probs
@@ -385,6 +419,7 @@ def attach_and_log_load_balancing_loss(
aux_loss: torch.Tensor,
aux_loss_name: str,
reduce_group: torch.distributed.ProcessGroup,
+ valid_token_count: Optional[Union[int, torch.Tensor]] = None,
reduce_group_has_dp: bool = False,
):
"""Attach aux loss function to activation and add to logging.
@@ -395,6 +430,9 @@ def attach_and_log_load_balancing_loss(
aux_loss (torch.Tensor): The auxiliary loss tensor.
aux_loss_name (str): The name of the auxiliary loss for logging.
reduce_group (torch.distributed.ProcessGroup): The group for reducing the loss.
+ valid_token_count (int or torch.Tensor, optional): Number of valid tokens excluding
+ padding tokens. Can be a Python int or a torch.Tensor (typically 0-d tensor).
+ If None, uses activation.shape[0]. Defaults to None.
reduce_group_has_dp (bool): Whether the reduce group has data parallel ranks.
Set this to True if the reduce group has data parallel ranks. This flag is used to
ensure the correct reduction in aux loss tracking.
@@ -422,17 +460,22 @@ def attach_and_log_load_balancing_loss(
# which scales both the main_loss gradient and aux_loss gradient by
# 1/(num_local_tokens * dp_size * num_micro_batches) in finalize_model_grads function.
# To correct this scaling, we need to scale the aux_loss by num_local_tokens here.
- activation = MoEAuxLossAutoScaler.apply(activation, aux_loss * activation.shape[0])
+ # Use valid_token_count (excluding padding) if provided, otherwise use total tokens.
+ num_tokens = valid_token_count if valid_token_count is not None else activation.shape[0]
+ activation = MoEAuxLossAutoScaler.apply(activation, aux_loss * num_tokens)
else:
activation = MoEAuxLossAutoScaler.apply(activation, aux_loss)
return activation
- def apply_z_loss(self, logits):
+ def apply_z_loss(self, logits, padding_mask: Optional[torch.Tensor] = None):
"""Encourages the router's logits to remain small to enhance stability.
Please refer to the ST-MoE paper (https://arxiv.org/pdf/2202.08906.pdf) for details.
Args:
logits (torch.Tensor): The logits of the router.
+ padding_mask (torch.Tensor, optional): Boolean mask indicating padding positions.
+ Shape [num_tokens]. True = padding (exclude),
+ False = valid (include). Defaults to None.
Returns:
torch.Tensor: The logits after applying the z-loss.
@@ -440,7 +483,7 @@ def apply_z_loss(self, logits):
if self.config.moe_z_loss_coeff is not None and self.training and torch.is_grad_enabled():
# Skip Z loss calculations when using torch.no_grad() or checkpointing.
moe_z_loss_coeff = self.config.moe_z_loss_coeff / self.tp_cp_group.size()
- z_loss = z_loss_func(logits, moe_z_loss_coeff)
+ z_loss = z_loss_func(logits, moe_z_loss_coeff, padding_mask=padding_mask)
scale_up = 1.0
if self.calculate_per_token_loss:
# The expected final scaling for z_loss gradients is
@@ -450,7 +493,9 @@ def apply_z_loss(self, logits):
# which scales both the main_loss gradient and z_loss gradient by
# 1/(num_local_tokens * dp_size * num_micro_batches) in finalize_model_grads().
# To correct this scaling, we need to scale the z_loss by num_local_tokens here.
- logits = MoEAuxLossAutoScaler.apply(logits, z_loss * logits.shape[0])
+ # Count valid tokens: sum of inverted mask (False -> True = valid)
+ num_tokens = (~padding_mask).sum() if padding_mask is not None else logits.shape[0]
+ logits = MoEAuxLossAutoScaler.apply(logits, z_loss * num_tokens)
else:
logits = MoEAuxLossAutoScaler.apply(logits, z_loss)
@@ -484,20 +529,32 @@ def apply_input_jitter(self, input: torch.Tensor):
return input
@jit_fuser
- def _apply_expert_bias(self, routing_map: torch.Tensor):
+ def _apply_expert_bias(
+ self, routing_map: torch.Tensor, padding_mask: Optional[torch.Tensor] = None
+ ):
"""
Update expert bias and tokens_per_expert
Prevent extra local tokens accumulation on evaluation or activation recomputation
+
+ Args:
+ routing_map (torch.Tensor): Token to expert routing map, [num_tokens, num_experts].
+ padding_mask (torch.Tensor, optional): Boolean mask indicating padding positions.
+ Shape [num_tokens]. True = padding (exclude), False = valid (include).
"""
if self.enable_expert_bias and torch.is_grad_enabled():
with torch.no_grad():
+ if padding_mask is not None:
+ routing_map = routing_map & (~padding_mask)
self.local_tokens_per_expert += routing_map.sum(dim=0)
- def routing(self, logits: torch.Tensor):
+ def routing(self, logits: torch.Tensor, padding_mask: Optional[torch.Tensor] = None):
"""Top-k routing function
Args:
logits (torch.Tensor): Logits tensor after gating.
+ padding_mask (torch.Tensor, optional): Boolean mask indicating padding positions.
+ Shape = [seq_length, bsz]. True=padding(exclude),
+ False=valid(include). Defaults to None.
Returns:
probs (torch.Tensor): The probabilities of token to experts assignment.
@@ -507,8 +564,12 @@ def routing(self, logits: torch.Tensor):
seq_length, bsz = logits.shape[:2]
logits = logits.view(-1, self.config.num_moe_experts)
+ # Flatten padding_mask to [num_tokens] if provided
+ if padding_mask is not None:
+ padding_mask = padding_mask.reshape(-1)
+
# Apply Z-Loss
- logits = self.apply_z_loss(logits)
+ logits = self.apply_z_loss(logits, padding_mask=padding_mask)
# Calculate probs and routing_map for token dispatching
if self.routing_type == "sinkhorn":
@@ -524,6 +585,7 @@ def routing(self, logits: torch.Tensor):
score_function=self.score_function,
expert_bias=self.expert_bias,
fused=self.config.moe_router_fusion,
+ router_replay=self.router_replay,
)
# Apply token dropping to probs and routing_map.
@@ -541,18 +603,35 @@ def routing(self, logits: torch.Tensor):
if self.training and torch.is_grad_enabled() and self.is_aux_loss_enabled():
# Calculate scores and routing_map for aux loss
routing_map_for_aux_loss, scores_for_aux_loss = compute_routing_scores_for_aux_loss(
- logits, self.topk, self.score_function, fused=self.config.moe_router_fusion
+ logits,
+ self.topk,
+ self.score_function,
+ fused=self.config.moe_router_fusion,
+ padding_mask=padding_mask,
+ )
+ probs = self._apply_aux_loss(
+ probs,
+ scores_for_aux_loss,
+ routing_map_for_aux_loss,
+ with_padding_mask=padding_mask is not None,
)
- probs = self._apply_aux_loss(probs, scores_for_aux_loss, routing_map_for_aux_loss)
probs = self._apply_seq_aux_loss(
- probs, scores_for_aux_loss, routing_map_for_aux_loss, seq_length, bsz
+ probs,
+ scores_for_aux_loss,
+ routing_map_for_aux_loss,
+ seq_length,
+ bsz,
+ with_padding_mask=padding_mask is not None,
)
probs = self._apply_global_aux_loss(
- probs, scores_for_aux_loss, routing_map_for_aux_loss
+ probs,
+ scores_for_aux_loss,
+ routing_map_for_aux_loss,
+ with_padding_mask=padding_mask is not None,
)
# Optionally apply expert bias
- self._apply_expert_bias(routing_map)
+ self._apply_expert_bias(routing_map, padding_mask=padding_mask)
return probs, routing_map
@@ -562,12 +641,15 @@ def reset_global_aux_loss_tracker(self):
self.global_tokens_per_expert.zero_()
self.ga_steps.zero_()
- def forward(self, input: torch.Tensor):
+ def forward(self, input: torch.Tensor, padding_mask: Optional[torch.Tensor] = None):
"""
Forward pass of the router.
Args:
input (torch.Tensor): Input tensor.
+ padding_mask (torch.Tensor, optional): Boolean mask indicating padding positions.
+ Shape = [seq_length, bsz]. True=padding(exclude),
+ False=valid(include). Defaults to None.
"""
self._maintain_float32_expert_bias()
@@ -579,7 +661,13 @@ def forward(self, input: torch.Tensor):
# Apply force load balancing with random logits for benchmark
logits = apply_random_logits(logits)
- probs, routing_map = self.routing(logits)
+ if self.config.moe_router_force_biased is not None:
+ # Apply biased logits with shared random bias across all ranks
+ logits = apply_biased_logits(
+ logits, self.config.moe_router_force_biased, self.layer_number
+ )
+
+ probs, routing_map = self.routing(logits, padding_mask=padding_mask)
return probs, routing_map
diff --git a/megatron/core/transformer/moe/router_replay.py b/megatron/core/transformer/moe/router_replay.py
new file mode 100644
index 00000000000..b6b8e26a0a6
--- /dev/null
+++ b/megatron/core/transformer/moe/router_replay.py
@@ -0,0 +1,161 @@
+# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+from enum import Enum
+from typing import Callable, List, Optional, Tuple
+
+import torch
+
+
+class RouterReplayAction(Enum):
+ """
+ A Enum to define the actions for router replay.
+ """
+
+ RECORD = "record" # Record the topk indices for replay
+ REPLAY_FORWARD = "replay_forward" # Replay the recorded topk indices for forward pass
+ REPLAY_BACKWARD = "replay_backward" # Replay topk indices for re-compute during backward pass
+
+
+class RouterReplay:
+ """
+ A class to manage the recording and replaying of MoE routing decisions.
+ It holds all router instances and provides static methods to globally
+ control recording and replaying.
+ """
+
+ # Static variable to hold all router instances, one per MoE layer.
+ global_router_replay_instances: List['RouterReplay'] = []
+
+ @staticmethod
+ def set_replay_data(all_layers_topk_indices: List[torch.Tensor]):
+ """
+ Distributes the topk indices for all layers to their respective RouterReplay instances.
+ :param all_layers_topk_indices: A list of tensors, where each tensor contains the
+ topk indices for a specific layer. The order
+ must match the instantiation order of the routers.
+ """
+ if len(all_layers_topk_indices) != len(RouterReplay.global_router_replay_instances):
+ raise ValueError(
+ f"The number of replay tensors ({len(all_layers_topk_indices)}) "
+ f"does not match instances ({len(RouterReplay.global_router_replay_instances)})."
+ )
+ for i, router_instance in enumerate(RouterReplay.global_router_replay_instances):
+ router_instance.set_target_indices(all_layers_topk_indices[i])
+
+ @staticmethod
+ def get_recorded_data() -> List[torch.Tensor]:
+ """
+ Collects the recorded topk indices from all RouterReplay instances.
+ :return: A list of tensors, each containing the recorded topk indices for a layer.
+ """
+ return [
+ router.get_recorded_indices() for router in RouterReplay.global_router_replay_instances
+ ]
+
+ @staticmethod
+ def clear_global_indices():
+ """Clears the recorded and target topk indices in all instances."""
+ for router in RouterReplay.global_router_replay_instances:
+ router.clear_indices()
+
+ @staticmethod
+ def set_global_router_replay_action(router_replay_action: RouterReplayAction):
+ """Sets the router replay action for all router instances."""
+ for router in RouterReplay.global_router_replay_instances:
+ router.set_router_replay_action(router_replay_action)
+
+ @staticmethod
+ def clear_global_router_replay_action():
+ """Clears the router replay action for all router instances."""
+ for router in RouterReplay.global_router_replay_instances:
+ router.clear_router_replay_action()
+
+ @staticmethod
+ def clear_global_router_replay_instances():
+ """Clear the global list of router replay instances to prevent memory leaks."""
+ RouterReplay.global_router_replay_instances.clear()
+
+ def __init__(self):
+ """Initializes a RouterReplay instance for a specific layer."""
+ self.target_topk_idx: Optional[torch.Tensor] = None # Target topk indices for replay
+ self.recorded_topk_idx: Optional[torch.Tensor] = None # Recorded topk indices for replay
+ self.router_replay_action: Optional[RouterReplayAction] = (
+ None # Router replay action for this layer
+ )
+ self.replay_backward_list: List[torch.Tensor] = (
+ []
+ ) # List of tensors for backward pass replay
+ RouterReplay.global_router_replay_instances.append(self)
+
+ def set_target_indices(self, topk_indices: torch.Tensor):
+ """Sets the target topk indices for replay."""
+ self.target_topk_idx = topk_indices
+ self.replay_backward_list.append(topk_indices)
+
+ def get_recorded_indices(self) -> Optional[torch.Tensor]:
+ """Returns the recorded topk indices."""
+ return self.recorded_topk_idx
+
+ def record_indices(self, topk_indices: torch.Tensor):
+ """Records the topk indices."""
+ self.recorded_topk_idx = topk_indices
+
+ def clear_indices(self):
+ """Clears the recorded and target topk indices."""
+ self.recorded_topk_idx = None
+ self.target_topk_idx = None
+ self.replay_backward_list = []
+
+ def set_router_replay_action(self, router_replay_action: RouterReplayAction):
+ """Sets the router replay action for this layer."""
+ self.router_replay_action = router_replay_action
+
+ def clear_router_replay_action(self):
+ """Clears the router replay action for this layer."""
+ self.router_replay_action = None
+
+ def get_replay_topk(
+ self,
+ scores: torch.Tensor,
+ topk: int,
+ num_groups: Optional[int] = None,
+ group_topk: Optional[int] = None,
+ default_compute_topk: Callable[
+ [torch.Tensor, int, Optional[int], Optional[int]], Tuple[torch.Tensor, torch.Tensor]
+ ] = None,
+ ) -> Tuple[torch.Tensor, torch.Tensor]:
+ """
+ A wrapper for top-k computation that handles different replay actions.
+
+ Args:
+ scores (torch.Tensor): The scores to compute top-k on.
+ topk (int): The number of top elements to select.
+ num_groups (Optional[int]): Number of expert groups for group-limited routing.
+ group_topk (Optional[int]): Number of groups to select for each token.
+ default_compute_topk (Callable): The default top-k computation function, which
+ should return a tuple of (values, indices).
+
+ Returns:
+ Tuple[torch.Tensor, torch.Tensor]: A tuple containing the top-k values and indices.
+ """
+ if self.router_replay_action == RouterReplayAction.RECORD:
+ probs, top_indices = default_compute_topk(
+ scores, topk, num_groups=num_groups, group_topk=group_topk
+ )
+ self.record_indices(top_indices)
+ return probs, top_indices
+ elif self.router_replay_action == RouterReplayAction.REPLAY_FORWARD:
+ top_indices = self.target_topk_idx
+ # Ensure indices are on the correct device
+ top_indices = top_indices.to(scores.device)
+ # Gather the scores for the replayed indices to get the probabilities
+ probs = scores.gather(1, top_indices)
+ return probs, top_indices
+ elif self.router_replay_action == RouterReplayAction.REPLAY_BACKWARD:
+ top_indices = self.replay_backward_list.pop(0)
+ # Ensure indices are on the correct device
+ top_indices = top_indices.to(scores.device)
+ # Gather the scores for the replayed indices to get the probabilities
+ probs = scores.gather(1, top_indices)
+ return probs, top_indices
+ else:
+ return default_compute_topk(scores, topk, num_groups, group_topk)
diff --git a/megatron/core/transformer/moe/shared_experts.py b/megatron/core/transformer/moe/shared_experts.py
index c4378a3377e..3cb34a36f26 100644
--- a/megatron/core/transformer/moe/shared_experts.py
+++ b/megatron/core/transformer/moe/shared_experts.py
@@ -1,7 +1,7 @@
# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved.
import warnings
-from copy import deepcopy
+from copy import copy
from typing import Optional
import torch
@@ -43,7 +43,7 @@ def __init__(
gate: bool,
pg_collection: Optional[ProcessGroupCollection] = None,
):
- config = deepcopy(config)
+ config = copy(config)
assert config.add_bias_linear == False, "bias is not supported in the shared experts, "
"please set '--disable-bias-linear' instead."
diff --git a/megatron/core/transformer/moe/token_dispatcher.py b/megatron/core/transformer/moe/token_dispatcher.py
index 0beae556cf7..d0da38d6322 100644
--- a/megatron/core/transformer/moe/token_dispatcher.py
+++ b/megatron/core/transformer/moe/token_dispatcher.py
@@ -16,6 +16,7 @@
gather_from_sequence_parallel_region,
reduce_scatter_to_sequence_parallel_region,
)
+from megatron.core.transformer.enums import CudaGraphScope
from megatron.core.transformer.moe.fused_a2a import (
fused_combine,
fused_dispatch,
@@ -36,6 +37,8 @@
from megatron.core.transformer.moe.shared_experts import SharedExpertMLP
from megatron.core.transformer.transformer_config import TransformerConfig
+logger = logging.getLogger(__name__)
+
""" We use the following notation throughout this file:
H: hidden size
B: micro batch size
@@ -76,6 +79,11 @@ def __init__(
self.tp_rank = utils.get_pg_rank(self.tp_group)
self.ep_size = utils.get_pg_size(self.ep_group)
+ # Attributes that need to be captured in cudagraph. These attributes are returned
+ # as cudagraph outputs when the cuda_graph_scope contains moe_preprocess.
+ self.cudagraph_attrs = []
+ self.valid_cudagraph_attrs = None
+
@abstractmethod
def dispatch_preprocess(
self, tokens: torch.Tensor, routing_map: torch.Tensor, probs: torch.Tensor
@@ -233,6 +241,10 @@ def __init__(
# device token permutation is enabled and **AllGahter** is performed.
self.global_local_map = None
+ # Attributes that need to be captured in cudagraph. These attributes are returned
+ # as cudagraph outputs when the cuda_graph_scope contains moe_preprocess.
+ self.cudagraph_attrs = ['routing_map']
+
def dispatch_preprocess(
self, hidden_states: torch.Tensor, routing_map: torch.Tensor, probs: torch.Tensor
):
@@ -425,12 +437,38 @@ def __init__(
"before_finish": 3,
"no_sync": 4,
}
- self.cuda_dtoh_point = "before_permutation_1"
+ if (
+ config.cuda_graph_impl == "transformer_engine"
+ and CudaGraphScope.moe_preprocess in config.cuda_graph_scope
+ ):
+ self.cuda_dtoh_point = "before_ep_alltoall"
+ else:
+ self.cuda_dtoh_point = "before_permutation_1"
if MoEAlltoAllTokenDispatcher.cuda_dtoh_stream is None:
MoEAlltoAllTokenDispatcher.cuda_dtoh_stream = torch.cuda.Stream()
+ # Attributes that need to be captured in cudagraph. These attributes are returned
+ # as cudagraph outputs when the cuda_graph_scope contains moe_preprocess.
+ self.cudagraph_attrs = [
+ 'tokens_per_expert',
+ 'input_splits',
+ 'output_splits',
+ 'output_splits_tp',
+ 'num_out_tokens',
+ 'num_global_tokens_per_local_expert',
+ 'reversed_local_input_permutation_mapping',
+ 'routing_map',
+ ]
+
self.shared_experts = None
+ def set_shared_experts(self, shared_experts):
+ """Set shared expert to the dispatcher."""
+ super().set_shared_experts(shared_experts)
+ if shared_experts.use_shared_expert_gate:
+ self.cudagraph_attrs.append('shared_experts.gate_score')
+ self.cudagraph_attrs.append('shared_experts.cached_fc1_input')
+
def preprocess(self, routing_map: torch.Tensor) -> torch.Tensor:
"""
Preprocesses the token routing map for All-to-All communication and token permutation.
@@ -1032,7 +1070,9 @@ def combine(
num_permuted_tokens=self.num_permuted_tokens,
pad_multiple=self.pad_multiple,
)
- # Release the used handle/num_permuted_tokens which could change in each iteration
+ # Release the used handle/num_permuted_tokens which could change in each iteration.
+ # For drop_and_pad mode, we don't need to reset the num_permuted_tokens and
+ # num_dispatched_tokens, because their values never change.
self.handle = None
if not self.drop_and_pad:
self.num_permuted_tokens = None
@@ -1315,6 +1355,7 @@ def __init__(
num_experts=self.tp_size * self.config.num_moe_experts,
config=self.config,
)
+ self.cudagraph_attrs = ['_comm_manager.token_probs', '_comm_manager.token_indices']
elif self.config.moe_flex_dispatcher_backend == "hybridep":
self._comm_manager = _HybridEPManager(
group=self.tp_ep_group,
@@ -1322,6 +1363,7 @@ def __init__(
num_experts=self.tp_size * self.config.num_moe_experts,
config=self.config,
)
+ self.cudagraph_attrs = ['_comm_manager.token_probs', '_comm_manager.routing_map']
else:
raise ValueError(
f"Invalid backend: {self.config.moe_flex_dispatcher_backend}"
diff --git a/megatron/core/transformer/multi_latent_attention.py b/megatron/core/transformer/multi_latent_attention.py
index 7a7b2e5930b..9689056e325 100644
--- a/megatron/core/transformer/multi_latent_attention.py
+++ b/megatron/core/transformer/multi_latent_attention.py
@@ -1,4 +1,4 @@
-# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved.
+# Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
import math
@@ -15,13 +15,16 @@
HAVE_EINOPS = False
-from megatron.core import parallel_state, tensor_parallel
+from megatron.core import tensor_parallel
from megatron.core.models.common.embeddings import (
RotaryEmbedding,
YarnRotaryEmbedding,
_yarn_get_mscale,
apply_rotary_pos_emb,
)
+from megatron.core.pipeline_parallel.fine_grained_activation_offload import (
+ FineGrainedActivationOffloadingInterface as off_interface,
+)
from megatron.core.process_groups_config import ProcessGroupCollection
from megatron.core.tensor_parallel.layers import ColumnParallelLinear
from megatron.core.tensor_parallel.mappings import (
@@ -36,7 +39,7 @@
from megatron.core.transformer.enums import AttnMaskType
from megatron.core.transformer.spec_utils import ModuleSpec, build_module
from megatron.core.transformer.transformer_config import MLATransformerConfig
-from megatron.core.utils import deprecate_inference_params, is_te_min_version
+from megatron.core.utils import deprecate_inference_params, get_pg_size, is_te_min_version
try:
from megatron.core.fusions.fused_mla_yarn_rope_apply import (
@@ -173,6 +176,7 @@ def __init__(
skip_bias_add=True,
is_expert=False,
tp_comm_buffer_name='proj',
+ tp_group=self.pg_collection.tp,
)
if (
@@ -238,13 +242,33 @@ def forward(
# Get the query, key and value tensors based on the type of attention -
# self or cross attn.
# query: [96, 1, 16, 128], key:[96, 1, 16, 128], value:[96, 1, 16, 128]
- query, key, value = self.get_query_key_value_tensors(
- hidden_states,
- key_value_states,
- position_ids,
- packed_seq_params,
- inference_context=inference_context,
- )
+ with off_interface(self.offload_qkv_linear, hidden_states, "qkv_linear") as hidden_states:
+ if self.config.experimental_attention_variant is None:
+ query, key, value = self.get_query_key_value_tensors(
+ hidden_states,
+ key_value_states,
+ position_ids,
+ packed_seq_params,
+ inference_context=inference_context,
+ )
+ elif self.config.experimental_attention_variant == "dsa":
+ query, key, value, q_compressed, _ = self.get_query_key_value_tensors(
+ hidden_states,
+ key_value_states,
+ position_ids,
+ packed_seq_params,
+ inference_context=inference_context,
+ return_compressed_tensors=True,
+ )
+ else:
+ raise ValueError(
+ f"Unsupported experimental attention variant: "
+ f"{self.config.experimental_attention_variant}"
+ )
+ if self.offload_qkv_linear:
+ query = off_interface.group_commit(
+ query, name="qkv_linear", forced_released_tensors=[hidden_states]
+ )
# ===================================================
# Adjust key, value for inference
@@ -272,14 +296,37 @@ def forward(
)
else:
if inference_context is None or inference_context.is_static_batching():
- core_attn_out = self.core_attention(
- query,
- key,
- value,
- attention_mask,
- packed_seq_params=packed_seq_params,
- attn_mask_type=attn_mask_type,
- )
+ with off_interface(
+ self.offload_core_attention and self.training, query, "core_attn"
+ ) as query:
+ if self.config.experimental_attention_variant is None:
+ core_attn_out = self.core_attention(
+ query,
+ key,
+ value,
+ attention_mask,
+ packed_seq_params=packed_seq_params,
+ attn_mask_type=attn_mask_type,
+ )
+ elif self.config.experimental_attention_variant == "dsa":
+ # For dsa we need to pass in the original hidden states and the compressed
+ # query representation.
+ core_attn_out = self.core_attention(
+ query,
+ key,
+ value,
+ x=hidden_states,
+ qr=q_compressed,
+ attention_mask=attention_mask,
+ attn_mask_type=attn_mask_type,
+ attention_bias=None,
+ packed_seq_params=packed_seq_params,
+ )
+ else:
+ raise ValueError(
+ f"Unsupported attention variant: "
+ f"{self.config.experimental_attention_variant}"
+ )
elif self.cache_mla_latents:
# Dynamic batching attention kernel.
q, k, v = (query, key, value)
@@ -300,6 +347,10 @@ def forward(
# Only rearrange if not in absorption mode (Flash MLA handles format correctly)
if not inference_context.is_decode_only():
core_attn_out = rearrange(core_attn_out, 's b h d -> s b (h d)')
+ if self.offload_core_attention and self.training:
+ core_attn_out = off_interface.group_commit(
+ core_attn_out, name="core_attn", forced_released_tensors=[query, key, value]
+ )
# We are doing absorption with cache mla latents and decode mode.
if self.cache_mla_latents and inference_context.is_decode_only():
@@ -325,7 +376,12 @@ def forward(
# =================
# Output. [sq, b, h]
# =================
- output, bias = self.linear_proj(core_attn_out)
+ with off_interface(self.offload_attn_proj, core_attn_out, "attn_proj") as core_attn_out:
+ output, bias = self.linear_proj(core_attn_out)
+ if self.offload_attn_proj:
+ output = off_interface.group_commit(
+ output, name="attn_proj", forced_released_tensors=[core_attn_out]
+ )
return output, bias
@@ -346,6 +402,9 @@ def __init__(
cp_comm_type: Optional[str] = None,
pg_collection: ProcessGroupCollection = None,
):
+ if pg_collection is None:
+ pg_collection = ProcessGroupCollection.use_mpu_process_groups()
+
super().__init__(
config=config,
submodules=submodules,
@@ -395,6 +454,11 @@ def __init__(
is_expert=False,
tp_comm_buffer_name='q_down_proj',
skip_weight_param_allocation=False,
+ tp_group=(
+ pg_collection.tp
+ if q_down_proj_kwargs.get('parallel_mode') != 'duplicated'
+ else None
+ ),
**q_down_proj_kwargs,
)
@@ -409,6 +473,7 @@ def __init__(
skip_bias_add=False,
is_expert=False,
tp_comm_buffer_name='q_up_proj',
+ tp_group=pg_collection.tp,
)
kv_down_proj_kwargs = {}
@@ -434,6 +499,11 @@ def __init__(
is_expert=False,
tp_comm_buffer_name='kv_down_proj',
skip_weight_param_allocation=False,
+ tp_group=(
+ pg_collection.tp
+ if kv_down_proj_kwargs.get('parallel_mode') != 'duplicated'
+ else None
+ ),
**kv_down_proj_kwargs,
)
@@ -448,6 +518,7 @@ def __init__(
skip_bias_add=False,
is_expert=False,
tp_comm_buffer_name='kv_up_proj',
+ tp_group=pg_collection.tp,
)
if self.config.q_lora_rank is not None:
@@ -474,6 +545,7 @@ def get_query_key_value_tensors(
inference_context=None,
*,
inference_params=None,
+ return_compressed_tensors=False,
):
"""
Derives `query`, `key` and `value` tensors from `hidden_states`.
@@ -483,6 +555,11 @@ def get_query_key_value_tensors(
assert (
hidden_states.ndim == 3
), f"hidden_states should be 3D, [s, b, n*h], got {hidden_states.ndim}D"
+ if packed_seq_params is not None:
+ assert (
+ packed_seq_params.local_cp_size is None
+ ), "hybrid_context_parallel is not supported with MLA yet and is planned for future. \
+ Please disable hybrid_context_parallel."
inference_context = deprecate_inference_params(inference_context, inference_params)
@@ -499,11 +576,13 @@ def get_query_key_value_tensors(
rotary_pos_sin = None
packed_seq = packed_seq_params is not None and packed_seq_params.qkv_format == 'thd'
if self.config.rope_type == "rope":
- rotary_pos_emb = self.rotary_pos_emb(rotary_seq_len, packed_seq=packed_seq)
+ rotary_pos_emb = self.rotary_pos_emb(
+ rotary_seq_len, packed_seq_params=packed_seq_params
+ )
else:
if self.config.apply_rope_fusion:
rotary_pos_cos, rotary_pos_sin = self.rotary_pos_emb.get_cached_cos_sin(
- rotary_seq_len, dtype=hidden_states.dtype, packed_seq=packed_seq
+ rotary_seq_len, dtype=hidden_states.dtype, packed_seq_params=packed_seq_params
)
rotary_pos_emb = None
assert inference_context is None, "Inference with MLA RoPE fusion is not supported"
@@ -512,9 +591,11 @@ def get_query_key_value_tensors(
and fused_apply_mla_rope_for_kv is not None
), "Fused MLA RoPE apply is not imported successfully"
else:
- rotary_pos_emb, mscale = self.rotary_pos_emb(rotary_seq_len, packed_seq=packed_seq)
+ rotary_pos_emb, mscale = self.rotary_pos_emb(
+ rotary_seq_len, packed_seq_params=packed_seq_params
+ )
- if packed_seq_params is not None:
+ if packed_seq_params is not None and packed_seq_params.qkv_format == 'thd':
if packed_seq_params.cu_seqlens_q_padded is not None:
cu_seqlens_q = packed_seq_params.cu_seqlens_q_padded
else:
@@ -568,12 +649,9 @@ def get_query_key_value_tensors(
kv_compressed, k_pos_emb = torch.split(
kv_combined, [self.config.kv_lora_rank, self.config.qk_pos_emb_head_dim], dim=-1
)
- if (
- parallel_state.get_tensor_model_parallel_world_size() > 1
- and self.config.sequence_parallel
- ):
+ if get_pg_size(self.tp_group) > 1 and self.config.sequence_parallel:
# k_pos_emb: [s, b, qk_pos_emb_head_dim]
- k_pos_emb = gather_from_sequence_parallel_region(k_pos_emb)
+ k_pos_emb = gather_from_sequence_parallel_region(k_pos_emb, group=self.tp_group)
if packed_seq_params is not None:
# If sequence packing, TE expect [t, h, d] shaped qkv input.
@@ -583,6 +661,16 @@ def get_query_key_value_tensors(
kv_compressed = kv_compressed.squeeze(1)
k_pos_emb = k_pos_emb.squeeze(1)
+ # =========================================
+ # Apply norm
+ # =========================================
+
+ if self.config.q_lora_rank is not None:
+ # q_compressed: [num_tokens, q_lora_rank]
+ q_compressed = self.q_layernorm(q_compressed)
+
+ kv_compressed = self.kv_layernorm(kv_compressed)
+
# =========================================
# QKV up projection and RoPE apply
# =========================================
@@ -593,7 +681,6 @@ def qkv_up_proj_and_rope_apply_for_cached_latent_kv(
if self.config.q_lora_rank is not None:
# q_compressed: [num_tokens, q_lora_rank]
# q: [num_tokens, n * (qk_head_dim + qk_pos_emb_head_dim)]
- q_compressed = self.q_layernorm(q_compressed)
q, _ = self.linear_q_up_proj(q_compressed)
else:
# q_compressed: [num_tokens, hidden_size]
@@ -603,8 +690,6 @@ def qkv_up_proj_and_rope_apply_for_cached_latent_kv(
# q: [num_tokens, n, q_head_dim]
q = q.view(*q.size()[:-1], self.num_attention_heads_per_partition, self.q_head_dim)
- kv_compressed = self.kv_layernorm(kv_compressed)
-
# [num_tokens, qk_pos_emb_head_dim] -> [num_tokens, 1, qk_pos_emb_head_dim]
k_pos_emb = torch.unsqueeze(k_pos_emb, -2)
@@ -668,7 +753,6 @@ def qkv_up_proj_and_rope_apply(q_compressed, kv_compressed, k_pos_emb, rotary_po
if self.config.q_lora_rank is not None:
# q_compressed: [num_tokens, q_lora_rank]
# q: [num_tokens, n * (qk_head_dim + qk_pos_emb_head_dim)]
- q_compressed = self.q_layernorm(q_compressed)
q, _ = self.linear_q_up_proj(q_compressed)
else:
# q_compressed: [num_tokens, hidden_size]
@@ -678,8 +762,6 @@ def qkv_up_proj_and_rope_apply(q_compressed, kv_compressed, k_pos_emb, rotary_po
# q: [num_tokens, n, q_head_dim]
q = q.view(*q.size()[:-1], self.num_attention_heads_per_partition, self.q_head_dim)
- kv_compressed = self.kv_layernorm(kv_compressed)
-
# kv: [num_tokens, n * (qk_head_dim + v_head_dim)]
kv, _ = self.linear_kv_up_proj(kv_compressed)
@@ -804,7 +886,10 @@ def qkv_up_proj_and_rope_apply(q_compressed, kv_compressed, k_pos_emb, rotary_po
q_compressed, kv_compressed, k_pos_emb, rotary_pos_emb
)
- return query, key, value
+ if return_compressed_tensors:
+ return query, key, value, q_compressed, kv_compressed
+ else:
+ return query, key, value
def uncompress_kv_from_cache(self, kv_cached):
"""
diff --git a/megatron/core/transformer/multi_token_prediction.py b/megatron/core/transformer/multi_token_prediction.py
index 8135e1c47a3..8d5c479aa59 100755
--- a/megatron/core/transformer/multi_token_prediction.py
+++ b/megatron/core/transformer/multi_token_prediction.py
@@ -1,4 +1,4 @@
-# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved.
+# Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
from contextlib import nullcontext
from dataclasses import dataclass
@@ -13,17 +13,17 @@
from megatron.core.fp8_utils import get_fp8_context
from megatron.core.models.backends import BackendSpecProvider, LocalSpecProvider
from megatron.core.packed_seq_params import PackedSeqParams
-from megatron.core.pipeline_parallel.utils import is_vp_last_stage
from megatron.core.process_groups_config import ProcessGroupCollection
from megatron.core.tensor_parallel import (
gather_from_tensor_model_parallel_region,
scatter_to_sequence_parallel_region,
)
-from megatron.core.transformer.enums import AttnMaskType
+from megatron.core.transformer.enums import AttnMaskType, LayerType
from megatron.core.transformer.module import MegatronModule
from megatron.core.transformer.spec_utils import ModuleSpec, build_module
from megatron.core.transformer.transformer_block import TransformerBlockSubmodules
from megatron.core.transformer.transformer_config import TransformerConfig
+from megatron.core.transformer.transformer_layer import get_transformer_layer_offset
from megatron.core.utils import (
get_pg_rank,
is_torch_min_version,
@@ -443,25 +443,103 @@ def get_mtp_layer_spec_for_backend(
return mtp_layer_spec
-def get_mtp_layer_offset(config: TransformerConfig) -> int:
+def mtp_on_this_rank(
+ config: TransformerConfig, ignore_virtual: Optional[bool] = True, vp_stage: Optional[int] = None
+) -> bool:
+ """
+ Check if there is MTP on the current rank.
+
+ Behavior:
+ - If a custom pipeline model parallel layout is provided in the config:
+ - If virtual pipeline parallelism is enabled (and `ignore_virtual` is False), checks
+ whether any MTP layers are present on this (pp_rank, vp_stage) pair.
+ - Otherwise, checks all virtual pipeline ranks of the current pipeline rank. Returns
+ True if any virtual sub-rank includes at least one MTP layer.
+ - If no custom layout is provided, assumes all MTP layers (if any) are placed on the last
+ pipeline stage. The function returns True only on the last pipeline stage.
+ """
+ mtp_on_this_rank = False
+ pp_rank = parallel_state.get_pipeline_model_parallel_rank()
+ if config.pipeline_model_parallel_layout is not None:
+ # with custom PP layout, we support put MTP layers on any pipeline stage
+ layout = config.pipeline_model_parallel_layout.layout
+ if (
+ not ignore_virtual
+ and parallel_state.get_virtual_pipeline_model_parallel_world_size() is not None
+ ):
+ assert vp_stage is not None, "vp_stage must be passed if virtual pipeline is enabled"
+ num_layers_to_build = layout[pp_rank][vp_stage].count(LayerType.mtp)
+ mtp_on_this_rank = num_layers_to_build > 0
+ else:
+ for vpp_rank in range(len(layout[pp_rank])):
+ num_layers_to_build = layout[pp_rank][vpp_rank].count(LayerType.mtp)
+ if num_layers_to_build > 0:
+ mtp_on_this_rank = True
+ break
+ else:
+ # without custom PP layout, we only support put all of MTP layers on the last pipeline stage
+ if config.mtp_num_layers is not None:
+ mtp_on_this_rank = parallel_state.is_pipeline_last_stage(
+ ignore_virtual=ignore_virtual, vp_stage=vp_stage
+ )
+ else:
+ mtp_on_this_rank = False
+ return mtp_on_this_rank
+
+
+def get_mtp_ranks(pp_ranks: List[int], config: TransformerConfig) -> List[int]:
+ """Get the ranks of the MTP layers."""
+ mtp_ranks = set()
+ if config.mtp_num_layers is None:
+ return []
+ if config.pipeline_model_parallel_layout is None:
+ return [pp_ranks[-1]]
+ layout = config.pipeline_model_parallel_layout.layout
+ for pp_rank in range(len(layout)):
+ for vpp_rank in range(len(layout[pp_rank])):
+ num_layers_to_build = layout[pp_rank][vpp_rank].count(LayerType.mtp)
+ if num_layers_to_build:
+ mtp_ranks.add(pp_ranks[pp_rank])
+ return list(mtp_ranks)
+
+
+def get_mtp_layer_offset(config: TransformerConfig, vp_stage: Optional[int] = None) -> int:
"""Get the offset of the MTP layer."""
- # Currently, we only support put all of MTP layers on the last pipeline stage.
- return 0
+ # TODO(shifangx): Currently, we only support put all of MTP layers
+ # on the last pipeline stage, so the offset is always 0.
+ # We will support more flexible MTP placement in the future.
+ if config.pipeline_model_parallel_size > 1:
+ if config.pipeline_model_parallel_layout:
+ offset = config.pipeline_model_parallel_layout.get_layer_offset(
+ layer_type=LayerType.mtp, vp_stage=vp_stage
+ )
+ else:
+ offset = 0
+ else:
+ offset = 0
+ return offset
def get_mtp_num_layers_to_build(
config: TransformerConfig, vp_stage: Optional[int] = None, pp_rank: Optional[int] = None
) -> int:
"""Get the number of MTP layers to build."""
- # Currently, we only support put all of MTP layers on the last pipeline stage.
- vp_size = config.virtual_pipeline_model_parallel_size
- if pp_rank is None:
- pp_rank = parallel_state.get_pipeline_model_parallel_rank()
- is_last_pp_stage = pp_rank == config.pipeline_model_parallel_size - 1
- if is_vp_last_stage(vp_stage=vp_stage, vp_size=vp_size) and is_last_pp_stage:
- return config.mtp_num_layers if config.mtp_num_layers else 0
+ if config.pipeline_model_parallel_layout is not None:
+ # If we have a custom PP layout, get the number of mtp layers in the layout array.
+ num_layers_to_build = config.pipeline_model_parallel_layout.get_num_layers_to_build(
+ layer_type=LayerType.mtp, vp_stage=vp_stage
+ )
+ assert num_layers_to_build == config.mtp_num_layers or num_layers_to_build == 0, (
+ f"Currently, we only support put all of MTP layers on the last pipeline stage, "
+ f"so the number of MTP layers to build ({num_layers_to_build}) must match "
+ f"mtp_num_layers ({config.mtp_num_layers}) or be 0."
+ )
else:
- return 0
+ if parallel_state.is_pipeline_last_stage(ignore_virtual=False, vp_stage=vp_stage):
+ num_layers_to_build = config.mtp_num_layers if config.mtp_num_layers else 0
+ else:
+ num_layers_to_build = 0
+ return num_layers_to_build
class MTPLossAutoScaler(torch.autograd.Function):
@@ -541,7 +619,7 @@ def __init__(
super().__init__(config=config)
self.sequence_parallel = config.sequence_parallel
self.submodules = submodules
- self.layer_number = layer_number
+ self.layer_number = layer_number + get_mtp_layer_offset(self.config, vp_stage)
self.vp_stage = vp_stage
self.cp_group = pg_collection.cp
@@ -583,8 +661,15 @@ def __init__(
skip_bias_add=False,
is_expert=False,
)
+
+ diff_transformer_layer_offset = self.config.num_layers - get_transformer_layer_offset(
+ self.config, vp_stage
+ )
self.transformer_layer = build_module(
- self.submodules.transformer_layer, config=self.config, vp_stage=vp_stage
+ self.submodules.transformer_layer,
+ config=self.config,
+ vp_stage=vp_stage,
+ layer_number=self.layer_number + diff_transformer_layer_offset,
)
self.final_layernorm = build_module(
@@ -1022,7 +1107,7 @@ def forward(
(Tensor): The mtp loss tensor of shape [b, s].
"""
# get hidden states from previous mtp stages
- offset = get_mtp_layer_offset(self.config)
+ offset = get_mtp_layer_offset(self.config, self.vp_stage)
hidden_states_list = list(torch.chunk(hidden_states, 1 + offset, dim=0))
hidden_states = hidden_states_list[offset]
for layer_number in range(len(self.layers)):
@@ -1067,7 +1152,7 @@ def sharded_state_dict(
sharded_state_dict = super().sharded_state_dict(prefix, sharded_offsets, metadata)
layer_prefix = f'{prefix}layers.'
for layer in self.layers:
- offset = get_mtp_layer_offset(self.config)
+ offset = get_mtp_layer_offset(self.config, self.vp_stage)
sharded_prefix = f'{layer_prefix}{layer.layer_number - 1 }.'
state_dict_prefix = f'{layer_prefix}{layer.layer_number - 1 - offset}.'
diff --git a/megatron/core/transformer/pipeline_parallel_layer_layout.py b/megatron/core/transformer/pipeline_parallel_layer_layout.py
index 56467bf0e9d..3ff2d6d4464 100644
--- a/megatron/core/transformer/pipeline_parallel_layer_layout.py
+++ b/megatron/core/transformer/pipeline_parallel_layer_layout.py
@@ -1,4 +1,4 @@
-# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved.
+# Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
import copy
import logging
@@ -127,15 +127,28 @@ def validate_layer_layout(self, num_layers: int, mtp_num_layers: int):
if LayerType.mtp in self.layout[pp_rank][-1]:
assert (
self.layout[pp_rank][-1].count(LayerType.mtp) == mtp_num_layers
- ), "All of the MTP layers must be in the same stage"
- assert (
- pp_rank == self.pipeline_model_parallel_size - 1
- and LayerType.loss in self.layout[pp_rank][-1]
- ), "MTP layers must be in the last stage together with Loss stage."
+ ), "All of the MTP layers must be in the same one virtual pipeline stage"
+ for vpp_rank in range(self.virtual_pipeline_model_parallel_size - 1):
+ assert LayerType.mtp not in self.layout[0][vpp_rank], (
+ f"Corrently we restrict that the MTP should not be in the first pp rank."
+ f"But got {self.layout[0]} for the first pp rank."
+ )
+ ## Detect MTP standalone usage.
+ mtp_standalone = False
+ for pp_rank in range(self.pipeline_model_parallel_size):
+ if (
+ LayerType.mtp in self.layout[pp_rank][-1]
+ and pp_rank != self.pipeline_model_parallel_size - 1
+ ):
+ mtp_standalone = True
+ break
+
# TODO: remove them in the future once they are supported
if self.flatten_layout.count(LayerType.encoder) > 0:
raise NotImplementedError("Encoder layer is not supported for flexible pipeline layout")
+ return mtp_standalone
+
def get_num_layers_to_build(
self,
layer_type: LayerType = LayerType.decoder,
diff --git a/megatron/core/transformer/spec_utils.py b/megatron/core/transformer/spec_utils.py
index b3de8541734..dbd2e08bccb 100644
--- a/megatron/core/transformer/spec_utils.py
+++ b/megatron/core/transformer/spec_utils.py
@@ -1,9 +1,12 @@
# Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved.
+import logging
import types
from dataclasses import dataclass, field
from typing import Tuple, Union
+logger = logging.getLogger(__name__)
+
@dataclass
class ModuleSpec:
@@ -25,6 +28,7 @@ class ModuleSpec:
module: Union[Tuple, type]
params: dict = field(default_factory=lambda: {})
submodules: type = None
+ metainfo: dict = field(default_factory=lambda: {})
def import_module(module_path: Tuple[str]):
@@ -37,12 +41,16 @@ def import_module(module_path: Tuple[str]):
try:
module = __import__(base_path, globals(), locals(), [name])
except ImportError as e:
- print(f"couldn't import module due to {e}")
+ logger.error(f"couldn't import module due to {e}")
return None
return vars(module)[name]
+# pylint: disable=missing-function-docstring
def get_module(spec_or_module: Union[ModuleSpec, type], **additional_kwargs):
+ """Retrieve the module class or function specified by a ModuleSpec or
+ return it as is if already provided.
+ """
# If a module clas is already provided return it as is
if isinstance(spec_or_module, (type, types.FunctionType)):
return spec_or_module
@@ -56,6 +64,7 @@ def get_module(spec_or_module: Union[ModuleSpec, type], **additional_kwargs):
def build_module(spec_or_module: Union[ModuleSpec, type], *args, **kwargs):
+ """Build a module from a ModuleSpec or return it as is if already provided."""
# If the passed `spec_or_module` is
# a `Function`, then return it as it is
# NOTE: to support an already initialized module add the following condition
diff --git a/megatron/core/transformer/transformer_block.py b/megatron/core/transformer/transformer_block.py
index 2f2ea5adb30..b28a66400e0 100755
--- a/megatron/core/transformer/transformer_block.py
+++ b/megatron/core/transformer/transformer_block.py
@@ -1,4 +1,4 @@
-# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved.
+# Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
import logging
from contextlib import nullcontext
from dataclasses import dataclass
@@ -18,7 +18,7 @@
from megatron.core.packed_seq_params import PackedSeqParams
from megatron.core.pipeline_parallel.utils import is_vp_first_stage, is_vp_last_stage
from megatron.core.process_groups_config import ProcessGroupCollection
-from megatron.core.transformer.enums import LayerType
+from megatron.core.transformer.enums import CudaGraphScope, LayerType
from megatron.core.transformer.module import GraphableMegatronModule, MegatronModule
from megatron.core.transformer.spec_utils import ModuleSpec, build_module
from megatron.core.transformer.transformer_config import TransformerConfig
@@ -374,7 +374,7 @@ def build_layer(layer_spec, layer_number):
# @TODO: add back account_for_embedding_in_pipeline_split (see issue #293)
# In pipeline parallelism, we want to add this LN only to the last stage of the pipeline
# self.post_process and self.post_layer_norm guide this behavior
- if self.submodules.layer_norm and self.post_process and self.post_layer_norm:
+ if self.has_final_layernorm_in_this_stage():
self.final_layernorm = build_module(
self.submodules.layer_norm,
config=self.config,
@@ -384,6 +384,34 @@ def build_layer(layer_spec, layer_number):
else:
self.final_layernorm = None # Either this or nn.Identity
+ def has_final_layernorm_in_this_stage(self):
+ """
+ Check if this vpp stage contains the final layernorm.
+ Note:
+ Final layernorm now has been moved from the post-process stage to the last decoder
+ layer by using this function.
+ There will be a small numeric difference because of grad norm reduction when final
+ layernorm is placed in different pipeline stages in deterministic mode. It can still
+ be bitwise aligned by disabling grad norm clipping.
+ """
+ if self.config.mtp_num_layers is None:
+ # for model without MTPLayer, the final layernorm is set in the stage which does
+ # post_process
+ return self.submodules.layer_norm and self.post_process and self.post_layer_norm
+ else:
+ # for model with MTPLayer, the final layernorm is set in the stage which has the
+ # last layer of the decoder
+ has_final_layernorm_in_this_stage = False
+ for layer in self.layers:
+ if layer.layer_number == self.config.num_layers:
+ has_final_layernorm_in_this_stage = True
+ break
+ return (
+ self.submodules.layer_norm
+ and has_final_layernorm_in_this_stage
+ and self.post_layer_norm
+ )
+
def _get_layer(self, layer_number: int):
return self.layers[layer_number]
@@ -397,12 +425,18 @@ def _checkpointed_forward(
attention_bias: Tensor,
packed_seq_params: PackedSeqParams,
use_inner_quantization_context: bool,
+ padding_mask: Optional[Tensor] = None,
):
"""Forward method with activation checkpointing."""
def custom(start: int, end: int):
def custom_forward(
- hidden_states, attention_mask, context, context_mask, rotary_pos_emb
+ hidden_states,
+ attention_mask,
+ context,
+ context_mask,
+ rotary_pos_emb,
+ padding_mask=None,
):
for index in range(start, end):
layer = self._get_layer(index)
@@ -433,6 +467,7 @@ def custom_forward(
attention_bias=attention_bias,
inference_context=None,
packed_seq_params=packed_seq_params,
+ padding_mask=padding_mask,
)
return hidden_states, context
@@ -452,6 +487,7 @@ def checkpoint_handler(forward_func):
context,
context_mask,
rotary_pos_emb,
+ padding_mask,
)
else:
return tensor_parallel.checkpoint(
@@ -462,6 +498,7 @@ def checkpoint_handler(forward_func):
context,
context_mask,
rotary_pos_emb,
+ padding_mask,
)
if self.config.recompute_method == 'uniform':
@@ -523,7 +560,7 @@ def _should_call_local_cudagraph(self, *args, **kwargs):
kwargs.get('inference_context') is not None
or kwargs.get('inference_params') is not None
)
- and self.config.cuda_graph_scope == 'full_iteration'
+ and CudaGraphScope.full_iteration in self.config.cuda_graph_scope
):
if kwargs['inference_context'].is_static_batching():
using_cuda_graph = kwargs['inference_context'].is_decode_only()
@@ -567,6 +604,7 @@ def forward(
inference_context: Optional[BaseInferenceContext] = None,
packed_seq_params: Optional[PackedSeqParams] = None,
sequence_len_offset: Optional[Tensor] = None,
+ padding_mask: Optional[Tensor] = None,
*,
inference_params: Optional[BaseInferenceContext] = None,
dynamic_inference_decode_only: Optional[bool] = None,
@@ -676,6 +714,7 @@ def forward(
attention_bias=attention_bias,
packed_seq_params=packed_seq_params,
use_inner_quantization_context=use_inner_quantization_context,
+ padding_mask=padding_mask,
)
else:
for l_no, layer in enumerate(self.layers):
@@ -708,6 +747,7 @@ def forward(
inference_context=inference_context,
packed_seq_params=packed_seq_params,
sequence_len_offset=sequence_len_offset,
+ padding_mask=padding_mask,
)
if (
@@ -765,6 +805,12 @@ def sharded_state_dict(
elif isinstance(self.config.moe_layer_freq, list):
non_homogeneous_layers = True
+ if isinstance(self.config.linear_attention_freq, int):
+ if self.config.linear_attention_freq > 1:
+ non_homogeneous_layers = True
+ elif isinstance(self.config.linear_attention_freq, list):
+ non_homogeneous_layers = True
+
if self.config.heterogeneous_block_specs:
non_homogeneous_layers = True
diff --git a/megatron/core/transformer/transformer_config.py b/megatron/core/transformer/transformer_config.py
index e342f717295..8f5462ff55b 100644
--- a/megatron/core/transformer/transformer_config.py
+++ b/megatron/core/transformer/transformer_config.py
@@ -9,8 +9,9 @@
from megatron.core.enums import Fp4Recipe, Fp8Recipe
from megatron.core.quantization.quant_config import RecipeConfig
-from megatron.core.transformer.enums import AttnBackend
+from megatron.core.transformer.enums import AttnBackend, CudaGraphScope
from megatron.core.transformer.pipeline_parallel_layer_layout import PipelineParallelLayerLayout
+from megatron.core.utils import experimental_api
from ..fusions.fused_bias_geglu import quick_gelu
from ..model_parallel_config import ModelParallelConfig
@@ -31,6 +32,7 @@
@dataclass
+@experimental_api
class TransformerConfig(ModelParallelConfig):
"""Configuration object for megatron-core transformers.
@@ -192,6 +194,9 @@ class TransformerConfig(ModelParallelConfig):
qk_layernorm: bool = False
"""Whether to apply `normalization` type of normalization to the query and key embeddings."""
+ qk_l2_norm: bool = False
+ """Whether to apply llama 4-style qk L2 norm."""
+
qk_clip: bool = False
"""Whether to clip the query and key weights. Needed for Muon MLA Model training."""
@@ -205,6 +210,9 @@ class TransformerConfig(ModelParallelConfig):
"""Whether to log the max attention logit across whole model. Decoupled from qk_clip,
defualts to False. Setting qk_clip will automatically log the max logit"""
+ attention_output_gate: bool = False
+ """Whether to apply output gate to the attention layers."""
+
test_mode: bool = False
"""Whether to run real-time tests."""
@@ -222,6 +230,60 @@ class TransformerConfig(ModelParallelConfig):
A list of integers: Defines a custom pattern where 1 means skip RoPE and 0 means apply RoPE.
For example, [0,1,1,0] means: apply RoPE, skip RoPE, skip RoPE, apply RoPE."""
+ ####################
+ # attention variant
+ ####################
+ experimental_attention_variant: Optional[str] = None
+ """Type of attention variant to use. Currently support gated_delta_net and dsa."""
+
+ ####################
+ # DSA
+ ####################
+ dsa_indexer_n_heads: Optional[int] = None
+ """Number of DSA indexer heads."""
+
+ dsa_indexer_head_dim: Optional[int] = None
+ """Dimension per DSA indexer head."""
+
+ dsa_indexer_topk: Optional[int] = None
+ """Number of top-k tokens to select in DSA indexer."""
+
+ dsa_indexer_loss_coeff: Optional[float] = None
+ """Coefficient for the DSA indexer KL divergence loss. Set to 0 to disable indexer loss."""
+
+ dsa_indexer_use_sparse_loss: Optional[bool] = None
+ """Whether to use sparse DSA indexer loss. If True, the indexer loss will be computed using the
+ top-k indices."""
+
+ ####################
+ # linear attention
+ ####################
+ linear_attention_type: Optional[str] = None
+ """Type of linear attention to use.
+ Deprecated. Use experimental_attention_variant instead."""
+
+ linear_attention_freq: Optional[Union[int, List[int]]] = None
+ """Frequency between LA (linear attention) layers
+ and SDPA (scaled dot-product attention) layers.
+ Accepts either:
+ - An integer N: Represents a (N-1):N ratio, meaning (N-1) LA layers for every 1 SDPA layer
+ - A list that defines a custom pattern, e.g.: [1,1,1,0,1,1,1,0,1,1,1,0]"""
+
+ linear_conv_kernel_dim: Optional[int] = None
+ """Conv kernel dimension for the gated delta net."""
+
+ linear_key_head_dim: Optional[int] = None
+ """Query and key head dimension for the gated delta net."""
+
+ linear_value_head_dim: Optional[int] = None
+ """Value and gate head dimension for the gated delta net."""
+
+ linear_num_key_heads: Optional[int] = None
+ """Number of query and key heads for the gated delta net."""
+
+ linear_num_value_heads: Optional[int] = None
+ """Number of value and gate heads for the gated delta net."""
+
####################
# initialization
####################
@@ -460,6 +522,9 @@ class TransformerConfig(ModelParallelConfig):
different orders to the hidden_states, causing minor numerical differences
in the hidden_states gradient."""
+ moe_shared_expert_gate: bool = False
+ """Enable gate for shared expert."""
+
moe_shared_expert_overlap: bool = False
"""Enable overlapping between shared expert computations and dispatcher communications.
Without this, the shared experts execute before the router."""
@@ -489,6 +554,9 @@ class TransformerConfig(ModelParallelConfig):
moe_router_topk: int = 2
"""Number of experts to route to for each token."""
+ enable_routing_replay: bool = False
+ """Enable routing replay for MoE."""
+
moe_router_topk_limited_devices: Optional[int] = None
"""Number of EP ranks to consider for each token in group-limited routing,
DEPRECATED and replaced by moe_router_num_groups and moe_router_group_topk.
@@ -554,6 +622,13 @@ class TransformerConfig(ModelParallelConfig):
"""[Experimental] Force load balancing with random logits for MoE router, supports naive topk
and group-limited topk. This is an experimental feature and only for benchmark."""
+ moe_router_force_biased: Optional[float] = None
+ """[Experimental] Apply random expert bias in normal distribution with specified std
+ to router logits. Shared seed across all ranks ensures identical bias.
+ If positive, generates new random bias each forward pass.
+ If negative, generates bias once per layer and reuses it (abs value is std).
+ This is an experimental feature for benchmarking purposes."""
+
moe_grouped_gemm: bool = False
"""When there are multiple experts per rank, compress multiple local (potentially small) gemms
in a single kernel launch to improve the utilization and performance by leveraging the Grouped
@@ -661,11 +736,11 @@ class TransformerConfig(ModelParallelConfig):
determines the scope of graph capture."""
cuda_graph_use_single_mempool: bool = False
- """When set to true, cudagraphs will be captured inside a single mempool, in which all
- cudagraphs may only be used once per step. If false, cudagraphs may be reused across
- microbatches. Enabling may reduce cudagraph memory overheads due to memory fragmentation,
- however may greatly increase the number of cudagraphs created when the number of microbatches
- is high."""
+ """[For `local` implementation only] When set to true, cudagraphs will be captured inside a
+ single mempool, in which all cudagraphs may only be used once per step. If false, cudagraphs may
+ be reused across microbatches. Enabling may reduce cudagraph memory overheads due to memory
+ fragmentation, however may greatly increase the number of cudagraphs created when the number of
+ microbatches is high."""
cuda_graph_retain_backward_graph: bool = False
"""When set to true, cudagraph backward passes will be graph captured with 'retain_grad=True'
@@ -687,11 +762,10 @@ class TransformerConfig(ModelParallelConfig):
excluding optimizer) is enabled.
"transformer_engine": capture the CUDA graph using TE make_graphed_callables()."""
- cuda_graph_scope: str = "full"
+ cuda_graph_scope: Optional[List[CudaGraphScope]] = None
"""Determines the CUDA graphs capturing scope.
- When cuda_graph_impl is set to "transformer_engine", valid values are "full" and "attn".
- "Full" scope captures a whole Transformer layer. "Attn" scope only captures operations in
- TransformerLayer._forward_attention().
+ When cuda_graph_impl is set to "transformer_engine", valid values are "attn", "mlp", "moe",
+ "moe_router", "moe_preprocess", "mamba". None means the full layer.
When cuda_graph_impl is set to "local", "full_iteration" can be specified as cuda_graph_scope
to enable whole iteration CUDA graph. All other values enable layerwise CUDA graph."""
@@ -779,6 +853,33 @@ class TransformerConfig(ModelParallelConfig):
"""Transformer implementation to use.
Options are 'transformer_engine' for Transformer Engine and 'local' for MCore."""
+ fallback_to_eager_attn: bool = False
+ """Whether to fallback to eager attention in TE implementation.
+ Suggested for when desired features are not available in TE implementation."""
+
+ #####################################
+ # Fine-grained Activation Offloading
+ #####################################
+ fine_grained_activation_offloading: bool = False
+ """If True, offload the input of the specified modules to the CPU.
+ Fine-grained activation offloading is a module-level offloading method
+ instead of a layer-level offloading method like cpu_offloading."""
+
+ offload_modules: Optional[list[str]] = None
+ """The submodules to offload its input.
+ choices: "attn_norm", "qkv_linear", "core_attn", "attn_proj",
+ "mlp_norm", "expert_fc1", "moe_act".
+ "attn_norm": offload the input of the normalization in the attention part.
+ "qkv_linear": offload the input of the qkv linear part.
+ "core_attn": offload the input of the core attention part.
+ "attn_proj": offload the input of the attn linear projection part.
+ "mlp_norm": offload the input of the normalization in the mlp part.
+ "expert_fc1": offload the input of the expert fc1 part.
+ "moe_act": offload the input of the moe act part.
+ """
+ min_offloaded_tensor_size: int = 1024 * 1024
+ """The minimum size of the tensor to be offloaded."""
+
def __post_init__(self):
"""Python dataclass method that is used to modify attributes after initialization.
See https://docs.python.org/3/library/dataclasses.html#post-init-processing for more
@@ -818,6 +919,57 @@ def __post_init__(self):
f"tensor_model_parallel_size ({self.tensor_model_parallel_size})."
)
+ if self.linear_attention_type is not None:
+ warnings.warn(
+ "linear_attention_type is deprecated, "
+ "use experimental_attention_variant instead."
+ )
+ self.experimental_attention_variant = self.linear_attention_type
+ self.linear_attention_type = None
+
+ if self.experimental_attention_variant in ["gated_delta_net"]:
+ assert (
+ self.linear_attention_freq is not None
+ ), f"linear_attention_freq must be set for linear attention."
+
+ if self.experimental_attention_variant == "gated_delta_net":
+ # Check required parameters
+ assert (
+ self.linear_conv_kernel_dim is not None
+ ), "linear_conv_kernel_dim must be set for gated delta net."
+ assert (
+ self.linear_key_head_dim is not None
+ ), "linear_key_head_dim must be set for gated delta net."
+ assert (
+ self.linear_value_head_dim is not None
+ ), "linear_value_head_dim must be set for gated delta net."
+ assert (
+ self.linear_num_key_heads is not None
+ ), "linear_num_key_heads must be set for gated delta net."
+ assert (
+ self.linear_num_value_heads is not None
+ ), "linear_num_value_heads must be set for gated delta net."
+ assert self.linear_num_value_heads % self.linear_num_key_heads == 0, (
+ f"linear_num_value_heads ({self.linear_num_value_heads}) must be a multiple of "
+ f"linear_num_key_heads ({self.linear_num_key_heads})."
+ )
+
+ # Check tensor parallelism compatibility
+ tp_cp_size = self.tensor_model_parallel_size * self.context_parallel_size
+ assert self.linear_num_key_heads % tp_cp_size == 0, (
+ f"{self.linear_num_key_heads=} must be a multiple of "
+ f"({self.tensor_model_parallel_size=} * {self.context_parallel_size=})."
+ )
+ assert self.linear_num_value_heads % tp_cp_size == 0, (
+ f"{self.linear_num_value_heads=} must be a multiple of "
+ f"({self.tensor_model_parallel_size=} * {self.context_parallel_size=})."
+ )
+ elif self.experimental_attention_variant == "dsa":
+ assert (
+ self.context_parallel_size == 1
+ ), "Currently context parallelism is not supported by DSAttention!"
+ assert not self.apply_rope_fusion, "RoPE fusion is not supported for DSAttention"
+
if self.fp8:
# cannot support first last layer bf16 with delayed scaling
if self.first_last_layers_bf16 and self.fp8_recipe == Fp8Recipe.delayed:
@@ -1098,6 +1250,32 @@ def __post_init__(self):
if "moe" not in self.recompute_modules:
self.recompute_modules.append("moe")
+ if self.fine_grained_activation_offloading:
+ assert (
+ not self.cpu_offloading
+ ), "fine_grained_activation_offloading cannot be enabled with cpu_offloading."
+ assert self.offload_modules is not None and len(self.offload_modules) > 0
+ allowed_modules = {
+ "core_attn",
+ "attn_proj",
+ "expert_fc1",
+ "moe_act",
+ "attn_norm",
+ "mlp_norm",
+ "qkv_linear",
+ }
+ invalid_modules = set(self.offload_modules) - allowed_modules
+ assert not invalid_modules, (
+ f'Invalid choices for offload_modules: {invalid_modules}. '
+ f'Allowed modules are: {allowed_modules}'
+ )
+ if "attn_proj" in self.offload_modules and "core_attn" not in self.offload_modules:
+ raise ValueError(
+ "attn_proj cannot be set to offload_modules alone without core_attn "
+ "because the input of attn_proj is the output of core_attn, "
+ "which is needed in core_attn.backward()."
+ )
+
if (
self.num_layers_in_first_pipeline_stage is not None
or self.num_layers_in_last_pipeline_stage is not None
@@ -1159,7 +1337,7 @@ def __post_init__(self):
self.virtual_pipeline_model_parallel_size = detected_vpp_size
# Check whether the layout is valid.
- self.pipeline_model_parallel_layout.validate_layer_layout(
+ self.mtp_standalone = self.pipeline_model_parallel_layout.validate_layer_layout(
num_layers=self.num_layers, mtp_num_layers=self.mtp_num_layers
)
@@ -1351,6 +1529,10 @@ def __post_init__(self):
"apply_rope_fusion is not available. Please install TE >= 1.4."
)
+ if self.fused_single_qkv_rope:
+ if self.attention_output_gate:
+ raise ValueError("fused_single_qkv_rope does not support gated attention for now.")
+
if self.multi_latent_attention and self.rotary_interleaved:
raise ValueError("rotary_interleaved does not work with multi_latent_attention.")
@@ -1480,30 +1662,135 @@ def __post_init__(self):
'use cuda_graph_impl=transformer_engine instead.'
)
self.cuda_graph_impl = "transformer_engine"
+
+ if self.cuda_graph_scope is None:
+ self.cuda_graph_scope = []
+ elif not isinstance(self.cuda_graph_scope, list):
+ if isinstance(self.cuda_graph_scope, CudaGraphScope):
+ self.cuda_graph_scope = [self.cuda_graph_scope]
+ else:
+ assert isinstance(self.cuda_graph_scope, str), (
+ "cuda_graph_scope must be a string that can be converted to a list of "
+ f"CudaGraphScope, got {self.cuda_graph_scope}."
+ )
+ self.cuda_graph_scope = self.cuda_graph_scope.split(',')
+ if all(isinstance(scope, str) for scope in self.cuda_graph_scope):
+ # Backward compatibility for "full" scope. Now we use an empty list instead.
+ if "full" in self.cuda_graph_scope:
+ assert self.cuda_graph_scope == [
+ "full"
+ ], "full scope cannot be used with other scopes."
+ warnings.warn(
+ "full scope is deprecated. "
+ "Use empty cuda_graph_scope to capture the whole layer."
+ )
+ self.cuda_graph_scope = []
+ else:
+ self.cuda_graph_scope = [CudaGraphScope[scope] for scope in self.cuda_graph_scope]
+ assert all(
+ isinstance(scope, CudaGraphScope) for scope in self.cuda_graph_scope
+ ), f"cuda_graph_scope must be a list of CudaGraphScope, got {self.cuda_graph_scope}."
+
if self.cuda_graph_impl != "none":
assert self.cuda_graph_impl in [
"transformer_engine",
"local",
], f"Invalid cuda graph implementation: {self.cuda_graph_impl}"
+
if self.cpu_offloading:
raise ValueError("CUDA graphs not supported with CPU offloading.")
- if self.recompute_granularity:
- if (
- self.recompute_granularity != "selective"
- or self.cuda_graph_impl != "transformer_engine"
- or self.cuda_graph_scope != "attn"
- ):
- raise ValueError("CUDA graphs not supported with activation recomputation.")
+
+ if self.cuda_graph_impl == "local":
+ assert not self.cuda_graph_scope or self.cuda_graph_scope == [
+ CudaGraphScope.full_iteration
+ ], (
+ "For local cuda graph implementation, the only valid value for "
+ "cuda_graph_scope is full_iteration, or an empty list to denote layerwise "
+ "graphs. To use other scopes, use cuda_graph_impl=transformer_engine."
+ )
+
+ if self.cuda_graph_impl == "transformer_engine":
+ assert CudaGraphScope.full_iteration not in self.cuda_graph_scope, (
+ "To use full iteration cuda graph, please use "
+ "cuda_graph_impl=local instead of cuda_graph_impl=transformer_engine."
+ )
+ assert (
+ CudaGraphScope.moe not in self.cuda_graph_scope
+ or CudaGraphScope.moe_router not in self.cuda_graph_scope
+ ), 'cuda_graph_scope must not contain both moe and moe_router.'
+ if CudaGraphScope.moe_preprocess in self.cuda_graph_scope:
+ assert (
+ CudaGraphScope.moe_router in self.cuda_graph_scope
+ ), 'moe_preprocess cuda graph is only supported with moe_router cuda graph.'
+ if self.num_moe_experts is None or self.num_moe_experts <= 1:
+ assert (
+ CudaGraphScope.moe not in self.cuda_graph_scope
+ and CudaGraphScope.moe_router not in self.cuda_graph_scope
+ ), 'moe cuda graph is only supported for MoE.'
else:
- for module in self.recompute_modules:
- if module in ['core_attn', 'mla_up_proj']:
- raise ValueError(
- f'attn cuda graph is not supported with {module} recompute.'
+ if self.moe_layer_freq == 1 or (
+ isinstance(self.moe_layer_freq, list) and 0 not in self.moe_layer_freq
+ ):
+ assert CudaGraphScope.mlp not in self.cuda_graph_scope, (
+ 'mlp cuda graph is only supported for dense layers, '
+ 'but not found in the model.'
+ )
+ if (
+ self.moe_expert_capacity_factor is None
+ or not self.moe_pad_expert_input_to_capacity
+ ):
+ assert (
+ CudaGraphScope.moe not in self.cuda_graph_scope
+ ), 'moe cuda graph is only supported with drop-padding MoE.'
+ if self.moe_token_dispatcher_type == 'alltoall' and (
+ self.moe_expert_capacity_factor is not None
+ or self.moe_router_padding_for_quantization
+ ):
+ assert CudaGraphScope.moe_preprocess not in self.cuda_graph_scope, (
+ 'moe_preprocess cuda graph is not supported when there are '
+ 'DtoH copies and synchronizations in the preprocess step.'
)
- if "layernorm" in self.recompute_modules:
- warnings.warn(
- "input_layernorm recompute is not supported with attention "
- "cudagraph. Will only recompute the pre_mlp_layernorm."
+
+ if self.recompute_granularity:
+ if self.recompute_granularity != "selective":
+ assert self.cuda_graph_scope == [
+ CudaGraphScope.full_iteration
+ ], "full recompute is only supported with full iteration CUDA graph."
+ else:
+ # The recompute module should be inside or outside of the graph scope.
+ # Recompute module coverring graph scope is not allowed.
+ if "moe" in self.recompute_modules:
+ assert (
+ CudaGraphScope.moe_router not in self.cuda_graph_scope
+ ), "moe recompute is not supported with moe_router CUDA graph."
+ # Graphed recompute module doesn't accept random number.
+ if (
+ not self.cuda_graph_scope
+ or CudaGraphScope.full_iteration in self.cuda_graph_scope
+ ):
+ full_cudagraph = True
+ else:
+ full_cudagraph = False
+ if self.attention_dropout != 0.0:
+ assert (
+ not full_cudagraph and CudaGraphScope.attn not in self.cuda_graph_scope
+ ) or "core_attn" not in self.recompute_modules, (
+ "attention dropout is not supported with graphed attention "
+ "recomputation."
+ )
+ if self.hidden_dropout != 0.0:
+ assert (
+ (not full_cudagraph and CudaGraphScope.mlp not in self.cuda_graph_scope)
+ or "mlp" not in self.recompute_modules
+ ) and (
+ (not full_cudagraph and CudaGraphScope.moe not in self.cuda_graph_scope)
+ or "moe" not in self.recompute_modules
+ ), "hidden dropout is not supported with graphed MLP/MoE recomputation."
+ if self.moe_input_jitter_eps is not None:
+ assert (
+ not full_cudagraph and CudaGraphScope.moe not in self.cuda_graph_scope
+ ) or "moe" not in self.recompute_modules, (
+ "moe_input_jitter_eps is not supported with graphed moe recomputation."
)
if self.moe_token_dispatcher_type in ["allgather"]:
@@ -1571,6 +1858,21 @@ def __post_init__(self):
assert (
self.mtp_num_layers is None or self.mtp_num_layers == 1
), 'MTP layernum only supports 1 when enabling overlap_moe_expert_parallel_comm.'
+ if self.mtp_num_layers == 1:
+ assert self.pipeline_model_parallel_size > 1, (
+ 'Pipeline model parallel size must be larger than 1 '
+ 'when enabling overlap_moe_expert_parallel_comm with MTP layer.'
+ )
+
+ if self.cuda_graph_impl != "none":
+ assert (
+ self.cuda_graph_impl == "transformer_engine"
+ and CudaGraphScope.moe not in self.cuda_graph_scope
+ and CudaGraphScope.mlp not in self.cuda_graph_scope
+ ), (
+ 'CUDA graph scope on moe and mlp is not '
+ 'supported with overlap_moe_expert_parallel_comm'
+ )
# Check delay_wgrad_compute compatibility
if self.delay_wgrad_compute:
@@ -1580,6 +1882,17 @@ def __post_init__(self):
assert (
not self.moe_use_legacy_grouped_gemm
), 'delay_wgrad_compute is not supported with legacy groupedgemm implementation'
+ if self.cuda_graph_impl == "transformer_engine":
+ assert is_te_min_version("2.10.0"), (
+ 'TE version >= 2.10.0 is required for delay_wgrad_compute with '
+ 'partial cuda graph'
+ )
+
+ if self.ep_overlap_early_attn_memory_release:
+ assert self.overlap_moe_expert_parallel_comm, (
+ 'overlap_moe_expert_parallel_comm must be enabled when enabling '
+ 'ep_overlap_early_attn_memory_release'
+ )
if self.context_parallel_size > 1 and self.cp_comm_type is not None:
if isinstance(self.cp_comm_type, list):
@@ -1633,6 +1946,25 @@ def __post_init__(self):
f"the number of layers ({self.num_layers})"
)
+ if self.fallback_to_eager_attn:
+ assert self.transformer_impl == "transformer_engine", (
+ f"fallback_to_eager_attn is only available with transformer_engine implementation,"
+ f" but got {self.transformer_impl=}."
+ )
+
+ if self.fallback_to_eager_attn or self.transformer_impl == "local":
+ if self.context_parallel_size > 1 and self.cp_comm_type is not None:
+ all_cp_comm_types_are_all_gather = (
+ all(item == "all_gather" for item in self.cp_comm_type)
+ if isinstance(self.cp_comm_type, list)
+ else self.cp_comm_type == "all_gather"
+ )
+ if not all_cp_comm_types_are_all_gather:
+ raise ValueError(
+ f"fallback_to_eager_attn only supports all_gather communication type "
+ f"for context parallelism, but got {self.cp_comm_type=} instead."
+ )
+
if self.transformer_impl == "inference_optimized":
assert self.normalization == "RMSNorm"
assert not self.layernorm_zero_centered_gamma
@@ -1647,6 +1979,7 @@ def __post_init__(self):
@dataclass
+@experimental_api
class MLATransformerConfig(TransformerConfig):
"""Configuration object for megatron-core Multi-Latent Attention (MLA) transformers.
@@ -1712,6 +2045,9 @@ def __post_init__(self):
if self.multi_latent_attention and self.apply_rope_fusion and self.rope_type != "yarn":
raise ValueError("apply_rope_fusion for MLA only works with YARN RoPE.")
+ if self.attention_output_gate:
+ raise NotImplementedError("Output gate is not supported for MLA yet.")
+
if self.cache_mla_latents:
assert (
self.apply_rope_fusion is False
diff --git a/megatron/core/transformer/transformer_layer.py b/megatron/core/transformer/transformer_layer.py
index b32614037a2..a486b6ed3d5 100644
--- a/megatron/core/transformer/transformer_layer.py
+++ b/megatron/core/transformer/transformer_layer.py
@@ -1,5 +1,6 @@
-# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved.
+# Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+import functools
import logging
import warnings
from abc import ABC
@@ -15,7 +16,8 @@
from megatron.core.dist_checkpointing.utils import apply_prefix_mapping
from megatron.core.packed_seq_params import PackedSeqParams
from megatron.core.process_groups_config import ProcessGroupCollection
-from megatron.core.transformer.enums import LayerType
+from megatron.core.transformer.cuda_graphs import is_graph_capturing
+from megatron.core.transformer.enums import CudaGraphScope, LayerType
from megatron.core.transformer.identity_op import IdentityFuncOp, IdentityOp
from megatron.core.transformer.mlp import MLP
from megatron.core.transformer.module import GraphableMegatronModule
@@ -372,19 +374,63 @@ def __init__(
# [Module 9: BiasDropoutFusion]
self.mlp_bda = build_module(submodules.mlp_bda)
+ self.is_moe_layer = isinstance(self.mlp, MoELayer)
+
self.recompute_input_layernorm = False
self.recompute_pre_mlp_layernorm = False
self.recompute_mlp = False
if self.config.recompute_granularity == 'selective':
if "layernorm" in self.config.recompute_modules:
- if (
- not isinstance(self.input_layernorm, IdentityOp)
- and self.config.cuda_graph_impl == "none"
- ):
+ if not isinstance(self.input_layernorm, IdentityOp):
self.recompute_input_layernorm = True
if self.config.fp8 or self.config.fp4:
self.self_attention.set_for_recompute_input_layernorm()
- if not isinstance(self.pre_mlp_layernorm, IdentityOp):
+
+ def can_recompute_pre_mlp_layernorm_for_cudagraph():
+ if (
+ not self.is_moe_layer
+ or CudaGraphScope.moe_router not in self.config.cuda_graph_scope
+ ):
+ # Not a MoE layer, or not capturing the router part.
+ return True
+ if (
+ self.config.moe_shared_expert_intermediate_size is not None
+ and self.config.moe_shared_expert_overlap
+ ):
+ # If shared expert overlap is used, we cannot make the pre-mlp layernorm
+ # recomputation, because the shared expert takes the layernorm output as
+ # input, and it is outside of the CUDA graph scope.
+ log_single_rank(
+ logger,
+ logging.WARNING,
+ "pre_mlp_layernorm recompute is not supported with moe router "
+ "cudagraph + shared expert overlap. Disabling pre_mlp_layernorm "
+ "recompute.",
+ )
+ return False
+ if CudaGraphScope.moe_preprocess in self.config.cuda_graph_scope and (
+ self.config.moe_token_dispatcher_type == "alltoall"
+ or self.config.moe_latent_size
+ ):
+ # Only when capturing the preprocess part and using alltoall token
+ # dispatcher or latent MoE can we make the pre-mlp layernorm recomputation.
+ # Because in other cases the layernorm output returns directly as one of the
+ # outputs of the cudagraph, which will be allocated a static buffer, thus
+ # not able to be released.
+ return True
+ log_single_rank(
+ logger,
+ logging.WARNING,
+ "pre_mlp_layernorm recompute is only supported with moe router + "
+ "preprocess cudagraph will alltoall token dispatcher or latent MoE. "
+ "Disabling pre_mlp_layernorm recompute.",
+ )
+ return False
+
+ if (
+ not isinstance(self.pre_mlp_layernorm, IdentityOp)
+ and can_recompute_pre_mlp_layernorm_for_cudagraph()
+ ):
self.recompute_pre_mlp_layernorm = True
if self.config.fp8 or self.config.fp4:
if isinstance(self.mlp, MoELayer):
@@ -396,8 +442,18 @@ def __init__(
set_save_original_input(self.mlp.linear_fc1)
if "mlp" in self.config.recompute_modules:
- if not isinstance(self.mlp, MoELayer):
+ if not self.is_moe_layer:
self.recompute_mlp = True
+ self.offload_attn_norm = (
+ self.config.fine_grained_activation_offloading
+ and "attn_norm" in self.config.offload_modules
+ and not isinstance(self.input_layernorm, IdentityOp)
+ )
+ self.offload_mlp_norm = (
+ self.config.fine_grained_activation_offloading
+ and "mlp_norm" in self.config.offload_modules
+ and not isinstance(self.pre_mlp_layernorm, IdentityOp)
+ )
# @jcasper how should we handle nvfuser?
# Set bias+dropout+add fusion grad_enable execution handler.
@@ -433,7 +489,12 @@ def forward(self, *args, **kwargs):
# runners in the cuda graph manager
kwargs.pop("dynamic_inference_decode_only", None)
hidden_states, context = self._forward_attention(*args, **kwargs)
- output = self._forward_mlp(hidden_states, kwargs.get("inference_context", None))
+
+ output = self._forward_mlp(
+ hidden_states,
+ kwargs.get("inference_context", None),
+ padding_mask=kwargs.get("padding_mask", None),
+ )
return output, context
def _forward_attention(
@@ -450,6 +511,7 @@ def _forward_attention(
inference_context: Optional[Any] = None,
packed_seq_params: Optional[PackedSeqParams] = None,
sequence_len_offset: Optional[Tensor] = None,
+ padding_mask: Optional[Tensor] = None,
*,
inference_params: Optional[Any] = None,
):
@@ -480,6 +542,9 @@ def _forward_attention(
context (Tensor): Updated context tensor if cross-attention is used,
otherwise None.
"""
+ from megatron.core.pipeline_parallel.fine_grained_activation_offload import (
+ FineGrainedActivationOffloadingInterface as off_interface,
+ )
inference_context = deprecate_inference_params(inference_context, inference_params)
@@ -489,11 +554,13 @@ def _forward_attention(
# Optional Input Layer norm
if self.recompute_input_layernorm:
self.input_layernorm_checkpoint = tensor_parallel.CheckpointWithoutOutput()
- input_layernorm_output = self.input_layernorm_checkpoint.checkpoint(
- self.input_layernorm, hidden_states
- )
+ with off_interface(self.offload_attn_norm, hidden_states, "attn_norm") as hidden_states:
+ input_layernorm_output = self.input_layernorm_checkpoint.checkpoint(
+ self.input_layernorm, hidden_states
+ )
else:
- input_layernorm_output = self.input_layernorm(hidden_states)
+ with off_interface(self.offload_attn_norm, hidden_states, "attn_norm") as hidden_states:
+ input_layernorm_output = self.input_layernorm(hidden_states)
# Self attention.
nvtx_range_push(suffix="self_attention")
@@ -527,6 +594,13 @@ def _forward_attention(
)
nvtx_range_pop(suffix="self_attn_bda")
+ # Delay the offload of the attention norm until after the self_attn_bda has been computed
+ # because the residual is needed in the self_attn_bda.
+ if self.offload_attn_norm:
+ hidden_states = off_interface.group_commit(
+ hidden_states, name="attn_norm", forced_released_tensors=[residual]
+ )
+
# Residual connection.
residual = hidden_states
@@ -553,28 +627,40 @@ def _forward_attention(
return hidden_states, context
- def _forward_mlp(self, hidden_states, inference_context=None):
+ def _forward_mlp(self, hidden_states, inference_context=None, padding_mask=None):
"""
Perform a forward pass through the feed-forward layer.
Args:
hidden_states (Tensor): Transformed hidden states before the MLP layernorm.
+ Shape [seq_length, batch_size, hidden_size].
+ inference_context: Inference context for optimizations.
+ padding_mask (Tensor, optional): Padding mask for MoE routing.
+ Shape [bsz, seq_length]. True = padding (exclude), False = valid (include).
+ Only used for MoE layers to exclude padding tokens from aux loss computations.
+ The MoELayer will internally transform this to [seq_length, bsz] format.
Returns:
output (Tensor): Transformed hidden states of shape [s, b, h].
"""
+ from megatron.core.pipeline_parallel.fine_grained_activation_offload import (
+ FineGrainedActivationOffloadingInterface as off_interface,
+ )
+
# Residual connection.
residual = hidden_states
# Optional Layer norm post the cross-attention.
if self.recompute_pre_mlp_layernorm:
self.pre_mlp_norm_checkpoint = tensor_parallel.CheckpointWithoutOutput()
- pre_mlp_layernorm_output = self.pre_mlp_norm_checkpoint.checkpoint(
- self.pre_mlp_layernorm, hidden_states
- )
+ with off_interface(self.offload_mlp_norm, hidden_states, "mlp_norm") as hidden_states:
+ pre_mlp_layernorm_output = self.pre_mlp_norm_checkpoint.checkpoint(
+ self.pre_mlp_layernorm, hidden_states
+ )
else:
- pre_mlp_layernorm_output = self.pre_mlp_layernorm(hidden_states)
+ with off_interface(self.offload_mlp_norm, hidden_states, "mlp_norm") as hidden_states:
+ pre_mlp_layernorm_output = self.pre_mlp_layernorm(hidden_states)
nvtx_range_push(suffix="mlp")
# Potentially chunk the MLP computation during prefill to minimize the peak activation size
@@ -596,10 +682,13 @@ def _forward_mlp(self, hidden_states, inference_context=None):
tensor_parallel.random.get_cuda_rng_tracker,
self.pg_collection.tp,
pre_mlp_layernorm_output,
+ padding_mask=padding_mask,
)
else:
mlp_output_with_bias = tensor_parallel.checkpoint(
- self.mlp, False, pre_mlp_layernorm_output
+ functools.partial(self.mlp, padding_mask=padding_mask),
+ False,
+ pre_mlp_layernorm_output,
)
elif should_chunk_mlp_for_prefill:
# Chunk input along sequence dimension
@@ -614,9 +703,8 @@ def _forward_mlp(self, hidden_states, inference_context=None):
bias_chunks = [bias for _, bias in outputs if bias is not None]
bias_output = torch.stack(bias_chunks, dim=0).sum(dim=0) if bias_chunks else None
mlp_output_with_bias = (mlp_output, bias_output)
-
else:
- mlp_output_with_bias = self.mlp(pre_mlp_layernorm_output)
+ mlp_output_with_bias = self.mlp(pre_mlp_layernorm_output, padding_mask=padding_mask)
if self.recompute_pre_mlp_layernorm:
# discard the output of the pre-mlp layernorm and register the recompute
@@ -626,6 +714,40 @@ def _forward_mlp(self, hidden_states, inference_context=None):
)
nvtx_range_pop(suffix="mlp")
+ if (
+ self.is_moe_layer
+ and self.config.cuda_graph_impl == "transformer_engine"
+ and self.training
+ and is_graph_capturing()
+ and CudaGraphScope.moe_router in self.config.cuda_graph_scope
+ ):
+ if self.recompute_pre_mlp_layernorm:
+ # Register the recompute hooks to all the cudagraph output tensors, because some
+ # tensors are in parallel execution paths and they all need pre_mlp_layernorm to be
+ # recomputed in backward pass. For example, the router path and the shared expert
+ # path. So only register in one path is risky.
+ for tensor in mlp_output_with_bias[1:]:
+ self.pre_mlp_norm_checkpoint.discard_output_and_register_recompute(tensor)
+ return list(mlp_output_with_bias) + [residual]
+ else:
+ return self._forward_post_mlp(mlp_output_with_bias, residual)
+
+ def _forward_post_mlp(self, mlp_output_with_bias, residual):
+ """
+ Perform operations after the MLP computation.
+
+ Args:
+ mlp_output_with_bias (Tensor): Output tensor of the MLP layer with bias.
+ residual (Tensor): Residual tensor.
+
+ Returns:
+ output (Tensor): Transformed hidden states of shape [s, b, h].
+ """
+
+ from megatron.core.pipeline_parallel.fine_grained_activation_offload import (
+ FineGrainedActivationOffloadingInterface as off_interface,
+ )
+
# TODO: could we move `bias_dropout_add_exec_handler` itself
# inside the module provided in the `bias_dropout_add_spec` module?
nvtx_range_push(suffix="mlp_bda")
@@ -634,6 +756,12 @@ def _forward_mlp(self, hidden_states, inference_context=None):
mlp_output_with_bias, residual, self.hidden_dropout
)
nvtx_range_pop(suffix="mlp_bda")
+ # Delay the offload of the mlp norm until after the mlp_bda has been computed
+ # because the residual is needed in the mlp_bda.
+ if self.offload_mlp_norm:
+ hidden_states = off_interface.group_commit(
+ hidden_states, name="mlp_norm", forced_released_tensors=[residual]
+ )
# Jit compiled function creates 'view' tensor. This tensor
# potentially gets saved in the MPU checkpoint function context,
@@ -680,7 +808,9 @@ def get_layer_static_inputs(self, seq_length, micro_batch_size):
"""
static_inputs = super().get_layer_static_inputs(seq_length, micro_batch_size)
- if not isinstance(self.self_attention, IdentityOp):
+ if not isinstance(self.self_attention, IdentityOp) and (
+ not self.config.cuda_graph_scope or CudaGraphScope.attn in self.config.cuda_graph_scope
+ ):
slen_per_cp = seq_length // self.config.context_parallel_size
static_inputs["attention_mask"] = (
~(torch.tril(torch.ones((slen_per_cp, seq_length))).bool())
@@ -694,18 +824,28 @@ def _get_submodules_under_cudagraphs(self):
"""
Get the submodules that are covered by cudagraphs.
"""
- if self.config.cuda_graph_scope == 'full':
- submodules = [self]
- else:
- assert (
- self.config.cuda_graph_scope == 'attn'
- ), f"Invalid cuda_graph_scope {self.config.cuda_graph_scope}"
- submodules = [
+ if not self.config.cuda_graph_scope:
+ return super()._get_submodules_under_cudagraphs()
+
+ submodules = []
+ if CudaGraphScope.attn in self.config.cuda_graph_scope:
+ submodules += [
self.input_layernorm,
self.self_attention,
self.pre_cross_attn_layernorm,
self.cross_attention,
]
+ if (not self.is_moe_layer and CudaGraphScope.mlp in self.config.cuda_graph_scope) or (
+ self.is_moe_layer and CudaGraphScope.moe in self.config.cuda_graph_scope
+ ):
+ submodules += [self.pre_mlp_layernorm, self.mlp]
+ elif self.is_moe_layer and CudaGraphScope.moe_router in self.config.cuda_graph_scope:
+ submodules += [self.pre_mlp_layernorm, self.mlp.router]
+ if (
+ self.config.moe_shared_expert_intermediate_size is not None
+ and not self.config.moe_shared_expert_overlap
+ ):
+ submodules += [self.mlp.shared_experts]
return submodules
def _te_cuda_graph_capture(self, *args, **kwargs):
@@ -716,12 +856,31 @@ def _te_cuda_graph_capture(self, *args, **kwargs):
attribute can be set to control the scope of the CUDA graph.
2. If context is None, it cannot be returned as output.
"""
- hidden_states, context = self._forward_attention(*args, **kwargs)
-
- if self.config.cuda_graph_scope == "full":
+ context = None
+ if not self.config.cuda_graph_scope or CudaGraphScope.attn in self.config.cuda_graph_scope:
+ hidden_states, context = self._forward_attention(*args, **kwargs)
+ else:
+ if len(args) > 0:
+ hidden_states = args[0]
+ else:
+ hidden_states = kwargs.pop("hidden_states")
+
+ if (
+ not self.config.cuda_graph_scope
+ or (not self.is_moe_layer and CudaGraphScope.mlp in self.config.cuda_graph_scope)
+ or (
+ self.is_moe_layer
+ and (
+ CudaGraphScope.moe in self.config.cuda_graph_scope
+ or CudaGraphScope.moe_router in self.config.cuda_graph_scope
+ )
+ )
+ ):
hidden_states = self._forward_mlp(hidden_states)
- cuda_graph_outputs = [hidden_states]
-
+ if not isinstance(hidden_states, list) and not isinstance(hidden_states, tuple):
+ cuda_graph_outputs = [hidden_states]
+ else:
+ cuda_graph_outputs = list(hidden_states)
if context is not None:
cuda_graph_outputs.append(context)
return tuple(cuda_graph_outputs)
@@ -733,6 +892,11 @@ def _te_cuda_graph_replay(self, *args, **kwargs):
However, CUDA graph accepts only Tensor inputs.
Hence, `inference_context` and `packed_seq_params` are excluded from input list.
"""
+ context = None
+ if self.config.cuda_graph_scope and CudaGraphScope.attn not in self.config.cuda_graph_scope:
+ hidden_states, context = self._forward_attention(*args, **kwargs)
+ args = (hidden_states,)
+ kwargs = {}
assert (kwargs.get('inference_context') is None) and (
kwargs.get('packed_seq_params') is None
@@ -742,19 +906,94 @@ def _te_cuda_graph_replay(self, *args, **kwargs):
"For inference cuda graph, please use cuda_graph_impl=local instead."
)
- cuda_graph_output = super()._te_cuda_graph_replay(*args, **kwargs)
+ cuda_graph_output = list(super()._te_cuda_graph_replay(*args, **kwargs))
if kwargs.get('context') is not None:
- context = cuda_graph_output[-1]
- cuda_graph_output = cuda_graph_output[:-1]
+ context = cuda_graph_output.pop()
+
+ if (
+ not self.config.cuda_graph_scope
+ or (not self.is_moe_layer and CudaGraphScope.mlp in self.config.cuda_graph_scope)
+ or (self.is_moe_layer and CudaGraphScope.moe in self.config.cuda_graph_scope)
+ ):
+ # CUDA Graph captures the whole MLP/MoE part. CUDA Graph output is the layer output.
+ assert len(cuda_graph_output) == 1, "CUDA Graph output should be the layer output."
+ output = cuda_graph_output.pop()
+ assert (
+ not self.config.overlap_moe_expert_parallel_comm
+ ), "EP overlap must be \
+ disabled when CUDA graph captures the whole MLP/MoE part."
+ elif self.is_moe_layer and CudaGraphScope.moe_router in self.config.cuda_graph_scope:
+ # CUDA Graph partially captures the MoE.
+ # The rest of the layer should go to the normal pass.
+ shared_expert_output, routing_map = None, None
+ # residual is the last element in the CUDA graph output.
+ residual = cuda_graph_output.pop()
+ if (
+ self.config.moe_shared_expert_intermediate_size is not None
+ and not self.config.moe_shared_expert_overlap
+ ):
+ # The shared expert output is the last second element in the CUDA graph output.
+ shared_expert_output = cuda_graph_output.pop()
+
+ if CudaGraphScope.moe_preprocess in self.config.cuda_graph_scope:
+ # CUDA graph output is [hidden_states, probs] + attributes outputs.
+ (hidden_states, probs), attr_outputs = cuda_graph_output[:2], cuda_graph_output[2:]
+ valid_cudagraph_attrs = self.mlp.token_dispatcher.valid_cudagraph_attrs
+ assert len(attr_outputs) == len(
+ valid_cudagraph_attrs
+ ), f"attr_outputs: {len(attr_outputs)} != {len(valid_cudagraph_attrs)}"
+ for i, attr_name in enumerate(valid_cudagraph_attrs):
+ hier_attr_name = attr_name.split('.')
+ attr = self.mlp.token_dispatcher
+ for name in hier_attr_name[:-1]:
+ attr = getattr(attr, name)
+ setattr(attr, hier_attr_name[-1], attr_outputs[i])
+ else:
+ # CUDA graph output is [hidden_states, probs, routing_map].
+ assert len(cuda_graph_output) == 3, (
+ "CUDA graph output should be [hidden_states, probs, routing_map], "
+ f"but got {len(cuda_graph_output)} elements"
+ )
+ hidden_states, probs, routing_map = cuda_graph_output
+
+ # Resume the MoELayer forward pass from the end of the CUDA graph scope.
+ # The MoE layer will skip redundant computations when we pass in the calculated values
+ # through the keyword arguments. See MoELayer.forward docstring for more details.
+ nvtx_range_push(suffix="mlp")
+ self.mlp.cudagraph_tensor_store.set(
+ hidden_states=hidden_states,
+ probs=probs,
+ routing_map=routing_map,
+ shared_expert_output=shared_expert_output,
+ )
+ # If EP overlap is enabled, remaining of mlp will be called as fine_grained_callables
+ # and should be skipped here.
+ if self.config.overlap_moe_expert_parallel_comm:
+ probs, routing_map = self.mlp.route(hidden_states)
+ hidden_states, probs = self.mlp.preprocess(hidden_states, probs, routing_map)
+ nvtx_range_pop(suffix="mlp")
+ return residual, hidden_states, probs, shared_expert_output
+ mlp_output_with_bias = self.mlp(hidden_states)
+ self.mlp.cudagraph_tensor_store.clear()
+ nvtx_range_pop(suffix="mlp")
+
+ output = self._forward_post_mlp(mlp_output_with_bias, residual)
else:
- context = None
- if self.config.cuda_graph_scope == "attn":
- # CUDA Graph only covers the attention layer. Feed-forward
- # layer still goes through the normal pass.
+ # If EP overlap is enabled, needs to return same outputs as submodule.attn
+ if self.config.overlap_moe_expert_parallel_comm:
+ assert len(cuda_graph_output) == 1, "CUDA Graph output should be the layer output."
+ residual = cuda_graph_output.pop()
+ if not self.is_moe_layer:
+ return residual, None, None, None
+ hidden_states = self.pre_mlp_layernorm(residual)
+ shared_expert_output = self.mlp.shared_experts_compute(hidden_states)
+ probs, routing_map = self.mlp.route(hidden_states)
+ hidden_states, probs = self.mlp.preprocess(hidden_states, probs, routing_map)
+ return residual, hidden_states, probs, shared_expert_output
+
+ # CUDA Graph does not capture the MLP/MoE part at all.
output = self._forward_mlp(*cuda_graph_output)
- else:
- output = cuda_graph_output[0]
return output, context
def _get_te_cuda_graph_replay_args(self, *args, **kwargs):
@@ -827,7 +1066,7 @@ def _should_call_local_cudagraph(self, *args, **kwargs):
(kwargs.get('inference_context') is not None)
or (kwargs.get('inference_params') is not None)
)
- and self.config.cuda_graph_scope != 'full_iteration'
+ and CudaGraphScope.full_iteration not in self.config.cuda_graph_scope
):
if kwargs['inference_context'].is_static_batching():
using_cuda_graph = kwargs['inference_context'].is_decode_only()
@@ -840,6 +1079,15 @@ def _should_call_local_cudagraph(self, *args, **kwargs):
return True
return False
+ def backward_dw_cudagraph(self, microbatch_idx):
+ """
+ CUDA Graph backward weight gradient computation for this layer.
+ """
+ cg_index = microbatch_idx % len(self.cuda_graphs)
+ if not hasattr(self.cuda_graphs[cg_index], 'backward_dw'):
+ return
+ self.cuda_graphs[cg_index].backward_dw()
+
def __call__(self, *args, **kwargs):
if self._should_call_local_cudagraph(*args, **kwargs):
# Inference mode.
diff --git a/megatron/core/utils.py b/megatron/core/utils.py
index afee82f8245..62ce07586be 100644
--- a/megatron/core/utils.py
+++ b/megatron/core/utils.py
@@ -73,6 +73,15 @@
logger = logging.getLogger(__name__)
+try:
+ # Register the TE CUDA kernels
+ import transformer_engine # pylint: disable=unused-import
+
+ # Alias the PyTorch wrapper so we can call tex.* APIs
+ import transformer_engine_torch as tex
+except ImportError:
+ # TE isn’t installed or the torch wrapper is missing
+ tex = None
try:
_torch_version = PkgVersion(torch.__version__)
@@ -508,6 +517,10 @@ def get_tensor_model_parallel_group_if_none(tp_group, is_expert=False, check_ini
if not torch.distributed.is_initialized():
return None
+ # if parallel_state is not initialized, pass `tp_group` thru
+ if not parallel_state.is_initialized():
+ return tp_group
+
if tp_group is None:
if torch.distributed.is_initialized() and torch.distributed.get_rank() == 0:
warnings.warn(
@@ -975,6 +988,12 @@ def make_tp_sharded_tensor_for_checkpoint(
# Pop group parameters from kwargs
tp_group = kwargs.pop('tp_group', None)
dp_cp_group = kwargs.pop('dp_cp_group', None)
+ # If there are any additional kwargs left, surface them for visibility
+ # (these will be forwarded to ShardedTensor.from_rank_offsets).
+ if kwargs:
+ logger.warning(
+ "make_tp_sharded_tensor_for_checkpoint received extra kwargs: %s", list(kwargs.keys())
+ )
prepend_axis_num = len(prepend_offsets)
@@ -1040,6 +1059,12 @@ def make_sharded_tensor_for_checkpoint(tensor, key, prepend_offsets=(), replica_
# Pop group parameters from kwargs
tp_group = kwargs.pop('tp_group', None)
dp_cp_group = kwargs.pop('dp_cp_group', None)
+ # If there are any additional kwargs left, surface them for visibility
+ # (these will be forwarded to ShardedTensor.from_rank_offsets).
+ if kwargs:
+ logger.warning(
+ "make_sharded_tensor_for_checkpoint received extra kwargs: %s", list(kwargs.keys())
+ )
prepend_axis_num = len(prepend_offsets)
@@ -2022,9 +2047,17 @@ def is_submodule(module, parent_module, strict=True):
########################
-def get_batch_on_this_cp_rank(batch: Dict[str, Any]):
+def get_batch_on_this_cp_rank(
+ batch: Dict[str, Any], cp_group: Optional[torch.distributed.ProcessGroup] = None
+):
"""Slice batch input along sequence dimension into multiple chunks,
which are parallelized across GPUs in a context parallel group.
+
+ Args:
+ batch (Dict[str, Any]): Input batch tensors.
+ cp_group (Optional[torch.distributed.ProcessGroup]): Context-parallel process group.
+ If provided, uses this group's size and rank. Otherwise, falls back to
+ the current context-parallel settings from parallel_state.
"""
# With causal masking, each token only attends to its prior tokens. Simply split
@@ -2033,12 +2066,18 @@ def get_batch_on_this_cp_rank(batch: Dict[str, Any]):
# we split sequence into 2*CP ranks. Assuming CP=2, we then get 4 chunks, chunk_0
# and chunk_3 are assigned to GPU0, chunk_1 and chunk_2 are assigned to GPU1, so
# that we can get balanced workload among GPUs in a context parallel group.
- cp_size = parallel_state.get_context_parallel_world_size()
- if cp_size > 1:
+ # Determine CP topology either from provided group or from current context parallel state
+ if cp_group is not None:
+ cp_size = get_pg_size(cp_group)
+ cp_rank = get_pg_rank(cp_group)
+ else:
+ cp_size = parallel_state.get_context_parallel_world_size()
cp_rank = parallel_state.get_context_parallel_rank()
+
+ if cp_size > 1:
for key, val in batch.items():
if val is not None:
- seq_dim = 1 if key != "attention_mask" else 2
+ seq_dim = 1 if key != 'attention_mask' else 2
val = val.view(
*val.shape[0:seq_dim],
2 * cp_size,
@@ -2055,6 +2094,103 @@ def get_batch_on_this_cp_rank(batch: Dict[str, Any]):
return batch
+def get_thd_batch_on_this_cp_rank(
+ batch: Dict[str, Any],
+ cu_seqlens: torch.Tensor,
+ cu_seqlens_padded: torch.Tensor,
+ max_seqlen: torch.Tensor,
+ cp_group: Optional[torch.distributed.ProcessGroup] = None,
+):
+ """Slice each sub-sample in a packed sample batch input along
+ sequence dimension into multiple chunks, which are parallelized
+ across GPUs in a context parallel group.
+ """
+ packed_seq_params = PackedSeqParams(
+ qkv_format="thd",
+ cu_seqlens_q=cu_seqlens,
+ cu_seqlens_kv=cu_seqlens,
+ cu_seqlens_q_padded=cu_seqlens_padded,
+ cu_seqlens_kv_padded=cu_seqlens_padded,
+ max_seqlen_q=int(max_seqlen[0].item()),
+ max_seqlen_kv=int(max_seqlen[0].item()),
+ )
+
+ if cp_group is not None:
+ cp_size = get_pg_size(cp_group)
+ cp_rank = get_pg_rank(cp_group)
+ else:
+ cp_size = parallel_state.get_context_parallel_world_size()
+ cp_rank = parallel_state.get_context_parallel_rank()
+ if cp_size > 1: # slice batch along sequence dimension for context parallelism
+ assert tex is not None and is_te_min_version("1.10.0"), (
+ "Please update Transformer Engine to >= 1.10 to use "
+ "Context Parallel with THD format data"
+ )
+ index = tex.thd_get_partitioned_indices(
+ cu_seqlens_padded, batch['tokens'].size(1), cp_size, cp_rank
+ )
+ for key, data in batch.items():
+ if key in {'attention_mask', 'cu_seqlens', 'cu_seqlens_padded', 'max_seqlen'}:
+ continue
+ batch[key] = data.index_select(1, index)
+
+ return batch, packed_seq_params
+
+
+################################
+### hybrid context parallel ###
+################################
+
+
+def get_batch_on_this_hybrid_cp_rank(
+ batch: Dict[str, Any],
+ local_cp_size: int,
+ cp_group: Optional[torch.distributed.ProcessGroup] = None,
+):
+ """Slice batch input along sequence dimension into multiple chunks,
+ which are parallelized across GPUs in a context parallel group.
+ """
+ assert local_cp_size is not None
+ if cp_group is None:
+ # Get the local cp group required for as defined by the HybridCPDataLoaderWrapper
+ if local_cp_size > 1:
+ cp_group = parallel_state.get_hybrid_data_context_parallel_groups(
+ group_size=local_cp_size
+ )
+ else:
+ # If cp group is provided, it must match the local cp size
+ # as defined by the HybridCPDataLoaderWrapper
+ assert cp_group.size() == local_cp_size
+
+ # Convert [seqlen] to [1, seqlen] similar to default collate_fn
+ # as hybrid_context_parallel dataloader wrapper does not go through default collate_fn
+ for key, data in batch.items():
+ if key in ['attention_mask']:
+ continue
+ batch[key] = torch.stack([data], 0)
+ sample_length = batch['tokens'].shape[1]
+ # TODO(pmannan): Take care of padding tokens here if not divisible by cp_size*2
+ # Create packed_seq_params for SBHD format with cp group information.
+ packed_seq_params = PackedSeqParams(
+ qkv_format="sbhd",
+ cu_seqlens_q=torch.tensor([0, sample_length], device="cuda", pin_memory=True),
+ cu_seqlens_kv=torch.tensor([0, sample_length], device="cuda", pin_memory=True),
+ cu_seqlens_q_padded=torch.tensor([0, sample_length], device="cuda", pin_memory=True),
+ cu_seqlens_kv_padded=torch.tensor([0, sample_length], device="cuda", pin_memory=True),
+ max_seqlen_q=sample_length,
+ max_seqlen_kv=sample_length,
+ local_cp_size=local_cp_size,
+ cp_group=cp_group,
+ )
+
+ if cp_group is not None and cp_group.size() > 1:
+ # When using hybrid_context_parallel, each sub-sample of a packed sample is
+ # required to be divisible by CP*DP*2 or CP*DP*TP*2 (if using sequence parallel)
+ batch = get_batch_on_this_cp_rank(batch, cp_group)
+
+ return batch, packed_seq_params
+
+
######################
### NVTX profiling ###
######################
diff --git a/megatron/training/arguments.py b/megatron/training/arguments.py
index 70871565411..b94b5b45544 100644
--- a/megatron/training/arguments.py
+++ b/megatron/training/arguments.py
@@ -22,7 +22,7 @@
from megatron.core.rerun_state_machine import RerunStateMachine
from megatron.core.transformer import MLATransformerConfig, TransformerConfig
from megatron.core.transformer.pipeline_parallel_layer_layout import PipelineParallelLayerLayout
-from megatron.core.transformer.enums import AttnBackend
+from megatron.core.transformer.enums import AttnBackend, CudaGraphScope
from megatron.core.transformer.heterogeneous.heterogeneous_config import (
HeterogeneousTransformerConfig,
MLPConfig,
@@ -70,6 +70,7 @@ def add_megatron_arguments(parser: argparse.ArgumentParser):
parser = _add_vision_args(parser)
parser = _add_moe_args(parser)
parser = _add_mla_args(parser)
+ parser = _add_experimental_attention_variant_args(parser)
parser = _add_heterogeneous_args(parser)
parser = _add_logging_args(parser)
parser = _add_straggler_detector_args(parser)
@@ -320,7 +321,7 @@ def moe_freq_type(x):
This allows defining arbitrary patterns of expert and dense layers.
The pattern length must match the total number of transformer layers.
Examples:
- "([0]+[1]*23)": 1 dense layer followed by 23 experts layers
+ "([0]+[1]*23)": 1 dense layer followed by 23 expert layers
"([1]*3+[0]*2)*2": Three expert layers followed by two dense layers, repeated twice.
"""
if isinstance(x, int):
@@ -333,6 +334,31 @@ def moe_freq_type(x):
# it's a single int but in str
return int(x)
+def la_freq_type(x):
+ """Frequency between LA (linear attention) layers and SDPA (scaled dot-product attention) layers.
+
+ Accepts either:
+ - An integer N: Represents a (N-1):N ratio, meaning (N-1) LA layers for every 1 SDPA layer
+ - A string "N": Same as above, but provided as a string
+ - A string containing a Python list expression that defines a custom pattern, e.g.:
+ "([1]*3+[0]*1)*3" evaluates to [1,1,1,0,1,1,1,0,1,1,1,0]
+ where 1 indicates an LA layer and 0 indicates a SDPA layer.
+ This allows defining arbitrary patterns of LA and SDPA layers.
+ The pattern length must match the total number of transformer layers.
+ Examples:
+ "([0]+[1]*23)": 1 SDPA layer followed by 23 LA layers
+ "([1]*3+[0]*2)*2": Three LA layers followed by two SDPA layers, repeated twice.
+ """
+ if x is None or isinstance(x, int):
+ return x
+ assert isinstance(x, str)
+ if '[' in x:
+ # it's a custom pattern
+ return _eval_pattern(x)
+ else:
+ # it's a single int but in str
+ return int(x)
+
def tuple_type(x):
"""
Convert a string to a tuple of integers.
@@ -718,6 +744,13 @@ def validate_args(args, defaults={}):
assert args.ckpt_format == "fsdp_dtensor", \
"Megatron FSDP only supports fsdp_dtensor checkpoint format"
+
+ if args.use_megatron_fsdp:
+ args.reuse_grad_buf_for_mxfp8_param_ag = False
+
+ if args.fsdp_manual_registration:
+ assert args.use_megatron_fsdp, "FSDP manual registration is only supported with Megatron FSDP"
+ assert args.nccl_ub, "FSDP manual registration is only supported with nccl-ub option"
# Parameters dtype.
args.params_dtype = torch.float
@@ -747,7 +780,7 @@ def validate_args(args, defaults={}):
if args.rank == 0:
print('accumulate and all-reduce gradients in fp32 for '
'bfloat16 data type.', flush=True)
- if args.cuda_graph_impl == "local" and args.cuda_graph_scope=="full_iteration":
+ if args.cuda_graph_impl == "local" and CudaGraphScope.full_iteration in args.cuda_graph_scope:
if not args.inference_dynamic_batching:
assert not args.check_for_nan_in_loss_and_grad, \
"--no-check-for-nan-in-loss-and-grad should be set with full_iteration CUDA graph"
@@ -875,10 +908,20 @@ def validate_args(args, defaults={}):
'residual connection in fp32 only supported when using fp16 or bf16.'
if args.moe_grouped_gemm:
- assert args.bf16, 'Currently GroupedGEMM for MoE only supports bf16 dtype.'
dc = torch.cuda.get_device_capability()
assert dc[0] >= 8, "Unsupported compute capability for GroupedGEMM kernels."
+ if args.no_weight_decay_cond_type is not None:
+ print_rank_0(
+ 'WARNING: --no-weight-decay-cond-type is deprecated. Please use --apply-wd-to-qk-layernorm instead.',
+ args.rank,
+ )
+ if args.no_weight_decay_cond_type == "apply_wd_to_qk_layernorm":
+ args.apply_wd_to_qk_layernorm = True
+ else:
+ raise ValueError(f"Invalid no_weight_decay_cond_type: {args.no_weight_decay_cond_type}")
+ args.no_weight_decay_cond_type = None
+
if args.weight_decay_incr_style == 'constant':
assert args.start_weight_decay is None
assert args.end_weight_decay is None
@@ -931,6 +974,13 @@ def validate_args(args, defaults={}):
if args.tp_comm_overlap:
assert args.sequence_parallel == True, 'Tensor parallel communication/GEMM overlap can happen only when sequence parallelism is enabled'
+ if args.hybrid_context_parallel:
+ assert not args.pipeline_model_parallel_size > 1, 'Hybrid context parallelism not supported with pipeline parallelism'
+ assert not args.enable_cuda_graph, 'Hybrid context parallelism not supported with CUDA Graph'
+ assert not args.use_megatron_fsdp, 'Hybrid context parallelism not supported with Megatron FSDP'
+ assert args.dataloader_type == 'single', 'Hybrid context parallelism only supported with single dataloader type'
+ assert args.calculate_per_token_loss, 'Hybrid context parallelism must be used with --calculate-per-token-loss'
+
# disable async_tensor_model_parallel_allreduce when
# model parallel memory optimization is enabled
if (args.tensor_model_parallel_size > 1 or args.context_parallel_size > 1) \
@@ -1051,8 +1101,6 @@ def validate_args(args, defaults={}):
assert args.num_experts is not None, "num_experts must be non None to use expert model parallelism"
assert args.num_experts % args.expert_model_parallel_size == 0, \
"Number of experts should be a multiple of expert model parallel_size."
- assert not args.fp16, \
- "Expert parallelism is not supported with fp16 training."
# MoE router check
if isinstance(args.moe_router_load_balancing_type, list) and len(args.moe_router_load_balancing_type) == 1:
@@ -1205,6 +1253,24 @@ def validate_args(args, defaults={}):
if args.rank == 0:
print('Warning: enabling --no-load-optim when skipping training.')
+ # Experimental attention variant check
+ if args.linear_attention_type is not None:
+ print_rank_0(
+ '--linear-attention-type is deprecated, use --experimental-attention-variant instead.',
+ args.rank,
+ )
+ args.experimental_attention_variant = args.linear_attention_type
+ del args.linear_attention_type
+
+ # Muon optimizer check
+ if 'muon' in args.optimizer:
+ assert not args.use_distributed_optimizer, "Muon optimizer does not support distributed optimizer for now."
+ assert not args.use_torch_fsdp2, "Muon optimizer does not support Torch-FSDP2 for now."
+ assert not args.use_megatron_fsdp, "Muon optimizer does not support Megatron-FSDP for now."
+ assert args.ckpt_format in ["torch", "torch_dist"], "Muon optimizer supports torch and torch_dist checkpoint format."
+ assert args.experimental_attention_variant is None, "Muon optimizer does not support attention variant for now."
+ assert not args.attention_output_gate, "Muon optimizer does not support attention output gate for now."
+
# Optimizer CPU offload check
if args.optimizer_cpu_offload:
assert args.use_precision_aware_optimizer, (
@@ -1216,6 +1282,11 @@ def validate_args(args, defaults={}):
"must be used in conjunction with `--fp8-recipe delayed`."
)
+ if args.offload_optimizer_states:
+ assert args.use_distributed_optimizer, "offload_optimizer_states is only supported with distributed optimizer"
+ assert args.optimizer == 'adam', "offload_optimizer_states is only supported with adam optimizer"
+ assert not args.use_megatron_fsdp, "offload_optimizer_states does not support Megatron-FSDP for now."
+
if args.non_persistent_ckpt_type == "local":
assert args.non_persistent_local_ckpt_dir is not None, "Tried to use local checkpointing without specifying --local-ckpt-dir!"
if args.replication:
@@ -1238,6 +1309,13 @@ def validate_args(args, defaults={}):
"when enabling delay_wgrad_compute"
)
+ if args.fine_grained_activation_offloading:
+ assert args.transformer_impl == 'transformer_engine', \
+ "Fine-grained activation offloading is only supported with transformer_engine implementation"
+ if is_te_min_version("2.10.0"):
+ assert os.getenv("NVTE_CPU_OFFLOAD_V1", "0") == "1", \
+ "For fine-grained activation offloading with TE >= 2.10.0, NVTE_CPU_OFFLOAD_V1 should be set to 1 to avoid offloading weights."
+
if args.mtp_num_layers:
assert not args.use_legacy_models, "The legacy Megatron models does not support Multi-Token Prediction (MTP)."
assert args.position_embedding_type == "rope" or args.position_embedding_type == "none", (
@@ -1250,17 +1328,29 @@ def validate_args(args, defaults={}):
# CUDA Graphs
if args.cuda_graph_impl != "none":
- if args.transformer_impl == 'transformer_engine' and not args.te_rng_tracker:
+ if (
+ "transformer_engine" in (args.transformer_impl, args.cuda_graph_impl)
+ and not args.te_rng_tracker
+ ):
args.te_rng_tracker = True
warn_rank_0("te_rng_tracker is not enabled, enabling it for CUDA graphs.", args.rank)
- assert "expandable_segments:True" not in os.getenv("PYTORCH_CUDA_ALLOC_CONF", ""), (
- "expandable_segments:True may not be safe when using CUDA Graphs with some specific parallel settings. "
- "The training may crash with illegal memory access."
- )
assert (
- args.recompute_granularity != 'full'
- ), 'recompute_granularity must not be full when CUDA Graphs are enabled.'
-
+ "expandable_segments:True" not in os.getenv("PYTORCH_CUDA_ALLOC_CONF", "")
+ or os.getenv("NCCL_GRAPH_REGISTER", "") == "0"
+ ), (
+ "Setting NCCL_GRAPH_REGISTER=0 to avoid illegal memory access when using "
+ "CUDA Graph with PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True."
+ )
+ if args.cuda_graph_scope == "full" or (
+ isinstance(args.cuda_graph_scope, list) and "full" in args.cuda_graph_scope
+ ):
+ if isinstance(args.cuda_graph_scope, list):
+ assert args.cuda_graph_scope == ["full"], "full scope cannot be used with other scopes."
+ args.cuda_graph_scope = []
+ warn_rank_0(
+ 'full scope is deprecated. Use empty cuda_graph_scope to capture the whole layer.'
+ )
+
if args.multi_latent_attention:
assert not args.group_query_attention, "Group query attention is mutually exclusive with multi latent attention."
@@ -1429,6 +1519,9 @@ def _add_transformer_engine_args(parser):
group.add_argument('--transformer-impl', default='transformer_engine',
choices=['local', 'transformer_engine', 'inference_optimized'],
help='Which Transformer implementation to use.')
+ group.add_argument('--fallback-to-eager-attn', action='store_true',
+ help='Fallback to eager attention in TE implementation. '
+ 'Suggested for when desired features are not available in TE implementation.')
group.add_argument('--fp8-param-gather', action='store_true',
help='Keep the compute param in fp8 (do not use any other intermediate '
'dtype) and perform the param all-gather in fp8.')
@@ -1498,22 +1591,28 @@ def _add_inference_args(parser):
help="Number of CUDA graph warmup steps")
group.add_argument('--external-cuda-graph', action='store_true',
help='Deprecated. Use --cuda-graph-impl=transformer_engine instead. '
- 'Use TE make_graphed_callables() to capture the CUDA graph.')
+ 'Use TE make_graphed_callables() to capture the CUDA graph. '
+ 'Use --cuda-graph-scope=\"attn\", \"mlp\", \"moe\", \"moe_router\", \"moe_preprocess\", \"mamba\" for partial capture. ')
group.add_argument('--cuda-graph-impl', type=str, default='none',
choices=['none', 'local', 'transformer_engine'],
help='Determines the CUDA graph capture implementation. '
'"none": no CUDA graph. '
'"local": capture the CUDA graph using MCore local implementation. --cuda-graph-scope=\"full_iteration\" enables whole iteration CUDA graph. '
'"transformer_engine": capture the CUDA graph using TE make_graphed_callables().')
- group.add_argument('--cuda-graph-scope', type=str, default='full',
- choices=['full', 'attn', 'full_iteration'],
- help='Determines the CUDA graphs capturing scope. Valid values are '
- '\"full\", \"attn\" and \"full_iteration\". \"Full\" scope captures a whole '
- 'Transformer layer. \"Attn\" scope only captures operations in '
- 'TransformerLayer._forward_attention(). \"ful_iteration\" scope captures a '
- 'whole iteration. '
- 'full_iteration scope is only supported with --cuda-graph-impl=local, '
- 'attn scope is only supported with --cuda-graph-impl=transformer_engine.')
+ group.add_argument('--cuda-graph-scope', nargs='+', type=lambda scope: CudaGraphScope[scope] if scope != "full" else scope, default=[],
+ help='Determines the CUDA graphs capturing scope. '
+ 'choices: "attn", "mlp", "moe", "moe_router", "moe_preprocess", "mamba", "full_iteration". '
+ '"attn": captures operations in TransformerLayer._forward_attention(). '
+ '"mlp": captures operations in TransformerLayer._forward_mlp() for a dense layer. '
+ '"moe": captures operations in TransformerLayer._forward_mlp() for a MoE layer. '
+ '"moe_router": captures operations in TransformerLayer._forward_mlp() up to MoELayer.router(), '
+ 'including the shared experts if they are not overlapped with EP comm. '
+ '"moe_preprocess": captures operations in MoELayer.preprocess(). Must be used together with "moe_router". '
+ '"mamba": captures the mamba layer. '
+ '"full_iteration": captures a whole iteration. '
+ 'full_iteration scope is only supported with --cuda-graph-impl=local, other scopes are only supported with --cuda-graph-impl=transformer_engine. '
+ 'If not specified, the default scope is to capture the whole Transformer layer. '
+ 'For backward compatibility, we still allow passing "full" to specify capturing the whole layer, and convert it to an empty list.')
group.add_argument('--use-legacy-static-engine', action='store_true', default=False,
help='Use legacy static engine. (Current static engine uses dynamic engine under the hood)',
dest='use_legacy_static_engine')
@@ -1674,6 +1773,8 @@ def _add_network_size_args(parser):
group.add_argument('--group-query-attention', action='store_true',
help='Use group-query attention.')
group.add_argument('--num-query-groups', type=int, default=1)
+ group.add_argument('--attention-output-gate', action='store_true',
+ help='Whether to apply output gate to the attention.')
group.add_argument('--softmax-type', type=str, default='vanilla',
choices=['learnable', 'vanilla', 'off-by-one'],
help='Type of softmax to use for the attention. Supports both a fixed offset and '
@@ -1996,6 +2097,8 @@ def _add_regularization_args(parser):
group.add_argument('--weight-decay-incr-style', type=str, default='constant',
choices=['constant', 'linear', 'cosine'],
help='Weight decay increment function.')
+ group.add_argument('--apply-wd-to-qk-layernorm', action='store_true',
+ help='Apply weight decay to qk layernorm as a special case.')
group.add_argument('--clip-grad', type=float, default=1.0,
help='Gradient clipping based on global L2 norm.')
group.add_argument('--adam-beta1', type=float, default=0.9,
@@ -2009,6 +2112,33 @@ def _add_regularization_args(parser):
'numerical stability')
group.add_argument('--sgd-momentum', type=float, default=0.9,
help='Momentum factor for sgd')
+ group.add_argument('--muon-momentum', type=float, default=0.9,
+ help='Momentum factor for Muon optimizer')
+ group.add_argument('--muon-no-split-qkv', action='store_false', default=True,
+ dest='muon_split_qkv',
+ help='Whether to split QKV parameters for Muon optimizer')
+ group.add_argument('--muon-use-nesterov', action='store_true',
+ help='Whether to use Nesterov-style momentum in the internal SGD')
+ group.add_argument('--muon-scale-mode', type=str, default='spectral',
+ choices=['spectral', 'unit_rms_norm', 'shape_scaling'],
+ help='Scale mode for Muon optimizer')
+ group.add_argument('--muon-fp32-matmul-prec', type=str, default='medium',
+ choices=['low', 'medium', 'high'],
+ help='FP32 matmul precision for Newton-Schulz iteration')
+ group.add_argument('--muon-num-ns-steps', type=int, default=5,
+ help='Number of Newton-Schulz steps for Muon optimizer')
+ group.add_argument('--muon-tp-mode', type=str, default='blockwise',
+ choices=['blockwise', 'duplicated', 'distributed'],
+ help='How to perform NS calculation for tensor model parallel weights')
+ group.add_argument('--muon-extra-scale-factor', type=float, default=1.0,
+ help='Additional scale factor for the muon update')
+
+ group.add_argument('--no-weight-decay-cond-type', type=str, choices=['apply_wd_to_qk_layernorm'],
+ help='Type of no weight decay condition. Choices: '
+ 'None (default): apply weight decay to 1D weights and biases.'
+ '"apply_wd_to_qk_layernorm": additionally apply weight decay to '
+ 'qk layernorm as a special case.'
+ 'DEPRECATED. Please use --apply-wd-to-qk-layernorm instead. ')
return parser
@@ -2241,7 +2371,7 @@ def _add_training_args(parser):
help='Enabled fusion of cross entropy loss calculation.',
dest='cross_entropy_loss_fusion')
group.add_argument('--cross-entropy-fusion-impl', type=str, default='native',
- choices=['native', 'te'],
+ choices=['native', 'te', 'linear'],
help='Implementation of cross entropy loss calculation.')
group.add_argument('--use-flash-attn', action='store_true',
help='use FlashAttention implementation of attention. '
@@ -2259,7 +2389,7 @@ def _add_training_args(parser):
group.add_argument('--qk-clip-threshold', type=float, default=100,
help='The balancing threshold for qk-clip.')
group.add_argument('--optimizer', type=str, default='adam',
- choices=['adam', 'sgd'],
+ choices=['adam', 'sgd', 'muon', 'dist_muon'],
help='Optimizer function')
group.add_argument('--optimizer-cpu-offload', action='store_true',
help='Offload optimizer state to CPU')
@@ -2277,6 +2407,14 @@ def _add_training_args(parser):
help='Disable pinning of CPU memory for gradients.')
group.add_argument('--no-pin-cpu-params', action='store_false', dest='pin_cpu_params',
help='Disable pinning of CPU memory for parameters.')
+ group.add_argument('--offload-optimizer-states',
+ action='store_true',
+ dest='offload_optimizer_states',
+ help='Offload optimizer states to CPU after each optimizer step and '
+ 'reload them before the next optimizer step. '
+ 'Only support TE FusedAdam optimizer.'
+ 'Note that this still uses pure GPU optimizer instead of '
+ 'HybridDeviceOptimizer for --optimizer-cpu-offload.')
group.add_argument('--dataloader-type', type=str, default=None,
choices=['single', 'cyclic', 'external'],
help='Single pass vs multiple pass data loader')
@@ -2317,6 +2455,14 @@ def _add_training_args(parser):
help='The communicator group names to use high priority streams.')
group.add_argument('--use-te-activation-func', action='store_true',
help='Use activation function kernel from Transformer Engine in MLP module.')
+ group.add_argument('--fine-grained-activation-offloading', action='store_true',
+ help='Enable fine-grained activation offloading.')
+ group.add_argument('--offload-modules', nargs='*', type=str, default=[],
+ help='The submodules to offload its input. Choices: "attn_norm", "qkv_linear", "core_attn", "attn_proj", "mlp_norm", "expert_fc1", "moe_act".')
+ group.add_argument('--min-offloaded-tensor-size', type=int, default=1024*1024,
+ help='The minimum size of the tensor to be offloaded.')
+ group.add_argument('--disable-jit-fuser', action='store_true',
+ help='Disable the JIT fuser.')
group.add_argument('--batch-invariant-mode', action='store_true',
help='Use batch-invariant kernels for deterministic forward execution regardless '
'of batch size. Ensures bitwise identical results when the same inputs are '
@@ -2324,7 +2470,6 @@ def _add_training_args(parser):
'which only ensures bitwise identical results when the same inputs are processed in the same batch configuration. '
'This will significantly affect speed of training and inference as the kernels are not full optimized.')
-
return parser
@@ -2722,7 +2867,10 @@ def _add_distributed_args(parser):
group.add_argument('--disable-symmetric-registration', action='store_true', dest='disable_symmetric_registration',
default=False, help='Disable symmetric (window) registration for NCCL userbuffer registration.'
'This option will force to use conventional (local) userbuffer registration when use-nccl-ub is set.')
- group.add_argument('--use-sharp', action='store_true',
+ group.add_argument('--fsdp-manual-registration', action='store_true', dest='fsdp_manual_registration',
+ default=False, help='Manually register the FSDP communication buffers to NCCL user buffer.'
+ 'This option is only effective when use-megatron-fsdp and use-nccl-ub is set.')
+ group.add_argument('--use-sharp', action='store_true',
help='Required to enable SHARP communication.')
group.add_argument('--sharp-enabled-group', type=str, default=None,
choices=['dp', 'dp_replica'],
@@ -2773,6 +2921,13 @@ def _add_distributed_args(parser):
'--hierarchical-context-parallel-sizes 2 4 indicates every two adjacent gpus '
'forms the first level of cp groups and the cp ranks with the same odevity '
'forms the second level of cp groups.')
+ group.add_argument('--max-seqlen-per-cp-rank', type=int, default=None,
+ help='Maximum sequence length per CP rank. This is used to calculate the '
+ 'number of sub-samples assigned to each CP rank when using heterogeneous context parallel.')
+ group.add_argument('--hybrid-context-parallel', action='store_true', default=False,
+ help='Enables hybrid context parallel. This is used to balance the workload '
+ 'of each CP rank when we use packed samples with variable sequence lengths. '
+ 'Requires --max-seqlen-per-cp-rank to be set.')
group.add_argument('--nccl-communicator-config-path', type=str, default=None,
help='Path to the yaml file with NCCL communicator '
'configurations. The number of min/max thread groups and thread '
@@ -3138,7 +3293,7 @@ def _add_moe_args(parser):
'- A string containing a Python list expression that defines a custom pattern, e.g.: '
'"([1]*3+[0]*1)*3" evaluates to [1,1,1,0,1,1,1,0,1,1,1,0] '
'where 1 indicates an expert layer and 0 indicates a dense layer. '
- 'Examples: "([0]+[1]*23)": 1 dense layer followed by 23 experts layers, '
+ 'Examples: "([0]+[1]*23)": 1 dense layer followed by 23 expert layers, '
'"([1]*3+[0]*2)*2": Three expert layers followed by two dense layers, repeated twice.')
group.add_argument('--moe-ffn-hidden-size', type=int, default=None,
help='The hidden size of each expert\'s feed-forward network (ffn). '
@@ -3146,13 +3301,9 @@ def _add_moe_args(parser):
group.add_argument('--moe-shared-expert-intermediate-size', type=int, default=None,
help='Shared expert total ffn hidden size. '
'It should be equal to "num_shared_experts * ffn_size_of_each_shared_expert" if there are multiple shared experts. '
- 'None means no shared expert. '
- 'By default, the shared experts execute before the router. However, when '
- '--moe-shared-expert-overlap or --overlap-moe-expert-parallel-comm is set, '
- 'the shared experts execute after the router, before the routed experts. '
- 'This makes the gradients from the router and the shared experts added in '
- 'different orders to the hidden_states, causing minor numerical differences '
- 'in the hidden_states gradient.')
+ 'None means no shared expert.')
+ group.add_argument('--moe-shared-expert-gate', action='store_true',
+ help='Enable gate for shared expert. Only effective when moe-shared-expert-intermediate-size is set.')
group.add_argument('--moe-shared-expert-overlap', action='store_true',
help='Enable overlapping between shared expert computations and dispatcher communications. '
'Without this, the shared experts execute before the router. '
@@ -3189,6 +3340,9 @@ def _add_moe_args(parser):
help='Score function for MoE TopK routing. Can be "softmax" or "sigmoid".')
group.add_argument('--moe-router-topk', type=int, default=2,
help='Number of experts to route to for each token. The default is 2.')
+ group.add_argument('--enable-routing-replay', action='store_true',
+ help='Enable routing replay for MoE routers. When enabled, the router will '
+ 'use a pre-defined routing table instead of computing it on the fly.')
group.add_argument('--moe-router-pre-softmax', action='store_true',
help='Enable pre-softmax routing for MoE, which means softmax is before the top-k selection. By default, softmax is done after top-k.')
group.add_argument('--moe-router-num-groups', type=int, default=None,
@@ -3209,6 +3363,12 @@ def _add_moe_args(parser):
'The default value 1e-3 is same as that used in DeepSeekV3.')
group.add_argument('--moe-router-force-load-balancing', action='store_true',
help='[Experimental] Force override routing to balance token distribution using random logits for MoE routers, supporting naive top-k and group-limited top-k. This experimental feature is for benchmarking purposes only!')
+ group.add_argument('--moe-router-force-biased', type=float, default=None,
+ help='[Experimental] Apply random expert bias in normal distribution with specified std to router logits. '
+ 'Shared seed across all ranks ensures identical bias. '
+ 'If positive, generates new random bias each forward pass. '
+ 'If negative, generates bias once per layer and reuses it (abs value is std). '
+ 'This experimental feature is for benchmarking purposes only!')
group.add_argument('--moe-router-padding-for-quantization', action='store_true',
help='Pad the routing_map to make sure the number of tokens each expert received '
'is a multiple of 16/32 for FP8/FP4 precision. It is suggested to enable this for '
@@ -3257,6 +3417,8 @@ def _add_moe_args(parser):
help='Overlap the EP A2A communication by batch-level overlapping in 1f1b stage.')
group.add_argument('--delay-wgrad-compute', action='store_true',
help='Delay the wgrad compute for batch-level overlapping')
+ group.add_argument('--ep-overlap-early-attn-memory-release', action='store_true',
+ help='Release the memory of the attention module early in EP overlap.')
group.add_argument('--moe-upcycling-granularity', type=int, default=1,
help='This param sepecifics how many times smaller is the expert hidden size compared with the original dense FFN hidden size. '
@@ -3290,6 +3452,45 @@ def _add_mla_args(parser):
return parser
+def _add_experimental_attention_variant_args(parser):
+ group = parser.add_argument_group(title="experimental_attention_variant")
+ group.add_argument('--experimental-attention-variant', default=None, choices=['gated_delta_net', 'dsa'], type=str,
+ help='Type of attention variant to use. Currently support gated_delta_net and dsa.')
+ # DSA
+ group.add_argument('--dsa-indexer-n-heads', default=None, type=int,
+ help='Number of indexer heads for sparse attention. If not set, defaults to num-attention-heads.')
+ group.add_argument('--dsa-indexer-head-dim', default=None, type=int,
+ help='Dimension per indexer head for sparse attention. If not set, defaults to kv-channels.')
+ group.add_argument('--dsa-indexer-topk', default=None, type=int,
+ help='Number of top-k tokens to select in sparse attention indexer.')
+ group.add_argument('--dsa-indexer-loss-coeff', default=0.0, type=float,
+ help='Coefficient for the indexer KL divergence loss. Set to 0 to disable indexer loss.')
+ group.add_argument('--dsa-indexer-use-sparse-loss', action='store_true',
+ help='Use sparse indexer loss. If set, the indexer loss will be computed using the top-k indices.')
+ # Linear attention
+ group.add_argument('--linear-attention-type', default=None, choices=['gated_delta_net'], type=str,
+ help='(Deprecated, use --experimental-attention-variant instead) Type of linear attention to use. Currently support gated_delta_net.')
+ group.add_argument('--linear-attention-freq', type=la_freq_type, default=None,
+ help='Frequency between LA (linear attention) layers and'
+ ' SDPA (scaled dot-product attention) layers. Accepts either: '
+ '- An integer N: Represents a (N-1):N ratio, meaning (N-1) LA layers for every 1 SDPA layer '
+ '- A string containing a Python list expression that defines a custom pattern, e.g.: '
+ '"([1]*3+[0]*1)*3" evaluates to [1,1,1,0,1,1,1,0,1,1,1,0] '
+ 'where 1 indicates an LA layer and 0 indicates a SDPA layer. '
+ 'Examples: "([0]+[1]*23)": 1 SDPA layer followed by 23 LA layers, '
+ '"([1]*3+[0]*2)*2": Three LA layers followed by two SDPA layers, repeated twice.')
+ group.add_argument('--linear-conv-kernel-dim', default=4, type=int,
+ help='Conv kernel dimension for the gated delta net.')
+ group.add_argument('--linear-key-head-dim', default=128, type=int,
+ help='Query and key head dimension for the gated delta net.')
+ group.add_argument('--linear-value-head-dim', default=128, type=int,
+ help='Value and gate head dimension for the gated delta net.')
+ group.add_argument('--linear-num-key-heads', default=16, type=int,
+ help='Number of query and key heads for the gated delta net.')
+ group.add_argument('--linear-num-value-heads', default=32, type=int,
+ help='Number of value and gate heads for the gated delta net.')
+ return parser
+
def _add_heterogeneous_args(parser):
"""
Heterogeneous models refer to transformer architectures where individual layers can differ
diff --git a/megatron/training/checkpointing.py b/megatron/training/checkpointing.py
index 325158780a4..f7ff7cd2775 100644
--- a/megatron/training/checkpointing.py
+++ b/megatron/training/checkpointing.py
@@ -505,6 +505,14 @@ def save_checkpoint(iteration, model, optimizer, opt_param_scheduler, num_floati
ensure_directory_exists(optim_checkpoint_name)
if not optimizer.is_stub_optimizer:
optimizer.save_parameter_state(optim_checkpoint_name)
+
+ # LayerWiseDistributedOptimizer save optimizer state to file on different ranks
+ if getattr(args, "optimizer", "adam").startswith("dist_") and args.ckpt_format == 'torch':
+ dp_rank = mpu.get_data_parallel_rank()
+ optim_checkpoint_name = os.path.join(os.path.dirname(checkpoint_name), f"layer_wise_optimizer_{dp_rank}.pt")
+ ensure_directory_exists(optim_checkpoint_name)
+ if not optimizer.is_stub_optimizer:
+ optimizer.save_state_dict_to_file(optim_checkpoint_name)
async_save_request = None
if args.async_save:
@@ -1464,13 +1472,13 @@ def load_checkpoint(ddp_model, optimizer, opt_param_scheduler, load_arg='load',
ckpt_args = state_dict.get("args")
if not hasattr(ckpt_args, "tensor_model_parallel_size"):
- print_rank_0("WARNING: TP size not found in checkpoint args, using 0 as default.")
+ print_rank_0("WARNING: TP size not found in checkpoint args, using 1 as default.")
if not hasattr(ckpt_args, "pipeline_model_parallel_size"):
- print_rank_0("WARNING: PP size not found in checkpoint args, using 0 as default.")
+ print_rank_0("WARNING: PP size not found in checkpoint args, using 1 as default.")
ckpt_tp_pp = (
- getattr(ckpt_args, "tensor_model_parallel_size", 0),
- getattr(ckpt_args, "pipeline_model_parallel_size", 0),
+ getattr(ckpt_args, "tensor_model_parallel_size", 1),
+ getattr(ckpt_args, "pipeline_model_parallel_size", 1),
)
run_tp_pp = (
args.tensor_model_parallel_size,
@@ -1715,7 +1723,12 @@ def load_model_state_dict(module, state_dict, strict: bool):
if not release and not args.finetune and not args.no_load_optim:
try:
# Load state dict.
- if not skip_load_to_model_and_opt and optimizer is not None and not optimizer.is_stub_optimizer:
+ if getattr(args, "optimizer", "adam").startswith("dist_") and args.ckpt_format == 'torch':
+ # LayerWiseDistributedOptimizer load optimizer state from file on different ranks
+ dp_rank = mpu.get_data_parallel_rank()
+ optim_checkpoint_name = os.path.join(os.path.dirname(checkpoint_name), f"layer_wise_optimizer_{dp_rank}.pt")
+ optimizer.load_state_dict_from_file(optim_checkpoint_name)
+ elif not skip_load_to_model_and_opt and optimizer is not None and not optimizer.is_stub_optimizer:
optimizer.load_state_dict(state_dict['optimizer'])
# Load distributed optimizer's custom parameter state.
@@ -1765,6 +1778,8 @@ def load_model_state_dict(module, state_dict, strict: bool):
# rng states.
if not release and not args.finetune and not args.no_load_rng and not ignore_rng_state:
try:
+ cuda_rng_tracker = tensor_parallel.get_cuda_rng_tracker()
+ graph_safe_rng = tensor_parallel.is_graph_safe_cuda_rng_tracker(cuda_rng_tracker)
if 'rng_state' in state_dict:
if args.ckpt_format == "fsdp_dtensor":
# FSDP DTensor checkpoints store rng_state in a different format.
@@ -1790,8 +1805,10 @@ def load_model_state_dict(module, state_dict, strict: bool):
# Check for empty states array
if not rng_state['rng_tracker_states']:
raise KeyError
- tensor_parallel.get_cuda_rng_tracker().set_states(
- rng_state['rng_tracker_states'])
+ rng_tracker_states = {
+ k: tensor_parallel.convert_cuda_rng_state(v, to_graphable=graph_safe_rng)
+ for k, v in rng_state['rng_tracker_states'].items()
+ }
else: # backward compatability
random.setstate(state_dict['random_rng_state'])
np.random.set_state(state_dict['np_rng_state'])
@@ -1800,8 +1817,11 @@ def load_model_state_dict(module, state_dict, strict: bool):
# Check for empty states array
if not state_dict['rng_tracker_states']:
raise KeyError
- tensor_parallel.get_cuda_rng_tracker().set_states(
- state_dict['rng_tracker_states'])
+ rng_tracker_states = {
+ k: tensor_parallel.convert_cuda_rng_state(v, to_graphable=graph_safe_rng)
+ for k, v in state_dict['rng_tracker_states'].items()
+ }
+ cuda_rng_tracker.set_states(rng_tracker_states)
except KeyError:
print_rank_0('Unable to load rng state from checkpoint {}. '
'Specify --no-load-rng or --finetune to prevent '
diff --git a/megatron/training/datasets/data_samplers.py b/megatron/training/datasets/data_samplers.py
index 1e7f47510d1..d33250520dd 100644
--- a/megatron/training/datasets/data_samplers.py
+++ b/megatron/training/datasets/data_samplers.py
@@ -39,14 +39,22 @@ def build_pretraining_data_loader(dataset, consumed_samples):
data_parallel_size=mpu.get_data_parallel_world_size(),
)
elif args.dataloader_type == 'single':
- # Megatron sampler
- batch_sampler = MegatronPretrainingSampler(
- total_samples=len(dataset),
- consumed_samples=consumed_samples,
- micro_batch_size=args.micro_batch_size,
- data_parallel_rank=mpu.get_data_parallel_rank(),
- data_parallel_size=mpu.get_data_parallel_world_size(),
- )
+ if args.hybrid_context_parallel:
+ batch_sampler = HybridCPMegatronPretrainingSampler(
+ total_samples=len(dataset),
+ consumed_samples=consumed_samples,
+ micro_batch_size=args.micro_batch_size,
+ global_batch_size=args.global_batch_size,
+ data_parallel_rank=mpu.get_data_parallel_rank(),
+ data_parallel_size=mpu.get_data_parallel_world_size())
+ else:
+ # Megatron sampler
+ batch_sampler = MegatronPretrainingSampler(
+ total_samples=len(dataset),
+ consumed_samples=consumed_samples,
+ micro_batch_size=args.micro_batch_size,
+ data_parallel_rank=mpu.get_data_parallel_rank(),
+ data_parallel_size=mpu.get_data_parallel_world_size())
elif args.dataloader_type == 'cyclic':
batch_sampler = MegatronPretrainingRandomSampler(
dataset,
@@ -71,15 +79,18 @@ def worker_init_fn(_):
worker_init_fn if args.exit_signal_handler and args.num_workers > 0 else None
)
# Torch dataloader.
- return torch.utils.data.DataLoader(
- dataset,
- batch_sampler=batch_sampler,
- num_workers=args.num_workers,
- pin_memory=True,
- persistent_workers=True if args.num_workers > 0 else False,
- worker_init_fn=maybe_worker_init_fn,
- )
-
+ if args.hybrid_context_parallel:
+ extra_kwargs = {"collate_fn": lambda x: x,}
+ else:
+ extra_kwargs = {}
+ return torch.utils.data.DataLoader(dataset,
+ batch_sampler=batch_sampler,
+ num_workers=args.num_workers,
+ pin_memory=True,
+ persistent_workers=True if args.num_workers > 0 else False,
+ worker_init_fn=maybe_worker_init_fn,
+ **extra_kwargs,
+ )
class MegatronPretrainingSampler:
"""
@@ -150,6 +161,49 @@ def __iter__(self):
start_idx, end_idx = self.get_start_end_idx()
yield batch[start_idx:end_idx]
+class HybridCPMegatronPretrainingSampler(MegatronPretrainingSampler):
+ """
+ Data sampler for hybrid context parallel (Hybrid CP) format.
+ This data sampler pulls in the entire global batch at once across all data parallel ranks.
+ This helps provide the Hybrid CP Dataloader Wrapper to schedule and load balance sub-samples
+ of the entire global batch.
+ """
+
+ def __init__(self, total_samples, consumed_samples, micro_batch_size, global_batch_size,
+ data_parallel_rank, data_parallel_size, drop_last=True):
+ super().__init__(total_samples, consumed_samples, micro_batch_size, data_parallel_rank, data_parallel_size, drop_last)
+ self.global_batch_size = global_batch_size
+ self.data_parallel_size = data_parallel_size
+ self.num_micro_batches = self.global_batch_size // self.micro_batch_times_data_parallel_size
+
+ def __len__(self):
+ return self.total_samples
+
+ def get_start_end_idx_global_batch(self):
+ start_idx = [self.data_parallel_rank * self.micro_batch_size + i * self.micro_batch_size * self.data_parallel_size for i in range(self.num_micro_batches)]
+ end_idx = [start_idx[i] + self.micro_batch_size for i in range(self.num_micro_batches)]
+ return start_idx, end_idx
+
+ def __iter__(self):
+ batch = []
+ # Last batch will be dropped if drop_last is not set False
+ for idx in range(self.consumed_samples, self.total_samples):
+ batch.append(idx)
+ if len(batch) == self.micro_batch_times_data_parallel_size * self.num_micro_batches:
+ start_idx, end_idx = self.get_start_end_idx_global_batch()
+ global_batch_idx = []
+ for i in range(self.num_micro_batches):
+ global_batch_idx.extend(batch[start_idx[i]:end_idx[i]])
+ yield global_batch_idx
+ batch = []
+
+ # Check the last partial batch and see drop_last is set
+ if len(batch) > 0 and not self.drop_last:
+ start_idx, end_idx = self.get_start_end_idx_global_batch()
+ global_batch_idx = []
+ for i in range(self.num_micro_batches):
+ global_batch_idx.extend(batch[start_idx[i]:end_idx[i]])
+ yield global_batch_idx
class RandomSeedDataset(Dataset):
"""
diff --git a/megatron/training/global_vars.py b/megatron/training/global_vars.py
index 62dbf701c1f..a718877b40c 100644
--- a/megatron/training/global_vars.py
+++ b/megatron/training/global_vars.py
@@ -9,6 +9,7 @@
from megatron.core import Timers
from megatron.core.config import set_experimental_flag
from megatron.core.energy_monitor import EnergyMonitor
+from megatron.core.jit import disable_jit_fuser
from megatron.core.num_microbatches_calculator import init_num_microbatches_calculator, unset_num_microbatches_calculator
from megatron.training.dist_signal_handler import DistributedSignalHandler
from megatron.training.tokenizer import build_tokenizer
@@ -112,6 +113,9 @@ def set_global_variables(args, build_tokenizer=True):
if args.exit_signal_handler:
_set_signal_handler(args.exit_signal)
+ if args.disable_jit_fuser:
+ disable_jit_fuser()
+
def unset_global_variables():
"""Unset global vars.
diff --git a/megatron/training/initialize.py b/megatron/training/initialize.py
index e88222fe7fe..1a119b127e4 100644
--- a/megatron/training/initialize.py
+++ b/megatron/training/initialize.py
@@ -371,6 +371,7 @@ def _initialize_distributed(get_embedding_ranks, get_position_embedding_ranks, s
use_sharp=args.use_sharp,
context_parallel_size=args.context_parallel_size,
hierarchical_context_parallel_sizes=args.hierarchical_context_parallel_sizes,
+ hybrid_context_parallel=args.hybrid_context_parallel,
expert_model_parallel_size=args.expert_model_parallel_size,
num_distributed_optimizer_instances=args.num_distributed_optimizer_instances,
expert_tensor_parallel_size=args.expert_tensor_parallel_size,
diff --git a/megatron/training/tokenizer/tokenizer.py b/megatron/training/tokenizer/tokenizer.py
index 08d8aacd9ea..5371c0d318e 100644
--- a/megatron/training/tokenizer/tokenizer.py
+++ b/megatron/training/tokenizer/tokenizer.py
@@ -48,7 +48,7 @@ def build_tokenizer(args, **kwargs):
tokenizer = _GPTSentencePieceTokenizer(args.tokenizer_model)
elif args.tokenizer_type == 'HuggingFaceTokenizer':
tokenizer = _HuggingFaceTokenizer(
- args.tokenizer_model, trust_remote_code = args.trust_remote_code, **kwargs,
+ args.tokenizer_model, trust_remote_code=args.trust_remote_code, **kwargs
)
elif args.tokenizer_type == 'Llama2Tokenizer':
assert args.tokenizer_model is not None
@@ -78,11 +78,7 @@ def build_tokenizer(args, **kwargs):
kwargs = dict()
if args.tokenizer_prompt_format == "nvlm-yi-34b":
- kwargs = {
- "from_slow": True,
- "legacy": False,
- "add_bos_token": True,
- }
+ kwargs = {"from_slow": True, "legacy": False, "add_bos_token": True}
# Currently, only HuggingFace tokenizers are supported.
underlying_tokenizer = transformers.AutoTokenizer.from_pretrained(
@@ -97,10 +93,7 @@ def build_tokenizer(args, **kwargs):
args.force_system_message,
)
elif args.tokenizer_type == "SFTTokenizer":
- tokenizer = SFTTokenizer(
- args.tokenizer_model,
- args.sft_tokenizer_prompt_format,
- )
+ tokenizer = SFTTokenizer(args.tokenizer_model, args.sft_tokenizer_prompt_format)
elif args.tokenizer_type == 'NullMultimodalTokenizer':
assert args.vocab_size is not None
tokenizer = _NullMultimodalTokenizer(args.vocab_size)
@@ -144,7 +137,7 @@ def __init__(self, pretrained_model_name_or_path, trust_remote_code=False, **kwa
self._tokenizer = transformers.AutoTokenizer.from_pretrained(
pretrained_model_name_or_path=pretrained_model_name_or_path,
trust_remote_code=trust_remote_code,
- **kwargs
+ **kwargs,
)
self._vocab = self._tokenizer.get_vocab()
self._inv_vocab = {token_id: token for token, token_id in self._vocab.items()}
@@ -367,6 +360,10 @@ def detokenize(self, token_ids):
def eod(self):
return self.eod_id
+ @property
+ def eos(self):
+ return self.eod_id
+
class _SentencePieceTokenizer(MegatronLegacyTokenizer):
"""SentencePieceTokenizer-Megatron wrapper"""
@@ -573,6 +570,10 @@ def mask(self):
def eod(self):
return self._eos_id
+ @property
+ def eos(self):
+ return self._eos_id
+
@property
def additional_special_tokens_ids(self):
return None
@@ -623,6 +624,10 @@ def mask(self):
def eod(self):
return self.eos_id
+ @property
+ def eos(self):
+ return self.eos_id
+
@property
def additional_special_tokens_ids(self):
return None
@@ -747,7 +752,7 @@ def bos(self) -> int:
@property
def eos(self) -> int:
return self._eos_id
-
+
@property
def pad(self) -> int:
return self._pad_id
@@ -858,19 +863,30 @@ def mask(self):
def eod(self):
return self._eod_id
+ @property
+ def eos(self):
+ return self._eod_id
+
@property
def additional_special_tokens_ids(self):
return None
+
class _NullMultimodalTokenizer(MegatronLegacyTokenizer):
def __init__(self, vocab_size, image_token=None, image_token_id=None):
super().__init__(None, vocab_size=vocab_size)
self._vocab_size_without_eod = int(vocab_size)
self._eod_id = self._vocab_size_without_eod
- from megatron.core.models.multimodal.llava_model import DEFAULT_IMAGE_TOKEN_INDEX, IMAGE_TOKEN
+ from megatron.core.models.multimodal.llava_model import (
+ DEFAULT_IMAGE_TOKEN_INDEX,
+ IMAGE_TOKEN,
+ )
+
self._image_token = image_token if image_token is not None else IMAGE_TOKEN
- self._image_token_id = image_token_id if image_token_id is not None else DEFAULT_IMAGE_TOKEN_INDEX
+ self._image_token_id = (
+ image_token_id if image_token_id is not None else DEFAULT_IMAGE_TOKEN_INDEX
+ )
def tokenize(self, text):
return [int(x) for x in text.split(' ')]
@@ -887,7 +903,9 @@ def offsets(self, ids: list[int], text: str) -> list[int]:
return offsets
def convert_tokens_to_ids(self, tokens):
- ids = [(int(t) if t != self._image_token else self._image_token_id) for t in tokens.split(' ')]
+ ids = [
+ (int(t) if t != self._image_token else self._image_token_id) for t in tokens.split(' ')
+ ]
return ids if len(ids) > 1 else ids[0]
@property
@@ -918,6 +936,10 @@ def mask(self):
def eod(self):
return self._eod_id
+ @property
+ def eos(self):
+ return self._eod_id
+
@property
def additional_special_tokens_ids(self):
return None
diff --git a/megatron/training/training.py b/megatron/training/training.py
index 3f23ca59f4a..13ad0025e43 100644
--- a/megatron/training/training.py
+++ b/megatron/training/training.py
@@ -1,4 +1,4 @@
-# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved.
+# Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
"""Pretrain utilities."""
@@ -12,7 +12,7 @@
import math
import os
import sys
-from typing import Any, Optional
+from typing import Any, Optional, Dict
import torch.distributed
@@ -49,8 +49,12 @@
from megatron.core import mpu, tensor_parallel
+from megatron.core.models.gpt.experimental_attention_variant_module_specs import (
+ is_linear_attention_variant,
+)
from megatron.core.utils import (
check_param_hashes_across_dp_replicas,
+ get_attr_wrapped_model,
get_model_config,
get_pg_size,
get_pg_rank,
@@ -64,11 +68,13 @@
is_vp_first_stage,
is_vp_last_stage,
)
+from megatron.core.optimizer import get_standard_config_overrides
from megatron.training.checkpointing import load_checkpoint
from megatron.training.checkpointing import save_checkpoint
from megatron.training.checkpointing import checkpoint_exists
from megatron.core.full_cuda_graph import FullCudaGraphWrapper
from megatron.core.transformer.cuda_graphs import TECudaGraphHelper
+from megatron.core.transformer.enums import CudaGraphScope
from megatron.core.transformer.module import Float16Module
from megatron.core.distributed import DistributedDataParallelConfig, TorchFullyShardedDataParallelConfig
from megatron.core.distributed import DistributedDataParallel as DDP
@@ -88,6 +94,7 @@
from megatron.core.distributed import finalize_model_grads
from megatron.core.enums import ModelType
from megatron.core.optimizer import get_megatron_optimizer, AdamOptimizerConfig, SGDOptimizerConfig, OptimizerConfig, ParamKey
+from megatron.core.optimizer.muon import get_megatron_muon_optimizer
from megatron.core.rerun_state_machine import (
get_rerun_state_machine,
destroy_rerun_state_machine,
@@ -99,9 +106,11 @@
from megatron.training.initialize import set_jit_fusion_options
from megatron.training.utils import get_batch_on_this_cp_rank, get_batch_on_this_tp_rank
from megatron.training.datasets.data_samplers import build_pretraining_data_loader
+from megatron.core.datasets.data_schedule import HybridCPDataLoaderWrapper
from megatron.core.optimizer_param_scheduler import OptimizerParamScheduler
from megatron.core.transformer.moe import upcycling_utils
from megatron.core.transformer.moe.moe_utils import track_moe_metrics
+from megatron.core.transformer.experimental_attention_variant.dsa import DSAIndexerLossLoggingHelper
from megatron.core.transformer.multi_token_prediction import MTPLossLoggingHelper
from megatron.core.parallel_state import (
destroy_global_memory_buffer,
@@ -274,9 +283,6 @@ def hybrid_flops(batch_size, seq_len, hidden_size,
def transformer_flops():
"""Calculate FLOPs for a standard Transformer model."""
# TODO(helenn/dnarayanan): Refactor this to reuse the helper methods.
- # Attention projection size.
- query_projection_size = args.kv_channels * args.num_attention_heads
- query_projection_to_hidden_size_ratio = query_projection_size / args.hidden_size
# Group Query Attention.
if not args.group_query_attention:
args.num_query_groups = args.num_attention_heads
@@ -326,18 +332,15 @@ def transformer_flops():
if args.moe_shared_expert_intermediate_size is None
else args.moe_shared_expert_intermediate_size
)
- # SwiGLU.
- gated_linear_multiplier = 3 / 2 if args.swiglu else 1
- # The 12x term below comes from the following factors; for more details, see
- # "APPENDIX: FLOATING-POINT OPERATIONS" in https://arxiv.org/abs/2104.04473.
# - 3x: Each GEMM in the model needs to be performed 3 times (forward pass,
# backward wgrad [weight gradient], backward dgrad [data gradient]).
- # - 2x: GEMMs of a particular size are stacked twice in the standard Transformer model
- # architectures implemented in this codebase (e.g., h->ffn_h GEMM and ffn_h->h GEMM
- # in MLP layer).
+ forward_backward_expansion_factor = 3
# - 2x: A GEMM of a m*n tensor with a n*k tensor requires 2mnk floating-point operations.
- expansion_factor = 3 * 2 * 2
+ fma_expansion_factor = 2
+ # - 3x (SwiGLU enabled): h->2*ffn_h GEMM and ffn_h->h GEMM are stacked.
+ # - 2x (SwiGLU disabled): h->ffn_h GEMM and ffn_h->h GEMM are stacked.
+ ffn_expansion_factor = 3 if args.swiglu else 2
if args.multi_latent_attention:
assert not args.group_query_attention
@@ -367,10 +370,9 @@ def transformer_flops():
+ args.num_attention_heads * (args.qk_head_dim + args.qk_pos_emb_head_dim)
+ 1
)
- self_attn_term = (
- 3
- * 2 # fwd(1) + bwd(2) *FMA
- * num_layers
+ standard_self_attn_term = (
+ forward_backward_expansion_factor
+ * fma_expansion_factor
* (
## q lora + rope + q norm
q_term
@@ -387,53 +389,136 @@ def transformer_flops():
## core attn
+ args.seq_length
* (args.num_attention_heads * (args.qk_head_dim + args.qk_pos_emb_head_dim))
- / 2
+ / 2 # causal mask (only half of the mask is non-zero)
+ args.seq_length * args.num_attention_heads * args.v_head_dim / 2
)
)
else:
## MHA or GQA
- self_attn_term = (
- expansion_factor
- * num_layers
- * args.hidden_size
- * args.hidden_size
+ query_projection_size = args.kv_channels * args.num_attention_heads
+ key_projection_size = args.kv_channels * args.num_query_groups
+ value_projection_size = args.kv_channels * args.num_query_groups
+ gate_projection_size = query_projection_size if args.attention_output_gate else 0
+ standard_self_attn_term = (
+ forward_backward_expansion_factor
+ * fma_expansion_factor
* (
- (
- 1
- + (args.num_query_groups / args.num_attention_heads)
- # # Only half of the attention matrix is non-zero and needs to be multiplied with V.
- + (args.seq_length / args.hidden_size / 2)
+ ## qkv proj
+ args.hidden_size
+ * (
+ query_projection_size
+ + key_projection_size
+ + value_projection_size
+ + gate_projection_size
)
- * query_projection_to_hidden_size_ratio
+ ## core attention
+ + query_projection_size
+ * args.seq_length
+ / 2 # causal mask (only half of the mask is non-zero)
+ * 2 # QK^T and (QK^T)V
+ ## out proj
+ + query_projection_size
+ * args.hidden_size
)
)
+ if is_linear_attention_variant(args.experimental_attention_variant):
+ # Calculate number of dense and MoE Transformer MLPs.
+ if isinstance(args.linear_attention_freq, int):
+ linear_attention_pattern = [
+ # [1,1,...,1,0,1,1,...,1,0,...]
+ 0 if ((i + 1) % args.linear_attention_freq == 0)
+ else 1 for i in range(num_layers)
+ ]
+ elif isinstance(args.linear_attention_freq, list):
+ linear_attention_pattern = args.linear_attention_freq
+ assert len(linear_attention_pattern) == num_layers, (
+ f"Invalid length of linear_attention_pattern: {len(linear_attention_pattern)}, "
+ f"expected {num_layers}, "
+ f"current linear attention pattern: {args.linear_attention_freq}"
+ )
+ elif args.linear_attention_freq is None:
+ # This should be caught by config validation, but raise here as a safety check
+ raise ValueError(
+ f"Linear attention type {args.experimental_attention_variant} is specified "
+ "but linear_attention_freq is None. "
+ "Please set linear_attention_freq to specify the LA/SDPA layer pattern."
+ )
+ else:
+ raise ValueError(
+ f"Invalid linear_attention_freq: {type(args.linear_attention_freq)},"
+ f" {args.linear_attention_freq}"
+ )
+ num_linear_attention_layers = sum(linear_attention_pattern)
+ num_standard_attention_layers = num_layers - num_linear_attention_layers
+
+ if args.experimental_attention_variant == "gated_delta_net":
+ # Calculate the FLOPs for the gated delta net attention.
+ qk_head_dim = args.linear_key_head_dim
+ v_head_dim = args.linear_value_head_dim
+ num_qk_heads = args.linear_num_key_heads
+ num_v_heads = args.linear_num_value_heads
+ qk_dim = qk_head_dim * num_qk_heads
+ v_dim = v_head_dim * num_v_heads
+ linear_self_attn_term = (
+ forward_backward_expansion_factor
+ * fma_expansion_factor
+ * (
+ ## in proj
+ args.hidden_size
+ * (2 * qk_dim + 2 * v_dim + 2 * num_v_heads)
+ ## conv1d
+ + args.linear_conv_kernel_dim
+ * (2 * qk_dim + v_dim)
+ ## gated delta rule
+ + num_v_heads
+ * (v_head_dim ** 2)
+ * 4 # KK^T, VK^T, S(a(I-bKK^T)), and SQ
+ ## out proj
+ + args.hidden_size
+ * v_dim
+ )
+ )
+ else:
+ raise ValueError(
+ "Invalid experimental_attention_variant: "
+ f"{args.experimental_attention_variant}"
+ )
+ else:
+ num_linear_attention_layers = 0
+ linear_self_attn_term = 0
+ num_standard_attention_layers = num_layers
+
+ self_attn_term = (
+ linear_self_attn_term * num_linear_attention_layers
+ + standard_self_attn_term * num_standard_attention_layers
+ )
+
total_floating_point_operations = (
batch_size
* args.seq_length
* (
# MLP
- expansion_factor
- * num_layers
+ forward_backward_expansion_factor
+ * fma_expansion_factor
* args.hidden_size
* (
# dense layer (deepseek v2, v3 style)
- (args.ffn_hidden_size * gated_linear_multiplier)
- * (num_dense_layers / num_layers)
+ (args.ffn_hidden_size * ffn_expansion_factor)
+ * num_dense_layers
# routed experts
- + (moe_ffn_hidden_size * num_experts_routed_to * gated_linear_multiplier)
- * (num_moe_layers / num_layers)
+ + (moe_ffn_hidden_size * num_experts_routed_to * ffn_expansion_factor)
+ * num_moe_layers
# Shared Experts.
- + (shared_expert_ffn_hidden_size * gated_linear_multiplier)
- * (num_moe_layers / num_layers)
+ + (shared_expert_ffn_hidden_size * ffn_expansion_factor)
+ * num_moe_layers
)
# Self Attention
+ self_attn_term
# MTP norms and proj
- + 3
- * 2
+ + forward_backward_expansion_factor
+ * fma_expansion_factor
* mtp_num_layers
* (
# MTP eh norm + final nrom
@@ -442,7 +527,11 @@ def transformer_flops():
+ 2 * args.hidden_size * args.hidden_size
)
# Logit.
- + 3 * 2 * args.hidden_size * args.padded_vocab_size * (mtp_num_layers + 1)
+ + forward_backward_expansion_factor
+ * fma_expansion_factor
+ * args.hidden_size
+ * args.padded_vocab_size
+ * (mtp_num_layers + 1) # MTP + final logit
)
)
return total_floating_point_operations
@@ -639,11 +728,16 @@ def pretrain(
args = get_args()
timers = get_timers()
+ if args.fine_grained_activation_offloading:
+ from megatron.core.pipeline_parallel.utils import (
+ set_ideal_affinity_for_current_gpu
+ )
+ set_ideal_affinity_for_current_gpu()
+
if args.batch_invariant_mode:
print_rank_0("Enabling batch invariant mode globally",flush=True)
enable_batch_invariant_mode()
-
if args.log_progress:
append_to_progress_log("Starting job")
@@ -1055,8 +1149,6 @@ def build_model():
kwargs['pad_buckets_for_high_nccl_busbw'] = args.ddp_pad_buckets_for_high_nccl_busbw
kwargs['reduce_scatter_with_fp32_accumulation'] = args.ddp_reduce_scatter_with_fp32_accumulation
kwargs['average_in_collective'] = args.ddp_average_in_collective
- if args.use_megatron_fsdp and args.use_precision_aware_optimizer:
- kwargs["preserve_fp32_weights"] = False
ddp_config = DistributedDataParallelConfig(**kwargs)
# In the Megatron FSDP and DDP use path, we need to initialize the bucket size.
@@ -1154,7 +1246,9 @@ def get_megatron_optimizer_config(args: Any) -> OptimizerConfig:
"""Return a Megatron optimizer config object from Megatron's arguments."""
config = None
- if args.optimizer == 'adam':
+ if args.optimizer == 'adam' or 'muon' in args.optimizer:
+ # TODO(deyuf): Muon needs both adam + muon but get() only receive one config
+ # So for now we keep using adam config that's back compat with old way
kwargs = {}
for f in dataclasses.fields(AdamOptimizerConfig):
if hasattr(args, f.name):
@@ -1169,17 +1263,9 @@ def get_megatron_optimizer_config(args: Any) -> OptimizerConfig:
else:
raise ValueError("Invalid optimizer type!")
- # Construct the appropriate config_overrides object.
- # TODO: add more logic here as needed down the road.
- if args.decoupled_lr is not None:
- decoupled_param_key = ParamKey(attr="is_embedding_or_output_parameter")
- decoupled_optimizer_config = copy.deepcopy(config)
- decoupled_optimizer_config.lr = args.decoupled_lr
- if args.decoupled_min_lr is not None:
- decoupled_optimizer_config.min_lr = args.decoupled_min_lr
- config_overrides = {decoupled_param_key: decoupled_optimizer_config}
- else:
- config_overrides = None
+ # Construct the appropriate config_overrides object. This default handles many cases, but
+ # can be added to as needed by the user, or replaced entirely with a custom override.
+ config_overrides = get_standard_config_overrides(config=config)
return config, config_overrides
@@ -1205,16 +1291,26 @@ def setup_model_and_optimizer(
config, config_overrides = get_megatron_optimizer_config(args)
config.timers = timers
- # If the user is asking for a non-zero embedding init std, skip weight decay for embeddings
- # to avoid embeddings from shrinking to zero as recommended in https://arxiv.org/abs/2312.16903
- # default_skip_embedding_weight_decay=args.embedding_init_method_std is not None,
- optimizer = get_megatron_optimizer(
- config,
- model,
- config_overrides=config_overrides,
- use_gloo_process_groups=args.enable_gloo_process_groups,
- dump_param_to_param_group_map=args.dump_param_to_param_group_map,
- )
+ if 'muon' not in config.optimizer:
+ # If the user is asking for a non-zero embedding init std, skip weight decay for embeddings
+ # to avoid embeddings from shrinking to zero as recommended in https://arxiv.org/abs/2312.16903
+ # default_skip_embedding_weight_decay=args.embedding_init_method_std is not None,
+ optimizer = get_megatron_optimizer(
+ config,
+ model,
+ config_overrides=config_overrides,
+ use_gloo_process_groups=args.enable_gloo_process_groups,
+ dump_param_to_param_group_map=args.dump_param_to_param_group_map,
+ )
+ else:
+ optimizer = get_megatron_muon_optimizer(
+ config,
+ model,
+ config_overrides=config_overrides,
+ use_gloo_process_groups=args.enable_gloo_process_groups,
+ layer_wise_distributed_optimizer='dist' in config.optimizer,
+ )
+
opt_param_scheduler = get_optimizer_param_scheduler(optimizer)
one_logger and one_logger.log_metrics({"app_build_optimzer_finish_time": one_logger_utils.get_timestamp_in_ms()})
@@ -1234,7 +1330,7 @@ def setup_model_and_optimizer(
# set dense model related args in to global args before getting dense model
args.num_experts = None
args.expert_model_parallel_size = 1
- args.ffn_hidden_size = moe_ffn_hidden_size * args.moe_upcycling_granularity
+ args.ffn_hidden_size = moe_ffn_hidden_size * args.moe_upcycling_granularity
# get dense model
dense_model_for_upcycling = get_model(model_provider_func, model_type)
@@ -1346,6 +1442,12 @@ def train_step(forward_step_func, data_iterator, model, optimizer, opt_param_sch
rerun_state_machine = get_rerun_state_machine()
while rerun_state_machine.should_run_forward_backward(data_iterator):
+ # Offload optimizer states to CPU if enabled.
+ if args.offload_optimizer_states:
+ for optim_instance in optimizer.chained_optimizers:
+ if isinstance(optim_instance, DistributedOptimizer):
+ optim_instance.offload_states()
+
# Set grad to zero.
for model_chunk in model:
model_chunk.zero_grad_buffer()
@@ -1365,10 +1467,27 @@ def train_step(forward_step_func, data_iterator, model, optimizer, opt_param_sch
# For the mxfp8_param with reuse_grad_buf_for_mxfp8_param_ag and dp_ag_overlap,
# we need to call the _copy_main_params_to_param_buffer() after the grad buffer
# is zeroed by zero_grad_buffer() because param and grad buffer are shared.
+ #
+ # However, we should skip this on the first iteration when forward_pre_hook is disabled,
+ # because:
+ # 1. The first iteration's params are already in param.data (from init or checkpoint).
+ # 2. Without forward_pre_hook, finish_param_sync() won't be called to zero the grad buffer,
+ # so the main grads will be polluted by the main params.
if args.reuse_grad_buf_for_mxfp8_param_ag and args.overlap_param_gather:
+ # Check if forward_pre_hook is enabled by checking if hooks are registered.
+ forward_pre_hook_enabled = len(model[0].remove_forward_pre_hook_handles) > 0
+ if forward_pre_hook_enabled:
+ for optim_instance in optimizer.chained_optimizers:
+ if isinstance(optim_instance, DistributedOptimizer):
+ optim_instance._copy_main_params_to_param_buffer()
+
+ # Release GPU memory for offloaded optimizer states.
+ # This needs to be done after _copy_main_params_to_param_buffer().
+ # Separate offload and release to allow early D2H transfer to overlap with other operations.
+ if args.offload_optimizer_states:
for optim_instance in optimizer.chained_optimizers:
if isinstance(optim_instance, DistributedOptimizer):
- optim_instance._copy_main_params_to_param_buffer()
+ optim_instance.release_offloaded_gpu_states()
# Forward pass.
losses_reduced = forward_backward_func(
@@ -1441,28 +1560,14 @@ def train_step(forward_step_func, data_iterator, model, optimizer, opt_param_sch
for key in losses_reduced[0].keys():
val = [x[key].view(-1) for x in losses_reduced]
if val[0].numel() == 2:
- if args.sft:
- # in mcore the normalization happens on micro batch instead of global
- val = torch.vstack(val)
- val = val[:, 0] / val[:, 1]
- val = val.mean()
- torch.distributed.all_reduce(
- val,
- group=mpu.get_data_parallel_group(with_context_parallel=True)
- )
- val /= torch.distributed.get_world_size(
- group=mpu.get_data_parallel_group(with_context_parallel=True)
- )
- loss_reduced[key] = val
- else:
- # there is one dict per microbatch. in new reporting, we average
- # over the total number of tokens across the global batch.
- val = torch.vstack(val).sum(dim=0)
- torch.distributed.all_reduce(
- val,
- group=mpu.get_data_parallel_group(with_context_parallel=True)
- )
- loss_reduced[key] = val[0] / val[1]
+ # there is one dict per microbatch. in new reporting, we average
+ # over the total number of tokens across the global batch.
+ val = torch.vstack(val).sum(dim=0)
+ torch.distributed.all_reduce(
+ val,
+ group=mpu.get_data_parallel_group(with_context_parallel=True)
+ )
+ loss_reduced[key] = val[0] / val[1]
elif val[0].numel() == 1:
# legacy behavior, we average over the number of microbatches
val = torch.cat(val).mean()
@@ -1494,6 +1599,7 @@ def training_log(
params_norm,
num_zeros_in_grad,
max_attention_logit,
+ pg_collection=None,
):
"""Log training information such as losses, timing, ...."""
args = get_args()
@@ -1687,12 +1793,23 @@ def training_log(
num_layers=layers,
moe_layer_freq=args.moe_layer_freq,
mtp_num_layers=args.mtp_num_layers,
+ pg_collection=pg_collection,
)
if args.mtp_num_layers is not None:
mtp_loss_scale = 1 / get_num_microbatches()
MTPLossLoggingHelper.track_mtp_metrics(
mtp_loss_scale, iteration, writer, wandb_writer, total_loss_dict
)
+ # Track sparse attention indexer loss
+ if args.dsa_indexer_loss_coeff is not None and args.dsa_indexer_loss_coeff > 0:
+ indexer_loss_scale = 1 / get_num_microbatches()
+ DSAIndexerLossLoggingHelper.track_indexer_metrics(
+ loss_scale=indexer_loss_scale,
+ iteration=iteration,
+ writer=writer,
+ wandb_writer=wandb_writer,
+ total_loss_dict=total_loss_dict,
+ )
if iteration % args.log_interval == 0:
if args.record_memory_history and (is_last_rank() or torch.distributed.get_backend() == 'fake'):
snapshot = torch.cuda.memory._snapshot()
@@ -1855,7 +1972,8 @@ def save_checkpoint_and_time(
# Stop timer to get accurate train interval time and exclude checkpointing duration
timers('interval-time').stop()
- energy_monitor.pause()
+ if args.log_energy:
+ energy_monitor.pause()
# Extra barrier is added to make sure all ranks report the max time.
timer_key = 'save-checkpoint-non-persistent' if non_persistent_ckpt else 'save-checkpoint'
@@ -1895,7 +2013,9 @@ def save_checkpoint_and_time(
)
# Recover timing
- energy_monitor.resume()
+ if args.log_energy:
+ energy_monitor.resume()
+
timers('interval-time', log_level=0).start(barrier=True)
@@ -2147,6 +2267,9 @@ def train(
energy_monitor = get_energy_monitor()
one_logger = get_one_logger()
+ if args.hybrid_context_parallel:
+ train_data_iterator = iter(HybridCPDataLoaderWrapper(train_data_iterator, config))
+
if args.run_workload_inspector_server:
try:
from workload_inspector.utils.webserver import run_server
@@ -2165,6 +2288,8 @@ def train(
for model_module in model:
model_module.train()
+ model_pg_collection = get_attr_wrapped_model(model[0], "pg_collection")
+
# Tracking loss.
total_loss_dict = {}
@@ -2211,7 +2336,21 @@ def train(
config.param_sync_func = [model_chunk.start_param_sync for model_chunk in model]
if len(model) == 1:
config.param_sync_func = config.param_sync_func[0]
- config.finalize_model_grads_func = finalize_model_grads
+
+ # Wrap finalize_model_grads to reload offloaded optimizer states before grad finalization.
+ # This allows H2D transfer to overlap with grad all-reduce.
+ if args.offload_optimizer_states:
+
+ def finalize_model_grads_with_state_reload(*fmg_args, **fmg_kwargs):
+ # Reload offloaded states for all DistributedOptimizer instances
+ for optim_instance in optimizer.chained_optimizers:
+ if isinstance(optim_instance, DistributedOptimizer):
+ optim_instance.reload_offloaded_states()
+ return finalize_model_grads(*fmg_args, **fmg_kwargs)
+
+ config.finalize_model_grads_func = finalize_model_grads_with_state_reload
+ else:
+ config.finalize_model_grads_func = finalize_model_grads
if args.log_energy:
energy_monitor.setup()
@@ -2253,7 +2392,7 @@ def train(
eval_iterations = 0
# Wrap forward_backward_func for Full iteration CUDA graph
forward_backward_func = get_forward_backward_func()
- if args.cuda_graph_impl == "local" and args.cuda_graph_scope=="full_iteration":
+ if args.cuda_graph_impl == "local" and CudaGraphScope.full_iteration in args.cuda_graph_scope:
forward_backward_func = FullCudaGraphWrapper(forward_backward_func, cuda_graph_warmup_steps=args.cuda_graph_warmup_steps)
def get_e2e_base_metrics():
@@ -2386,12 +2525,13 @@ def get_e2e_base_metrics():
# Capture CUDA Graphs.
if (
args.cuda_graph_impl == "transformer_engine"
- and iteration == args.cuda_graph_warmup_steps
+ and not cuda_graph_helper.graphs_created()
+ and iteration - start_iteration == args.cuda_graph_warmup_steps
):
- if iteration > start_iteration and should_disable_forward_pre_hook(args):
+ if args.cuda_graph_warmup_steps > 0 and should_disable_forward_pre_hook(args):
disable_forward_pre_hook(model, param_sync=False)
cuda_graph_helper.create_cudagraphs()
- if iteration > start_iteration and should_disable_forward_pre_hook(args):
+ if args.cuda_graph_warmup_steps > 0 and should_disable_forward_pre_hook(args):
enable_forward_pre_hook(model)
cuda_graph_helper.cuda_graph_set_manual_hooks()
@@ -2469,12 +2609,29 @@ def get_e2e_base_metrics():
# Set the manual hooks here since it's not set right after the capturing.
if (
args.cuda_graph_impl == "transformer_engine"
- and iteration == args.cuda_graph_warmup_steps
+ and args.cuda_graph_warmup_steps == 0
):
+ assert (
+ cuda_graph_helper.graphs_created()
+ ), "CUDA Graphs should have been created."
cuda_graph_helper.cuda_graph_set_manual_hooks()
iteration += 1
+ # If requested, manually register FSDP communication buffers after a short warmup.
+ if (
+ getattr(args, "fsdp_manual_registration", False)
+ and getattr(args, "use_megatron_fsdp", False)
+ and iteration == start_iteration + 1
+ ):
+ for model_chunk in model:
+ if isinstance(model_chunk, megatron_FSDP) and getattr(
+ model_chunk.ddp_config, "fsdp_manual_registration", False
+ ):
+ pad_buf = getattr(model_chunk, "param_and_grad_buffer", None)
+ if pad_buf is not None:
+ pad_buf.manual_buffer_registration()
+
if getattr(args, 'perform_rl_step', False) and args.rl_use_sequence_packing:
iteration_sequences = rl_utils.get_iteration_sequence_count(args)
# Track bins separately for packed mode
@@ -2530,6 +2687,7 @@ def get_e2e_base_metrics():
params_norm,
num_zeros_in_grad,
max_attention_logit,
+ pg_collection=model_pg_collection,
)
# Evaluation.
@@ -2596,6 +2754,10 @@ def get_e2e_base_metrics():
if should_exit:
break
+ # Destroy CUDA Graphs.
+ if args.cuda_graph_impl == "transformer_engine" and cuda_graph_helper.graphs_created():
+ cuda_graph_helper.delete_cuda_graphs()
+
one_logger_utils.track_e2e_metrics()
# Flush TensorBoard, WandB writers and one-logger.
@@ -2671,7 +2833,7 @@ def evaluate(
eval_batch_size = args.global_batch_size
eval_num_microbatches = eval_batch_size // (args.micro_batch_size * args.data_parallel_size)
forward_backward_func = get_forward_backward_func()
- if args.cuda_graph_impl == "local" and args.cuda_graph_scope=="full_iteration":
+ if args.cuda_graph_impl == "local" and CudaGraphScope.full_iteration in args.cuda_graph_scope:
forward_backward_func = FullCudaGraphWrapper(forward_backward_func, cuda_graph_warmup_steps=args.cuda_graph_warmup_steps)
if eval_iters is None:
@@ -2819,7 +2981,7 @@ def evaluate_and_print_results(
eval_iters = [args.eval_iters]
else:
eval_iters = args.eval_iters
-
+
if args.full_validation:
assert len(eval_iters) == len(data_iterators)
@@ -2835,7 +2997,7 @@ def evaluate_and_print_results(
eval_iters = [args.eval_iters]
else:
eval_iters = args.eval_iters
-
+
for index, (iterator, iterations) in enumerate(zip(data_iterators, eval_iters)):
suffix = ""
if args.multiple_validation_sets:
@@ -2914,18 +3076,20 @@ def get_train_valid_test_num_samples():
return (train_samples, eval_samples, test_samples)
-def build_train_valid_test_datasets(build_train_valid_test_datasets_provider, train_valid_test_num_samples=None):
+def build_train_valid_test_datasets(build_train_valid_test_datasets_provider, train_valid_test_num_samples=None, vp_stage=None):
"""Build pretraining datasets."""
if train_valid_test_num_samples is None:
train_valid_test_num_samples = get_train_valid_test_num_samples()
- print_rank_0(' > datasets target sizes (minimum size):')
print_rank_0(' train: {}'.format(train_valid_test_num_samples[0]))
print_rank_0(' validation: {}'.format(train_valid_test_num_samples[1]))
print_rank_0(' test: {}'.format(train_valid_test_num_samples[2]))
- return build_train_valid_test_datasets_provider(train_valid_test_num_samples)
+ if vp_stage is not None:
+ return build_train_valid_test_datasets_provider(train_valid_test_num_samples, vp_stage=vp_stage)
+ else:
+ return build_train_valid_test_datasets_provider(train_valid_test_num_samples)
-def build_train_valid_test_data_loaders(build_train_valid_test_datasets_provider):
+def build_train_valid_test_data_loaders(build_train_valid_test_datasets_provider, vp_stage=None):
"""Build pretraining data loaders."""
args = get_args()
@@ -2964,7 +3128,10 @@ def build_train_valid_test_data_loaders(build_train_valid_test_datasets_provider
else:
# Build datasets.
- train_ds, valid_ds, test_ds = build_train_valid_test_datasets(build_train_valid_test_datasets_provider)
+ train_ds, valid_ds, test_ds = build_train_valid_test_datasets(
+ build_train_valid_test_datasets_provider,
+ vp_stage=vp_stage,
+ )
valid_ds = [valid_ds] if not isinstance(valid_ds, list) else valid_ds
if args.skip_train:
train_dataloader = None
@@ -3004,14 +3171,15 @@ def build_train_valid_test_data_loaders(build_train_valid_test_datasets_provider
return train_dataloader, valid_dataloaders, test_dataloader
-def build_train_valid_test_data_iterators(build_train_valid_test_datasets_provider):
+def build_train_valid_test_data_iterators(build_train_valid_test_datasets_provider, vp_stage=None):
"""Build pretraining data iterators."""
args = get_args()
# Build loaders.
train_dataloader, valid_dataloaders, test_dataloader = build_train_valid_test_data_loaders(
- build_train_valid_test_datasets_provider
+ build_train_valid_test_datasets_provider,
+ vp_stage=vp_stage
)
# Build iterators.
@@ -3040,7 +3208,7 @@ def _get_iterator(dataloader_type, dataloader):
if valid_dataloaders is not None:
# when using full validation, we need to override eval iters with the correct
- # number of iterations on tp rank 0 so that it can be distributed to the other
+ # number of iterations on tp rank 0 so that it can be distributed to the other
# ranks later
if args.full_validation:
if args.multiple_validation_sets:
diff --git a/megatron/training/utils.py b/megatron/training/utils.py
index 669f1972f02..4730a525271 100644
--- a/megatron/training/utils.py
+++ b/megatron/training/utils.py
@@ -1,4 +1,4 @@
-# Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved.
+# Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
"""General utilities."""
import json
@@ -38,6 +38,7 @@
from megatron.core.utils import (
get_batch_on_this_cp_rank,
get_data_parallel_group_if_dtensor,
+ is_torch_min_version,
to_local_if_dtensor,
unwrap_model,
)
@@ -282,6 +283,9 @@ def report_memory(name):
string += ' | max allocated: {}'.format(torch.cuda.max_memory_allocated() / mega_bytes)
string += ' | reserved: {}'.format(torch.cuda.memory_reserved() / mega_bytes)
string += ' | max reserved: {}'.format(torch.cuda.max_memory_reserved() / mega_bytes)
+ if is_torch_min_version("2.6.0"):
+ # device usage is not supported in torch < 2.6.0
+ string += ' | device usage: {}'.format(torch.cuda.device_memory_used() / mega_bytes)
if mpu.get_data_parallel_rank() == 0:
print("[Rank {}] {}".format(torch.distributed.get_rank(), string), flush=True)
@@ -511,7 +515,7 @@ def get_blend_and_blend_per_split(args):
return blend, blend_per_split
-def get_batch_on_this_tp_rank(data_iterator):
+def get_batch_on_this_tp_rank(data_iterator, mtp_on_this_rank: bool = False):
args = get_args()
@@ -537,68 +541,141 @@ def _broadcast(item):
else data["attention_mask"].cuda(non_blocking=True)
),
'position_ids': data["position_ids"].cuda(non_blocking=True),
+ 'cu_seqlens': (
+ None
+ if "cu_seqlens" not in data
+ else data["cu_seqlens"].cuda(non_blocking=True)
+ ),
+ 'max_seqlen': (
+ None
+ if "max_seqlen" not in data
+ else data["max_seqlen"].cuda(non_blocking=True)
+ ),
+ 'local_cp_size': (
+ None
+ if "local_cp_size" not in data
+ else data["local_cp_size"].cuda(non_blocking=True)
+ ),
}
- if args.pipeline_model_parallel_size == 1:
+ def _broadcast_cu_seqlens(cu_seqlens):
+ dev = torch.cuda.current_device()
+ n = 0 if cu_seqlens is None else int(cu_seqlens.numel())
+ n_tensor = torch.tensor(n, dtype=torch.int64, device=dev)
+ _broadcast(n_tensor)
+
+ if n == 0:
+ buf = torch.empty(0, dtype=torch.int32, device=dev)
+ else:
+ assert isinstance(cu_seqlens, torch.Tensor)
+ assert cu_seqlens.dtype == torch.int32
+ assert cu_seqlens.shape[0] == 1, "micro-batch-size must be 1 for packing"
+ buf = cu_seqlens.to(device=dev, non_blocking=True).contiguous()
+ _broadcast(buf)
+
+ if args.hybrid_context_parallel:
+ seq_len = torch.tensor(batch['tokens'].shape[0], dtype=torch.int32, device=torch.cuda.current_device())
+ _broadcast(seq_len)
+
+ if args.pipeline_model_parallel_size == 1 or mtp_on_this_rank:
_broadcast(batch['tokens'])
_broadcast(batch['labels'])
_broadcast(batch['loss_mask'])
_broadcast(batch['attention_mask'])
_broadcast(batch['position_ids'])
+ _broadcast_cu_seqlens(batch['cu_seqlens'])
+ _broadcast(batch['max_seqlen'])
+ _broadcast(batch['local_cp_size'])
elif mpu.is_pipeline_first_stage():
_broadcast(batch['tokens'])
_broadcast(batch['attention_mask'])
_broadcast(batch['position_ids'])
+ _broadcast_cu_seqlens(batch['cu_seqlens'])
+ _broadcast(batch['max_seqlen'])
elif mpu.is_pipeline_last_stage():
# Multi-Token Prediction (MTP) layers need tokens and position_ids to calculate embedding.
# Currently the Multi-Token Prediction (MTP) layers is fixed on the last stage, so we need
# to broadcast tokens and position_ids to all of the tensor parallel ranks on the last stage.
- if args.mtp_num_layers is not None:
- _broadcast(batch['tokens'])
- _broadcast(batch['position_ids'])
_broadcast(batch['labels'])
_broadcast(batch['loss_mask'])
_broadcast(batch['attention_mask'])
else:
-
+ if args.hybrid_context_parallel:
+ seq_len = torch.tensor(0, dtype=torch.int32, device=torch.cuda.current_device())
+ _broadcast(seq_len)
+ shape = (seq_len.item())
+ else:
+ shape = (args.micro_batch_size, args.seq_length)
+
tokens = torch.empty(
- (args.micro_batch_size, args.seq_length),
+ shape,
dtype=torch.int64,
device=torch.cuda.current_device(),
)
labels = torch.empty(
- (args.micro_batch_size, args.seq_length),
+ shape,
dtype=torch.int64,
device=torch.cuda.current_device(),
)
loss_mask = torch.empty(
- (args.micro_batch_size, args.seq_length),
+ shape,
dtype=torch.float32,
device=torch.cuda.current_device(),
)
if args.create_attention_mask_in_dataloader:
+ shape_attention_mask = (args.micro_batch_size, 1, args.seq_length, args.seq_length) if not args.hybrid_context_parallel else (1, 1, shape[0], shape[0])
attention_mask = torch.empty(
- (args.micro_batch_size, 1, args.seq_length, args.seq_length),
+ shape_attention_mask,
dtype=torch.bool,
device=torch.cuda.current_device(),
)
else:
attention_mask = None
position_ids = torch.empty(
- (args.micro_batch_size, args.seq_length),
+ shape,
dtype=torch.int64,
device=torch.cuda.current_device(),
)
- if args.pipeline_model_parallel_size == 1:
+ cu_seqlens = None
+ max_seqlen = torch.empty(
+ 1,
+ dtype=torch.int32,
+ device=torch.cuda.current_device(),
+ ) if args.hybrid_context_parallel else None
+ local_cp_size = torch.empty(
+ 1,
+ dtype=torch.int32,
+ device=torch.cuda.current_device(),
+ ) if args.hybrid_context_parallel else None
+
+ def _broadcast_cu_seqlens():
+ dev = torch.cuda.current_device()
+
+ n = torch.empty((), dtype=torch.int64, device=dev)
+ _broadcast(n)
+ n = int(n.item())
+
+ if n == 0:
+ cu_seqlens = torch.empty(0, dtype=torch.int32, device=dev)
+ else:
+ cu_seqlens = torch.empty((args.micro_batch_size, n), dtype=torch.int32, device=dev)
+ _broadcast(cu_seqlens)
+
+ return cu_seqlens if n > 0 else None
+
+ if args.pipeline_model_parallel_size == 1 or mtp_on_this_rank:
_broadcast(tokens)
_broadcast(labels)
_broadcast(loss_mask)
_broadcast(attention_mask)
_broadcast(position_ids)
+ cu_seqlens = _broadcast_cu_seqlens()
+ _broadcast(max_seqlen)
+ _broadcast(local_cp_size)
elif mpu.is_pipeline_first_stage():
labels = None
@@ -607,18 +684,17 @@ def _broadcast(item):
_broadcast(tokens)
_broadcast(attention_mask)
_broadcast(position_ids)
+ cu_seqlens = _broadcast_cu_seqlens()
+ _broadcast(max_seqlen)
elif mpu.is_pipeline_last_stage():
# Multi-Token Prediction (MTP) layers need tokens and position_ids to calculate embedding.
# Currently the Multi-Token Prediction (MTP) layers is fixed on the last stage, so we need
# to broadcast tokens and position_ids to all of the tensor parallel ranks on the last stage.
- if args.mtp_num_layers is not None:
- _broadcast(tokens)
- _broadcast(position_ids)
- else:
- tokens = None
- position_ids = None
-
+ tokens = None
+ position_ids = None
+ cu_seqlens = None
+ max_seqlen = None
_broadcast(labels)
_broadcast(loss_mask)
_broadcast(attention_mask)
@@ -629,6 +705,9 @@ def _broadcast(item):
'loss_mask': loss_mask,
'attention_mask': attention_mask,
'position_ids': position_ids,
+ 'cu_seqlens': cu_seqlens,
+ 'max_seqlen': max_seqlen,
+ 'local_cp_size': local_cp_size,
}
return batch
diff --git a/pretrain_gpt.py b/pretrain_gpt.py
index 6b602d33243..cfb5e1b5f1f 100644
--- a/pretrain_gpt.py
+++ b/pretrain_gpt.py
@@ -1,4 +1,4 @@
-# Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved.
+# Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
"""Pretrain and SFT GPT."""
@@ -14,8 +14,10 @@
from megatron.core.enums import ModelType
from megatron.core.models.gpt import GPTModel
from megatron.core.rerun_state_machine import get_rerun_state_machine
+from megatron.core.utils import get_attr_wrapped_model, get_thd_batch_on_this_cp_rank, get_batch_on_this_hybrid_cp_rank, StragglerDetector
from megatron.core.tokenizers.text.utils.build_tokenizer import build_tokenizer
-from megatron.core.utils import StragglerDetector, get_attr_wrapped_model
+from megatron.core.transformer.multi_token_prediction import mtp_on_this_rank, get_mtp_ranks
+from megatron.training.arguments import core_transformer_config_from_args
from megatron.training import get_args, get_timers, get_tokenizer, inprocess_restart, pretrain, print_rank_0
from megatron.training.datasets.sft_dataset import SFTDataset
from megatron.training.datasets.fim_dataset import GPTFIMDataset, GPTFIMDatasetConfig
@@ -38,19 +40,39 @@
stimer = StragglerDetector()
-def get_batch(data_iterator, vp_stage=None):
+def get_batch(data_iterator, vp_stage: Optional[int] = None):
"""Generate a batch."""
+ args = get_args()
+ config = core_transformer_config_from_args(args)
# TODO: this is pretty hacky, find a better way
- if not is_first_or_last_pipeline_stage(vp_stage):
- return None, None, None, None, None
+ if not is_first_or_last_pipeline_stage(vp_stage) and (
+ (not mtp_on_this_rank(config, ignore_virtual=False, vp_stage=vp_stage))):
+ return None, None, None, None, None, None
# get batches based on the TP rank you are on
- batch = get_batch_on_this_tp_rank(data_iterator)
-
- # slice batch along sequence dimension for context parallelism
- batch = get_batch_on_this_cp_rank(batch)
+ batch = get_batch_on_this_tp_rank(
+ data_iterator,
+ mtp_on_this_rank=mtp_on_this_rank(config, ignore_virtual=False, vp_stage=vp_stage)
+ )
- return batch.values()
+ cu_seqlens = batch.pop('cu_seqlens', None)
+ cu_seqlens_padded = batch.pop('cu_seqlens_padded', None)
+ max_seqlen = batch.pop('max_seqlen', None)
+ local_cp_size = batch.pop('local_cp_size', None)
+ if local_cp_size is not None:
+ local_cp_size = int(local_cp_size.item())
+
+ if cu_seqlens is None and local_cp_size is None:
+ # slice batch along sequence dimension for context parallelism
+ batch = get_batch_on_this_cp_rank(batch) # The implementation of this function is in MCore
+ packed_seq_params = None
+ elif local_cp_size is None: # Packed THD format
+ assert max_seqlen.dim() == 1
+ batch, packed_seq_params = get_thd_batch_on_this_cp_rank(batch, cu_seqlens, cu_seqlens_padded, max_seqlen)
+ else: # Hybrid CP format
+ batch, packed_seq_params = get_batch_on_this_hybrid_cp_rank(batch, local_cp_size)
+
+ return (*batch.values(), packed_seq_params)
# define spiky loss as a loss that's 10x the max loss observed
@@ -135,7 +157,7 @@ def forward_step(data_iterator, model: GPTModel, return_schedule_plan: bool = Fa
global stimer
with stimer(bdata=True):
vp_stage = get_attr_wrapped_model(model, "vp_stage")
- tokens, labels, loss_mask, attention_mask, position_ids = get_batch(data_iterator, vp_stage)
+ tokens, labels, loss_mask, attention_mask, position_ids, packed_seq_params = get_batch(data_iterator, vp_stage)
timers('batch-generator').stop()
with stimer:
@@ -151,7 +173,7 @@ def forward_step(data_iterator, model: GPTModel, return_schedule_plan: bool = Fa
return schedule_plan, partial(loss_func, loss_mask, model=model)
else:
output_tensor = model(
- tokens, position_ids, attention_mask, labels=labels, loss_mask=loss_mask
+ tokens, position_ids, attention_mask, labels=labels, loss_mask=loss_mask, packed_seq_params=packed_seq_params
)
# [ModelOpt]: model is needed to access ModelOpt distillation losses
@@ -159,7 +181,12 @@ def forward_step(data_iterator, model: GPTModel, return_schedule_plan: bool = Fa
def is_dataset_built_on_rank(vp_stage=None):
- return is_first_or_last_pipeline_stage(vp_stage) and parallel_state.get_tensor_model_parallel_rank() == 0
+ args = get_args()
+ config = core_transformer_config_from_args(args)
+ return (
+ is_first_or_last_pipeline_stage(vp_stage)
+ or mtp_on_this_rank(config, ignore_virtual=False, vp_stage=vp_stage)
+ ) and parallel_state.get_tensor_model_parallel_rank() == 0
def core_gpt_dataset_config_from_args(args):
@@ -192,6 +219,10 @@ def core_gpt_dataset_config_from_args(args):
"object_storage_cache_path": args.object_storage_cache_path,
"mid_level_dataset_surplus": args.mid_level_dataset_surplus,
"allow_ambiguous_pad_tokens": args.allow_ambiguous_pad_tokens,
+ "context_parallel_size": args.context_parallel_size,
+ "data_parallel_size": args.data_parallel_size,
+ "sequence_parallel_size": args.tensor_model_parallel_size*args.sequence_parallel,
+ "hybrid_context_parallel": args.hybrid_context_parallel,
}
# add FIM args to the config
@@ -240,6 +271,7 @@ def train_valid_test_datasets_provider(train_val_test_num_samples, vp_stage=None
print_rank_0("> building train, validation, and test datasets for GPT ...")
+ is_dataset_built = partial(is_dataset_built_on_rank, vp_stage=vp_stage)
train_ds, valid_ds, test_ds = BlendedMegatronDatasetBuilder(
dataset_type, train_val_test_num_samples, partial(is_dataset_built_on_rank, vp_stage=vp_stage), config
).build()
@@ -249,6 +281,21 @@ def train_valid_test_datasets_provider(train_val_test_num_samples, vp_stage=None
return train_ds, valid_ds, test_ds
+def get_embedding_ranks(pp_ranks: List[int]):
+ """Get the embedding ranks."""
+ embedding_ranks = [pp_ranks[0]]
+ if len(pp_ranks) > 1:
+ args = get_args()
+ if not args.untie_embeddings_and_output_weights:
+ embedding_ranks.append(pp_ranks[-1])
+ config = core_transformer_config_from_args(args)
+ mtp_ranks = get_mtp_ranks(pp_ranks, config)
+ embedding_ranks.extend(mtp_ranks)
+ embedding_ranks = list(set(embedding_ranks))
+ embedding_ranks = sorted(embedding_ranks)
+ return embedding_ranks
+
+
if __name__ == "__main__":
# Temporary for transition to core datasets
@@ -265,4 +312,5 @@ def train_valid_test_datasets_provider(train_val_test_num_samples, vp_stage=None
args_defaults={'tokenizer_type': 'GPT2BPETokenizer'},
extra_args_provider=add_modelopt_args if has_nvidia_modelopt else None,
store=store,
+ get_embedding_ranks=get_embedding_ranks,
)
diff --git a/pretrain_mamba.py b/pretrain_mamba.py
index 45b646a6cc0..ca2008620be 100644
--- a/pretrain_mamba.py
+++ b/pretrain_mamba.py
@@ -44,6 +44,13 @@ def get_batch(data_iterator, vp_stage=None):
# get batches based on the TP rank you are on
batch = get_batch_on_this_tp_rank(data_iterator)
+
+ # Support for Packed Sequence (Unused in this script)
+ cu_seqlens = batch.pop('cu_seqlens', None)
+ cu_seqlens_padded = batch.pop('cu_seqlens_padded', None)
+ max_seqlen = batch.pop('max_seqlen', None)
+ # Support for Hybrid Context Parallel (Unused in this script)
+ local_cp_size = batch.pop('local_cp_size', None)
# slice batch along sequence dimension for context parallelism
batch = get_batch_on_this_cp_rank(batch)
diff --git a/pyproject.toml b/pyproject.toml
index a4e6b9c7965..22ee405cb4f 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -1,7 +1,7 @@
# Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved.
[build-system]
-requires = ["setuptools>=80.0.0", "pybind11", "packaging>=24.2"]
+requires = ["setuptools<80.0.0", "pybind11", "packaging>=24.2"]
build-backend = "setuptools.build_meta"
[tool.setuptools]
@@ -68,7 +68,7 @@ mlm = ["flask-restful", "sentencepiece", "tiktoken", "wandb", "transformers"]
dev = [
"nvidia-modelopt[torch]; sys_platform != 'darwin'",
- "transformer-engine[pytorch,core_cu13]>=2.9.0a0,<2.11.0",
+ "transformer-engine[pytorch,core_cu13]>=2.9.0a0,<2.12.0",
"nvidia-resiliency-ext",
"tqdm",
"einops~=0.8",
@@ -84,6 +84,8 @@ dev = [
"flashinfer-python",
"wget",
"onnxscript",
+ "flash-linear-attention~=0.3.2",
+ "emerging_optimizers",
"fastapi~=0.50", # Forcing a little bit more recent version of fastapi to be compatible with pydantic 2.0
"datasets",
]
@@ -172,12 +174,13 @@ override-dependencies = [
]
[tool.uv.sources]
+
flash_mla = [
{ git = "https://github.com/deepseek-ai/FlashMLA", rev = "9edee0c022cd0938148a18e334203b0aab43aa19" },
]
-transformer-engine = { git = "https://github.com/NVIDIA/TransformerEngine.git", rev = "release_v2.10" } # on `release_v2.10`
+transformer-engine = { git = "https://github.com/NVIDIA/TransformerEngine.git", rev = "release_v2.11" }
nemo-run = { git = "https://github.com/NVIDIA-NeMo/Run.git", rev = "01a9a8ba360f7b2908728ad0516e0ad9d936966d" }
-emerging_optimizers = { git = "https://github.com/NVIDIA-NeMo/Emerging-Optimizers.git", rev = "fb1add873e7851ec34b48581ea1b15761b73d189" }
+emerging_optimizers = { git = "https://github.com/NVIDIA-NeMo/Emerging-Optimizers.git", rev = "v0.1.0" }
[tool.isort]
profile = "black" # black-compatible
diff --git a/tests/functional_tests/shell_test_utils/_run_training.sh b/tests/functional_tests/shell_test_utils/_run_training.sh
index 1d0e77a3477..72fd187d19d 100644
--- a/tests/functional_tests/shell_test_utils/_run_training.sh
+++ b/tests/functional_tests/shell_test_utils/_run_training.sh
@@ -159,7 +159,7 @@ MASTER_PORT=${MASTER_PORT:-6000}
NUM_NODES=${NUM_NODES:-${SLURM_NNODES:-1}}
GPUS_PER_NODE=${GPUS_PER_NODE:-8}
NODE_RANK=${SLURM_NODEID:-${SLURM_NODEID:-0}}
-LAST_RANK=7
+LAST_RANK=$((GPUS_PER_NODE - 1))
export LOG_DIR=$OUTPUT_PATH/logs/$REPEAT
mkdir -p $LOG_DIR
@@ -170,7 +170,7 @@ DISTRIBUTED_ARGS=(
--master_port $MASTER_PORT
--node_rank $NODE_RANK
--log-dir $LOG_DIR
- --tee "0:3,7:3"
+ --tee "0:3,$LAST_RANK:3"
--redirects "3"
)
diff --git a/tests/functional_tests/shell_test_utils/run_ci_test.sh b/tests/functional_tests/shell_test_utils/run_ci_test.sh
index d2c55838565..20267536a0f 100644
--- a/tests/functional_tests/shell_test_utils/run_ci_test.sh
+++ b/tests/functional_tests/shell_test_utils/run_ci_test.sh
@@ -51,6 +51,8 @@ set -exo pipefail
# Extract settings from params file
TEST_TYPE=$(cat $TRAINING_PARAMS_PATH |
/usr/local/bin/yq '.TEST_TYPE')
+ENABLE_LIGHTWEIGHT_MODE=$(cat $TRAINING_PARAMS_PATH |
+ /usr/local/bin/yq '.ENV_VARS.ENABLE_LIGHTWEIGHT_MODE // "false"')
MODE=$(cat $TRAINING_PARAMS_PATH |
/usr/local/bin/yq '.MODE // "pretraining"')
@@ -67,6 +69,7 @@ mkdir -p $CHECKPOINT_SAVE_PATH
mkdir -p $CHECKPOINT_LOAD_PATH || true
_CHECKPOINT_LOAD_PATH=$CHECKPOINT_LOAD_PATH
_CHECKPOINT_SAVE_PATH=$CHECKPOINT_SAVE_PATH
+_TENSORBOARD_PATH=$TENSORBOARD_PATH
SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd)
ROOT_DIR=$(realpath $SCRIPT_DIR/../../../)
@@ -133,6 +136,10 @@ for i in $(seq 1 $N_REPEAT); do
# First run never loads from a checkpoint
export RUN_NUMBER=1
+ DIR=$(dirname "$_TENSORBOARD_PATH")
+ FILE=$(basename "$_TENSORBOARD_PATH")
+ export TENSORBOARD_PATH=$DIR/$i/$FILE
+ mkdir -p $(dirname $TENSORBOARD_PATH)
export REPEAT=$i
export CHECKPOINT_SAVE_PATH=$_CHECKPOINT_SAVE_PATH
export TRAINING_EXIT_CODE=0
diff --git a/tests/functional_tests/test_cases/bert/bert_release/model_config.yaml b/tests/functional_tests/test_cases/bert/bert_release/model_config.yaml
index ab5558fa7d2..278ad6c17a8 100644
--- a/tests/functional_tests/test_cases/bert/bert_release/model_config.yaml
+++ b/tests/functional_tests/test_cases/bert/bert_release/model_config.yaml
@@ -27,7 +27,7 @@ MODEL_ARGS:
--pipeline-model-parallel-size: 8
# Data args
--data-path: ${DATA_BLEND}
- --vocab-file: ${DATA_PATH}/text/the_pile/bert_shard00/vocab.txt
+ --vocab-file: ${DATA_PATH}/vocab.txt
--split: 949,50,1
--data-cache-path: ${DATA_CACHE_PATH}
# EVAL_AND_LOGGING_ARGS
diff --git a/tests/functional_tests/test_cases/ci_base_config.yml b/tests/functional_tests/test_cases/ci_base_config.yml
new file mode 100644
index 00000000000..739f343da9d
--- /dev/null
+++ b/tests/functional_tests/test_cases/ci_base_config.yml
@@ -0,0 +1,14 @@
+MODEL_ARGS:
+ # Add logging args
+ --log-timers-to-tensorboard: true
+ --log-memory-to-tensorboard: true
+ --log-num-zeros-in-grad: true
+ --log-params-norm: true
+ --log-validation-ppl-to-tensorboard: true
+ --log-throughput: true
+ --log-interval: 1
+ --logging-level: 40
+ --tensorboard-dir: ${TENSORBOARD_PATH}
+ # Add checkpointing args
+ --save: ${CHECKPOINT_SAVE_PATH}
+ --load: ${CHECKPOINT_LOAD_PATH}
diff --git a/tests/functional_tests/test_cases/gpt/gpt3_15b_8t_release_sm/golden_values_dev_dgx_h100.json b/tests/functional_tests/test_cases/gpt/gpt3_15b_8t_release_sm/golden_values_dev_dgx_h100.json
index 02bcf7fe698..523227bf433 100644
--- a/tests/functional_tests/test_cases/gpt/gpt3_15b_8t_release_sm/golden_values_dev_dgx_h100.json
+++ b/tests/functional_tests/test_cases/gpt/gpt3_15b_8t_release_sm/golden_values_dev_dgx_h100.json
@@ -1 +1,10242 @@
-{"lm loss": {"start_step": 1, "end_step": 13000, "step_interval": 5, "values": {"1": 12.98419, "5": 12.93858, "10": 12.06404, "15": 11.97882, "20": 10.53588, "25": 10.11952, "30": 9.7286, "35": 9.44173, "40": 9.2373, "45": 9.03763, "50": 8.85277, "55": 8.64259, "60": 8.60098, "65": 8.50179, "70": 8.41326, "75": 8.31346, "80": 8.16921, "85": 8.09253, "90": 7.97894, "95": 7.91859, "100": 7.82704, "105": 7.71191, "110": 7.62418, "115": 7.52685, "120": 7.48107, "125": 7.48004, "130": 7.33364, "135": 7.26758, "140": 7.23146, "145": 7.04647, "150": 7.17621, "155": 7.00383, "160": 6.89968, "165": 6.91293, "170": 6.84228, "175": 6.85916, "180": 6.81429, "185": 6.7203, "190": 6.66124, "195": 6.59364, "200": 6.64046, "205": 6.64305, "210": 6.5179, "215": 6.51519, "220": 6.51027, "225": 6.46653, "230": 6.47574, "235": 6.42409, "240": 6.36976, "245": 6.3778, "250": 6.29868, "255": 6.43438, "260": 6.34377, "265": 6.28803, "270": 6.23364, "275": 6.26123, "280": 6.19076, "285": 6.19886, "290": 6.15022, "295": 6.12619, "300": 6.11141, "305": 6.01886, "310": 6.08556, "315": 6.07169, "320": 5.99243, "325": 5.93189, "330": 5.99792, "335": 6.0145, "340": 5.93453, "345": 5.92339, "350": 5.87179, "355": 5.84258, "360": 5.85866, "365": 5.81752, "370": 5.80407, "375": 5.80516, "380": 5.85848, "385": 5.78993, "390": 5.81141, "395": 5.68051, "400": 5.66121, "405": 5.68906, "410": 5.66202, "415": 5.70461, "420": 5.63851, "425": 5.66062, "430": 5.62802, "435": 5.56913, "440": 5.62147, "445": 5.52803, "450": 5.58428, "455": 5.5123, "460": 5.49325, "465": 5.56828, "470": 5.54845, "475": 5.49678, "480": 5.46247, "485": 5.49185, "490": 5.47566, "495": 5.47856, "500": 5.42533, "505": 5.38883, "510": 5.44319, "515": 5.42148, "520": 5.47608, "525": 5.31477, "530": 5.33216, "535": 5.36, "540": 5.33276, "545": 5.41314, "550": 5.37099, "555": 5.23374, "560": 5.32665, "565": 5.27809, "570": 5.25324, "575": 5.28184, "580": 5.23593, "585": 5.21762, "590": 5.22346, "595": 5.22561, "600": 5.26751, "605": 5.22896, "610": 5.2012, "615": 5.18737, "620": 5.19543, "625": 5.19655, "630": 5.14985, "635": 5.12452, "640": 5.09298, "645": 5.13279, "650": 5.14481, "655": 5.11963, "660": 5.0475, "665": 5.1142, "670": 5.04119, "675": 5.01723, "680": 5.05635, "685": 5.00678, "690": 5.01633, "695": 4.96228, "700": 4.97301, "705": 4.95571, "710": 4.97305, "715": 4.87719, "720": 4.85764, "725": 4.80769, "730": 4.84352, "735": 4.82916, "740": 4.8644, "745": 4.74895, "750": 4.75764, "755": 4.8023, "760": 4.78257, "765": 4.76428, "770": 4.69615, "775": 4.69212, "780": 4.684, "785": 4.7405, "790": 4.67498, "795": 4.64675, "800": 4.61184, "805": 4.61203, "810": 4.65393, "815": 4.60253, "820": 4.62914, "825": 4.58486, "830": 4.57946, "835": 4.56275, "840": 4.48603, "845": 4.50743, "850": 4.45704, "855": 4.51258, "860": 4.43583, "865": 4.52116, "870": 4.47717, "875": 4.38345, "880": 4.41849, "885": 4.38985, "890": 4.43389, "895": 4.42652, "900": 4.39808, "905": 4.32618, "910": 4.35391, "915": 4.34171, "920": 4.38377, "925": 4.38353, "930": 4.30961, "935": 4.29021, "940": 4.36235, "945": 4.31265, "950": 4.35051, "955": 4.28087, "960": 4.19218, "965": 4.27537, "970": 4.26236, "975": 4.24397, "980": 4.22146, "985": 4.17523, "990": 4.13237, "995": 4.18304, "1000": 4.2308, "1005": 4.18908, "1010": 4.17127, "1015": 4.13273, "1020": 4.15962, "1025": 4.22451, "1030": 4.1247, "1035": 4.10028, "1040": 4.13078, "1045": 4.11598, "1050": 4.15985, "1055": 4.0973, "1060": 4.1011, "1065": 4.06848, "1070": 4.05654, "1075": 4.07977, "1080": 4.07117, "1085": 4.06586, "1090": 4.02567, "1095": 4.09923, "1100": 4.06572, "1105": 4.071, "1110": 4.03046, "1115": 4.00398, "1120": 4.00356, "1125": 4.00318, "1130": 4.04694, "1135": 4.01101, "1140": 4.00143, "1145": 3.93633, "1150": 4.03948, "1155": 4.00619, "1160": 3.9851, "1165": 3.86882, "1170": 3.92684, "1175": 3.93038, "1180": 3.95878, "1185": 3.96184, "1190": 3.93034, "1195": 3.94125, "1200": 3.89932, "1205": 3.87266, "1210": 3.97831, "1215": 3.84899, "1220": 3.87263, "1225": 3.81413, "1230": 3.92846, "1235": 3.89312, "1240": 3.87537, "1245": 3.80079, "1250": 3.82885, "1255": 3.8493, "1260": 3.8874, "1265": 3.79336, "1270": 3.88167, "1275": 3.8455, "1280": 3.82143, "1285": 3.85735, "1290": 3.87159, "1295": 3.84912, "1300": 3.81711, "1305": 3.82495, "1310": 3.82551, "1315": 3.81192, "1320": 3.81405, "1325": 3.72789, "1330": 3.79529, "1335": 3.76889, "1340": 3.76123, "1345": 3.74704, "1350": 3.73516, "1355": 3.76572, "1360": 3.73919, "1365": 3.73441, "1370": 3.73284, "1375": 3.74202, "1380": 3.74909, "1385": 3.7541, "1390": 3.66502, "1395": 3.73883, "1400": 3.72777, "1405": 3.66183, "1410": 3.67095, "1415": 3.64909, "1420": 3.7012, "1425": 3.71121, "1430": 3.6707, "1435": 3.65988, "1440": 3.63587, "1445": 3.67497, "1450": 3.67892, "1455": 3.65003, "1460": 3.64827, "1465": 3.69085, "1470": 3.63258, "1475": 3.6929, "1480": 3.65433, "1485": 3.65353, "1490": 3.61514, "1495": 3.60996, "1500": 3.65344, "1505": 3.69268, "1510": 3.55544, "1515": 3.60731, "1520": 3.63573, "1525": 3.59446, "1530": 3.59436, "1535": 3.59439, "1540": 3.59806, "1545": 3.59944, "1550": 3.56476, "1555": 3.56897, "1560": 3.61095, "1565": 3.62486, "1570": 3.58908, "1575": 3.54769, "1580": 3.59493, "1585": 3.58838, "1590": 3.47011, "1595": 3.51425, "1600": 3.50212, "1605": 3.55557, "1610": 3.56864, "1615": 3.50712, "1620": 3.52052, "1625": 3.47381, "1630": 3.50275, "1635": 3.55197, "1640": 3.51791, "1645": 3.54635, "1650": 3.49031, "1655": 3.4704, "1660": 3.52131, "1665": 3.45881, "1670": 3.51236, "1675": 3.50045, "1680": 3.46681, "1685": 3.49078, "1690": 3.48353, "1695": 3.49776, "1700": 3.46185, "1705": 3.40868, "1710": 3.48728, "1715": 3.49675, "1720": 3.43132, "1725": 3.44122, "1730": 3.42961, "1735": 3.4644, "1740": 3.45584, "1745": 3.43534, "1750": 3.41748, "1755": 3.42246, "1760": 3.37607, "1765": 3.42468, "1770": 3.43261, "1775": 3.379, "1780": 3.4355, "1785": 3.42306, "1790": 3.37807, "1795": 3.40594, "1800": 3.34684, "1805": 3.39025, "1810": 3.34654, "1815": 3.4221, "1820": 3.41309, "1825": 3.38102, "1830": 3.32801, "1835": 3.42581, "1840": 3.38756, "1845": 3.42483, "1850": 3.39575, "1855": 3.37642, "1860": 3.35084, "1865": 3.38387, "1870": 3.29854, "1875": 3.45603, "1880": 3.34133, "1885": 3.36396, "1890": 3.34216, "1895": 3.39809, "1900": 3.36794, "1905": 3.30189, "1910": 3.32844, "1915": 3.3151, "1920": 3.36278, "1925": 3.33318, "1930": 3.31578, "1935": 3.31145, "1940": 3.36438, "1945": 3.26186, "1950": 3.40063, "1955": 3.30708, "1960": 3.31486, "1965": 3.28405, "1970": 3.29999, "1975": 3.33744, "1980": 3.34165, "1985": 3.23762, "1990": 3.32593, "1995": 3.28362, "2000": 3.27303, "2005": 3.26618, "2010": 3.28661, "2015": 3.22715, "2020": 3.27479, "2025": 3.27135, "2030": 3.27231, "2035": 3.29709, "2040": 3.26992, "2045": 3.23395, "2050": 3.27144, "2055": 3.32457, "2060": 3.28203, "2065": 3.24424, "2070": 3.30416, "2075": 3.24361, "2080": 3.22085, "2085": 3.30421, "2090": 3.15415, "2095": 3.29493, "2100": 3.23056, "2105": 3.19586, "2110": 3.20308, "2115": 3.24137, "2120": 3.18045, "2125": 3.21484, "2130": 3.22503, "2135": 3.27763, "2140": 3.1948, "2145": 3.21466, "2150": 3.20867, "2155": 3.2365, "2160": 3.20615, "2165": 3.25296, "2170": 3.23909, "2175": 3.17384, "2180": 3.22049, "2185": 3.25755, "2190": 3.24863, "2195": 3.15892, "2200": 3.2045, "2205": 3.17192, "2210": 3.12378, "2215": 3.19067, "2220": 3.19593, "2225": 3.18949, "2230": 3.13791, "2235": 3.19063, "2240": 3.21593, "2245": 3.1871, "2250": 3.21088, "2255": 3.1585, "2260": 3.14105, "2265": 3.23218, "2270": 3.21271, "2275": 3.1566, "2280": 3.17614, "2285": 3.16486, "2290": 3.17692, "2295": 3.20197, "2300": 3.13847, "2305": 3.15963, "2310": 3.12614, "2315": 3.06316, "2320": 3.11512, "2325": 3.17382, "2330": 3.12397, "2335": 3.12649, "2340": 3.17195, "2345": 3.12462, "2350": 3.129, "2355": 3.11726, "2360": 3.16031, "2365": 3.09266, "2370": 3.15197, "2375": 3.13019, "2380": 3.11082, "2385": 3.09359, "2390": 3.09567, "2395": 3.09807, "2400": 3.09966, "2405": 3.10436, "2410": 3.09007, "2415": 3.09491, "2420": 3.08537, "2425": 3.07877, "2430": 3.08079, "2435": 3.06761, "2440": 3.08574, "2445": 3.05747, "2450": 3.12167, "2455": 3.15832, "2460": 3.08596, "2465": 3.07656, "2470": 3.03663, "2475": 3.06421, "2480": 3.10252, "2485": 3.06485, "2490": 3.06573, "2495": 3.08845, "2500": 3.05671, "2505": 3.105, "2510": 3.12399, "2515": 3.0532, "2520": 3.07806, "2525": 3.02426, "2530": 3.04842, "2535": 3.09401, "2540": 3.07984, "2545": 3.05538, "2550": 3.00469, "2555": 3.07001, "2560": 3.04403, "2565": 3.12, "2570": 3.00976, "2575": 3.0601, "2580": 3.08548, "2585": 3.02156, "2590": 3.06606, "2595": 2.99925, "2600": 3.0841, "2605": 3.06879, "2610": 3.05401, "2615": 3.06935, "2620": 2.99191, "2625": 3.01384, "2630": 3.03627, "2635": 3.05041, "2640": 3.01088, "2645": 3.05612, "2650": 3.02233, "2655": 2.98756, "2660": 3.01604, "2665": 3.03817, "2670": 2.98547, "2675": 2.97442, "2680": 3.00378, "2685": 3.00171, "2690": 2.99912, "2695": 2.99265, "2700": 3.03079, "2705": 2.98376, "2710": 2.97975, "2715": 2.96047, "2720": 3.02663, "2725": 2.99565, "2730": 3.05827, "2735": 3.04913, "2740": 3.02027, "2745": 3.02502, "2750": 3.02065, "2755": 2.96792, "2760": 2.99447, "2765": 3.00785, "2770": 2.98958, "2775": 2.99278, "2780": 3.02294, "2785": 2.95383, "2790": 2.96474, "2795": 2.95595, "2800": 2.98985, "2805": 2.94051, "2810": 2.99046, "2815": 2.95976, "2820": 3.0756, "2825": 3.03639, "2830": 3.01855, "2835": 2.92175, "2840": 2.92574, "2845": 2.96102, "2850": 2.96997, "2855": 2.96207, "2860": 2.94977, "2865": 2.91535, "2870": 2.99202, "2875": 2.92084, "2880": 2.96303, "2885": 2.91779, "2890": 2.98572, "2895": 2.93253, "2900": 2.95289, "2905": 3.00499, "2910": 2.92994, "2915": 2.94325, "2920": 2.95516, "2925": 2.94427, "2930": 2.95621, "2935": 2.94005, "2940": 2.94552, "2945": 2.9075, "2950": 2.97913, "2955": 2.91177, "2960": 2.97029, "2965": 2.87292, "2970": 2.96107, "2975": 2.99603, "2980": 2.94257, "2985": 3.04155, "2990": 2.93897, "2995": 2.87114, "3000": 2.9422, "3005": 2.89655, "3010": 2.93538, "3015": 2.91032, "3020": 2.91995, "3025": 2.91883, "3030": 2.92686, "3035": 2.95815, "3040": 2.9312, "3045": 2.83504, "3050": 2.8988, "3055": 2.89613, "3060": 2.92461, "3065": 2.92459, "3070": 2.88159, "3075": 2.86953, "3080": 2.9243, "3085": 2.90325, "3090": 2.91754, "3095": 2.92816, "3100": 2.86703, "3105": 2.92918, "3110": 2.90236, "3115": 2.94681, "3120": 2.95312, "3125": 2.86217, "3130": 2.93048, "3135": 2.92489, "3140": 2.87699, "3145": 2.91715, "3150": 2.85701, "3155": 2.8442, "3160": 2.83887, "3165": 2.84564, "3170": 2.89213, "3175": 2.90452, "3180": 2.85788, "3185": 2.89571, "3190": 2.90627, "3195": 2.92723, "3200": 2.92789, "3205": 2.85912, "3210": 2.86987, "3215": 2.91563, "3220": 2.87374, "3225": 2.86935, "3230": 2.815, "3235": 2.87434, "3240": 2.8734, "3245": 2.90299, "3250": 2.86289, "3255": 2.8503, "3260": 2.85959, "3265": 2.86936, "3270": 2.85223, "3275": 2.86681, "3280": 2.79974, "3285": 2.81123, "3290": 2.86928, "3295": 2.92038, "3300": 2.87938, "3305": 2.86113, "3310": 2.85785, "3315": 2.80615, "3320": 2.8258, "3325": 2.82403, "3330": 2.82839, "3335": 2.8465, "3340": 2.82742, "3345": 2.84707, "3350": 2.84121, "3355": 2.85933, "3360": 2.79899, "3365": 2.85514, "3370": 2.84537, "3375": 2.84431, "3380": 2.84971, "3385": 2.87822, "3390": 2.8682, "3395": 2.81029, "3400": 2.78472, "3405": 2.82816, "3410": 2.84591, "3415": 2.86134, "3420": 2.82742, "3425": 2.81129, "3430": 2.82982, "3435": 2.8906, "3440": 2.81795, "3445": 2.86973, "3450": 2.81507, "3455": 2.7888, "3460": 2.8203, "3465": 2.84691, "3470": 2.83475, "3475": 2.7758, "3480": 2.84173, "3485": 2.82043, "3490": 2.8933, "3495": 2.84901, "3500": 2.84084, "3505": 2.82498, "3510": 2.81285, "3515": 2.83554, "3520": 2.77982, "3525": 2.80208, "3530": 2.84998, "3535": 2.78436, "3540": 2.83771, "3545": 2.81048, "3550": 2.79684, "3555": 2.8181, "3560": 2.82828, "3565": 2.82912, "3570": 2.80393, "3575": 2.80372, "3580": 2.82073, "3585": 2.83541, "3590": 2.8298, "3595": 2.77921, "3600": 2.74897, "3605": 2.79083, "3610": 2.8488, "3615": 2.75492, "3620": 2.80351, "3625": 2.88338, "3630": 2.77314, "3635": 2.78625, "3640": 2.78253, "3645": 2.76926, "3650": 2.80301, "3655": 2.81469, "3660": 2.76524, "3665": 2.7858, "3670": 2.77457, "3675": 2.77711, "3680": 2.80733, "3685": 2.80194, "3690": 2.8055, "3695": 2.81135, "3700": 2.78752, "3705": 2.78408, "3710": 2.75166, "3715": 2.80201, "3720": 2.79409, "3725": 2.78884, "3730": 2.84096, "3735": 2.80547, "3740": 2.74952, "3745": 2.78986, "3750": 2.8022, "3755": 2.79576, "3760": 2.75757, "3765": 2.75586, "3770": 2.75989, "3775": 2.76991, "3780": 2.76383, "3785": 2.7793, "3790": 2.74202, "3795": 2.79511, "3800": 2.80269, "3805": 2.75159, "3810": 2.80354, "3815": 2.76482, "3820": 2.78758, "3825": 2.73331, "3830": 2.74563, "3835": 2.81464, "3840": 2.72812, "3845": 2.71424, "3850": 2.77453, "3855": 2.71774, "3860": 2.80173, "3865": 2.75469, "3870": 2.77531, "3875": 2.75779, "3880": 2.78968, "3885": 2.78424, "3890": 2.74541, "3895": 2.79804, "3900": 2.76127, "3905": 2.72353, "3910": 2.74147, "3915": 2.75183, "3920": 2.79462, "3925": 2.77792, "3930": 2.70759, "3935": 2.73982, "3940": 2.75131, "3945": 2.74267, "3950": 2.725, "3955": 2.77958, "3960": 2.75991, "3965": 2.74216, "3970": 2.75653, "3975": 2.72552, "3980": 2.73817, "3985": 2.75045, "3990": 2.69347, "3995": 2.78059, "4000": 2.73558, "4005": 2.7658, "4010": 2.70885, "4015": 2.72538, "4020": 2.74956, "4025": 2.733, "4030": 2.65924, "4035": 2.69455, "4040": 2.74652, "4045": 2.74857, "4050": 2.78817, "4055": 2.7239, "4060": 2.71419, "4065": 2.6515, "4070": 2.80691, "4075": 2.75748, "4080": 2.71884, "4085": 2.74977, "4090": 2.67836, "4095": 2.69073, "4100": 2.7114, "4105": 2.73822, "4110": 2.72956, "4115": 2.70127, "4120": 2.73267, "4125": 2.70389, "4130": 2.69553, "4135": 2.6893, "4140": 2.68057, "4145": 2.77973, "4150": 2.70801, "4155": 2.73792, "4160": 2.76329, "4165": 2.72099, "4170": 2.67438, "4175": 2.71828, "4180": 2.72666, "4185": 2.72916, "4190": 2.73806, "4195": 2.70222, "4200": 2.71066, "4205": 2.73922, "4210": 2.67309, "4215": 2.66565, "4220": 2.65928, "4225": 2.70242, "4230": 2.71409, "4235": 2.7326, "4240": 2.70413, "4245": 2.69658, "4250": 2.71363, "4255": 2.64837, "4260": 2.7266, "4265": 2.73863, "4270": 2.72157, "4275": 2.68943, "4280": 2.70233, "4285": 2.73114, "4290": 2.68765, "4295": 2.69223, "4300": 2.69956, "4305": 2.70313, "4310": 2.73003, "4315": 2.71191, "4320": 2.69906, "4325": 2.70557, "4330": 2.7106, "4335": 2.69172, "4340": 2.6976, "4345": 2.72675, "4350": 2.67431, "4355": 2.69349, "4360": 2.71041, "4365": 2.78314, "4370": 2.73369, "4375": 2.74431, "4380": 2.71504, "4385": 2.69901, "4390": 2.70191, "4395": 2.75058, "4400": 2.66501, "4405": 2.66954, "4410": 2.68278, "4415": 2.70442, "4420": 2.7077, "4425": 2.72158, "4430": 2.69277, "4435": 2.68035, "4440": 2.69127, "4445": 2.67896, "4450": 2.65272, "4455": 2.69044, "4460": 2.70302, "4465": 2.70631, "4470": 2.6731, "4475": 2.68761, "4480": 2.65626, "4485": 2.69968, "4490": 2.65279, "4495": 2.70894, "4500": 2.70235, "4505": 2.69674, "4510": 2.64893, "4515": 2.70162, "4520": 2.66802, "4525": 2.66816, "4530": 2.6736, "4535": 2.67113, "4540": 2.70729, "4545": 2.65603, "4550": 2.70177, "4555": 2.68009, "4560": 2.65895, "4565": 2.63898, "4570": 2.6402, "4575": 2.66692, "4580": 2.68858, "4585": 2.68157, "4590": 2.61727, "4595": 2.66336, "4600": 2.67638, "4605": 2.68094, "4610": 2.66704, "4615": 2.66234, "4620": 2.65727, "4625": 2.68721, "4630": 2.6742, "4635": 2.64708, "4640": 2.69575, "4645": 2.64774, "4650": 2.7018, "4655": 2.70661, "4660": 2.67838, "4665": 2.68918, "4670": 2.67556, "4675": 2.68422, "4680": 2.66596, "4685": 2.65737, "4690": 2.70912, "4695": 2.65528, "4700": 2.67174, "4705": 2.65146, "4710": 2.68366, "4715": 2.64964, "4720": 2.72485, "4725": 2.62902, "4730": 2.65213, "4735": 2.68926, "4740": 2.64614, "4745": 2.65553, "4750": 2.65737, "4755": 2.65793, "4760": 2.66632, "4765": 2.64489, "4770": 2.62202, "4775": 2.65194, "4780": 2.65519, "4785": 2.68655, "4790": 2.65201, "4795": 2.67305, "4800": 2.62427, "4805": 2.64099, "4810": 2.65942, "4815": 2.65033, "4820": 2.6668, "4825": 2.65019, "4830": 2.6151, "4835": 2.64832, "4840": 2.65513, "4845": 2.6348, "4850": 2.62482, "4855": 2.60337, "4860": 2.65757, "4865": 2.62536, "4870": 2.63972, "4875": 2.61897, "4880": 2.62576, "4885": 2.62632, "4890": 2.67912, "4895": 2.65961, "4900": 2.618, "4905": 2.61823, "4910": 2.63845, "4915": 2.61463, "4920": 2.65397, "4925": 2.64838, "4930": 2.57129, "4935": 2.65193, "4940": 2.63034, "4945": 2.63777, "4950": 2.62825, "4955": 2.61794, "4960": 2.61856, "4965": 2.65951, "4970": 2.6008, "4975": 2.65676, "4980": 2.62049, "4985": 2.63225, "4990": 2.65645, "4995": 2.58184, "5000": 2.6621, "5005": 2.6658, "5010": 2.68112, "5015": 2.63396, "5020": 2.64091, "5025": 2.68726, "5030": 2.64362, "5035": 2.61873, "5040": 2.62248, "5045": 2.60699, "5050": 2.62641, "5055": 2.65014, "5060": 2.64375, "5065": 2.68893, "5070": 2.60617, "5075": 2.61421, "5080": 2.61231, "5085": 2.60499, "5090": 2.59441, "5095": 2.65086, "5100": 2.64984, "5105": 2.61053, "5110": 2.66408, "5115": 2.62171, "5120": 2.67055, "5125": 2.6309, "5130": 2.615, "5135": 2.61462, "5140": 2.57424, "5145": 2.62966, "5150": 2.63646, "5155": 2.61887, "5160": 2.66278, "5165": 2.58409, "5170": 2.59136, "5175": 2.62185, "5180": 2.60659, "5185": 2.62099, "5190": 2.6266, "5195": 2.67047, "5200": 2.5968, "5205": 2.60868, "5210": 2.60701, "5215": 2.64792, "5220": 2.58826, "5225": 2.55166, "5230": 2.6359, "5235": 2.61417, "5240": 2.62802, "5245": 2.64006, "5250": 2.61297, "5255": 2.62612, "5260": 2.5619, "5265": 2.59802, "5270": 2.58865, "5275": 2.61781, "5280": 2.61032, "5285": 2.60442, "5290": 2.63245, "5295": 2.62071, "5300": 2.57979, "5305": 2.59834, "5310": 2.60591, "5315": 2.5881, "5320": 2.61539, "5325": 2.64615, "5330": 2.6015, "5335": 2.58439, "5340": 2.56291, "5345": 2.65819, "5350": 2.62526, "5355": 2.57953, "5360": 2.59528, "5365": 2.62373, "5370": 2.61518, "5375": 2.63002, "5380": 2.58083, "5385": 2.56502, "5390": 2.58666, "5395": 2.61597, "5400": 2.60909, "5405": 2.54774, "5410": 2.61298, "5415": 2.59619, "5420": 2.61443, "5425": 2.62678, "5430": 2.62674, "5435": 2.57707, "5440": 2.58734, "5445": 2.633, "5450": 2.6473, "5455": 2.61252, "5460": 2.59272, "5465": 2.60502, "5470": 2.60189, "5475": 2.62728, "5480": 2.58753, "5485": 2.59002, "5490": 2.57733, "5495": 2.57075, "5500": 2.56937, "5505": 2.61715, "5510": 2.62664, "5515": 2.58137, "5520": 2.55697, "5525": 2.5859, "5530": 2.66433, "5535": 2.62339, "5540": 2.57109, "5545": 2.59633, "5550": 2.54936, "5555": 2.57342, "5560": 2.56447, "5565": 2.60758, "5570": 2.65168, "5575": 2.63138, "5580": 2.57564, "5585": 2.59822, "5590": 2.56185, "5595": 2.58521, "5600": 2.55512, "5605": 2.59879, "5610": 2.58291, "5615": 2.58198, "5620": 2.58123, "5625": 2.55147, "5630": 2.57081, "5635": 2.63484, "5640": 2.59425, "5645": 2.56995, "5650": 2.58004, "5655": 2.54766, "5660": 2.55881, "5665": 2.58604, "5670": 2.56686, "5675": 2.60728, "5680": 2.52861, "5685": 2.56813, "5690": 2.6039, "5695": 2.55782, "5700": 2.59695, "5705": 2.596, "5710": 2.57921, "5715": 2.58424, "5720": 2.53643, "5725": 2.6038, "5730": 2.57366, "5735": 2.61087, "5740": 2.59519, "5745": 2.56, "5750": 2.54216, "5755": 2.55997, "5760": 2.62481, "5765": 2.56328, "5770": 2.5429, "5775": 2.58373, "5780": 2.57701, "5785": 2.53911, "5790": 2.56461, "5795": 2.60179, "5800": 2.54494, "5805": 2.53531, "5810": 2.55658, "5815": 2.52456, "5820": 2.59694, "5825": 2.50599, "5830": 2.49558, "5835": 2.59597, "5840": 2.53979, "5845": 2.5528, "5850": 2.61315, "5855": 2.5102, "5860": 2.56169, "5865": 2.51778, "5870": 2.57574, "5875": 2.60723, "5880": 2.58596, "5885": 2.56757, "5890": 2.58608, "5895": 2.55562, "5900": 2.61651, "5905": 2.55716, "5910": 2.59828, "5915": 2.61008, "5920": 2.58733, "5925": 2.55324, "5930": 2.57568, "5935": 2.55168, "5940": 2.57131, "5945": 2.5204, "5950": 2.55562, "5955": 2.586, "5960": 2.56741, "5965": 2.62046, "5970": 2.55594, "5975": 2.58503, "5980": 2.55843, "5985": 2.56032, "5990": 2.55653, "5995": 2.55873, "6000": 2.55658, "6005": 2.51961, "6010": 2.5612, "6015": 2.52607, "6020": 2.53453, "6025": 2.55768, "6030": 2.6046, "6035": 2.54228, "6040": 2.54868, "6045": 2.49077, "6050": 2.5963, "6055": 2.5204, "6060": 2.54409, "6065": 2.52518, "6070": 2.52918, "6075": 2.5364, "6080": 2.53607, "6085": 2.59714, "6090": 2.57034, "6095": 2.53592, "6100": 2.5428, "6105": 2.52487, "6110": 2.55483, "6115": 2.58495, "6120": 2.55695, "6125": 2.53683, "6130": 2.47322, "6135": 2.5563, "6140": 2.55589, "6145": 2.55739, "6150": 2.52565, "6155": 2.50872, "6160": 2.54299, "6165": 2.57304, "6170": 2.54638, "6175": 2.60079, "6180": 2.51196, "6185": 2.55194, "6190": 2.49345, "6195": 2.57854, "6200": 2.55164, "6205": 2.5377, "6210": 2.52088, "6215": 2.51358, "6220": 2.56539, "6225": 2.51406, "6230": 2.51072, "6235": 2.56268, "6240": 2.55115, "6245": 2.52327, "6250": 2.53069, "6255": 2.57365, "6260": 2.52537, "6265": 2.57441, "6270": 2.52397, "6275": 2.56565, "6280": 2.52297, "6285": 2.5207, "6290": 2.51982, "6295": 2.50722, "6300": 2.55559, "6305": 2.52486, "6310": 2.51259, "6315": 2.53731, "6320": 2.4894, "6325": 2.59818, "6330": 2.555, "6335": 2.51085, "6340": 2.51313, "6345": 2.55702, "6350": 2.556, "6355": 2.52448, "6360": 2.52293, "6365": 2.48409, "6370": 2.53563, "6375": 2.49779, "6380": 2.56282, "6385": 2.58189, "6390": 2.50441, "6395": 2.55121, "6400": 2.5086, "6405": 2.5278, "6410": 2.51466, "6415": 2.52482, "6420": 2.54258, "6425": 2.53509, "6430": 2.57978, "6435": 2.54444, "6440": 2.53907, "6445": 2.53125, "6450": 2.53474, "6455": 2.52399, "6460": 2.51849, "6465": 2.56225, "6470": 2.52104, "6475": 2.52654, "6480": 2.48826, "6485": 2.52861, "6490": 2.50978, "6495": 2.49978, "6500": 2.52402, "6505": 2.49432, "6510": 2.54199, "6515": 2.5101, "6520": 2.51003, "6525": 2.49503, "6530": 2.54392, "6535": 2.53282, "6540": 2.53291, "6545": 2.56194, "6550": 2.50127, "6555": 2.55627, "6560": 2.51016, "6565": 2.52281, "6570": 2.58445, "6575": 2.52324, "6580": 2.49815, "6585": 2.50823, "6590": 2.5097, "6595": 2.49807, "6600": 2.49539, "6605": 2.54253, "6610": 2.4797, "6615": 2.56766, "6620": 2.53402, "6625": 2.51202, "6630": 2.51431, "6635": 2.47464, "6640": 2.54106, "6645": 2.59681, "6650": 2.51024, "6655": 2.4983, "6660": 2.57419, "6665": 2.52156, "6670": 2.5674, "6675": 2.46861, "6680": 2.54697, "6685": 2.53564, "6690": 2.51427, "6695": 2.48573, "6700": 2.52463, "6705": 2.52218, "6710": 2.49347, "6715": 2.51687, "6720": 2.50996, "6725": 2.52089, "6730": 2.52013, "6735": 2.4825, "6740": 2.51535, "6745": 2.49672, "6750": 2.55754, "6755": 2.47484, "6760": 2.54212, "6765": 2.48878, "6770": 2.51847, "6775": 2.50828, "6780": 2.53878, "6785": 2.47177, "6790": 2.54553, "6795": 2.49868, "6800": 2.52671, "6805": 2.51099, "6810": 2.50296, "6815": 2.52064, "6820": 2.48696, "6825": 2.5071, "6830": 2.54063, "6835": 2.50678, "6840": 2.50885, "6845": 2.52492, "6850": 2.47583, "6855": 2.512, "6860": 2.50239, "6865": 2.49001, "6870": 2.55392, "6875": 2.47561, "6880": 2.55072, "6885": 2.47892, "6890": 2.54905, "6895": 2.50384, "6900": 2.49072, "6905": 2.51205, "6910": 2.5215, "6915": 2.51823, "6920": 2.5328, "6925": 2.54741, "6930": 2.49289, "6935": 2.521, "6940": 2.50604, "6945": 2.46237, "6950": 2.48628, "6955": 2.5288, "6960": 2.51952, "6965": 2.49196, "6970": 2.47065, "6975": 2.52409, "6980": 2.45258, "6985": 2.51631, "6990": 2.52932, "6995": 2.46179, "7000": 2.49172, "7005": 2.47011, "7010": 2.47632, "7015": 2.51983, "7020": 2.46705, "7025": 2.45424, "7030": 2.48487, "7035": 2.47988, "7040": 2.50783, "7045": 2.52359, "7050": 2.52831, "7055": 2.44161, "7060": 2.47409, "7065": 2.48138, "7070": 2.48981, "7075": 2.49452, "7080": 2.53479, "7085": 2.48717, "7090": 2.47618, "7095": 2.4999, "7100": 2.51585, "7105": 2.4884, "7110": 2.487, "7115": 2.50558, "7120": 2.47286, "7125": 2.46376, "7130": 2.48693, "7135": 2.51456, "7140": 2.50032, "7145": 2.49769, "7150": 2.51016, "7155": 2.50401, "7160": 2.47274, "7165": 2.45638, "7170": 2.50459, "7175": 2.50355, "7180": 2.50497, "7185": 2.48172, "7190": 2.46296, "7195": 2.46639, "7200": 2.50998, "7205": 2.49029, "7210": 2.44246, "7215": 2.47885, "7220": 2.4456, "7225": 2.51269, "7230": 2.50805, "7235": 2.48249, "7240": 2.47867, "7245": 2.50035, "7250": 2.50922, "7255": 2.49324, "7260": 2.46058, "7265": 2.45308, "7270": 2.47086, "7275": 2.49781, "7280": 2.49343, "7285": 2.42363, "7290": 2.47944, "7295": 2.48626, "7300": 2.41751, "7305": 2.44554, "7310": 2.44899, "7315": 2.48986, "7320": 2.48389, "7325": 2.45917, "7330": 2.4893, "7335": 2.47688, "7340": 2.46486, "7345": 2.49515, "7350": 2.5106, "7355": 2.49669, "7360": 2.48037, "7365": 2.46906, "7370": 2.47138, "7375": 2.4508, "7380": 2.49622, "7385": 2.48448, "7390": 2.47337, "7395": 2.47339, "7400": 2.48169, "7405": 2.43994, "7410": 2.48078, "7415": 2.47113, "7420": 2.49398, "7425": 2.45774, "7430": 2.52358, "7435": 2.49185, "7440": 2.52151, "7445": 2.5101, "7450": 2.4751, "7455": 2.45401, "7460": 2.46474, "7465": 2.47685, "7470": 2.44899, "7475": 2.45681, "7480": 2.51145, "7485": 2.45042, "7490": 2.47478, "7495": 2.48246, "7500": 2.49584, "7505": 2.44104, "7510": 2.43501, "7515": 2.41997, "7520": 2.49389, "7525": 2.49884, "7530": 2.47668, "7535": 2.4601, "7540": 2.47288, "7545": 2.47471, "7550": 2.49181, "7555": 2.45487, "7560": 2.42922, "7565": 2.51106, "7570": 2.4857, "7575": 2.439, "7580": 2.45825, "7585": 2.48256, "7590": 2.48193, "7595": 2.46508, "7600": 2.46362, "7605": 2.44863, "7610": 2.44948, "7615": 2.42526, "7620": 2.54441, "7625": 2.47879, "7630": 2.42526, "7635": 2.42739, "7640": 2.45364, "7645": 2.47151, "7650": 2.46303, "7655": 2.48304, "7660": 2.4532, "7665": 2.4342, "7670": 2.4426, "7675": 2.45588, "7680": 2.48517, "7685": 2.43208, "7690": 2.48, "7695": 2.45485, "7700": 2.48159, "7705": 2.49878, "7710": 2.49483, "7715": 2.44384, "7720": 2.4696, "7725": 2.47981, "7730": 2.45864, "7735": 2.47057, "7740": 2.43882, "7745": 2.45157, "7750": 2.43921, "7755": 2.46722, "7760": 2.45122, "7765": 2.45511, "7770": 2.47144, "7775": 2.45332, "7780": 2.41653, "7785": 2.44516, "7790": 2.48285, "7795": 2.44125, "7800": 2.46355, "7805": 2.48202, "7810": 2.50258, "7815": 2.48733, "7820": 2.44788, "7825": 2.51471, "7830": 2.45477, "7835": 2.4697, "7840": 2.47907, "7845": 2.46064, "7850": 2.41717, "7855": 2.47244, "7860": 2.49887, "7865": 2.42434, "7870": 2.46693, "7875": 2.44544, "7880": 2.45287, "7885": 2.46023, "7890": 2.47026, "7895": 2.44872, "7900": 2.4404, "7905": 2.43773, "7910": 2.42565, "7915": 2.48107, "7920": 2.47699, "7925": 2.4218, "7930": 2.47199, "7935": 2.44975, "7940": 2.42126, "7945": 2.46977, "7950": 2.44424, "7955": 2.4204, "7960": 2.49038, "7965": 2.5188, "7970": 2.52207, "7975": 2.44798, "7980": 2.44076, "7985": 2.46872, "7990": 2.43169, "7995": 2.46954, "8000": 2.43641, "8005": 2.41891, "8010": 2.45749, "8015": 2.46841, "8020": 2.48116, "8025": 2.47363, "8030": 2.45173, "8035": 2.47071, "8040": 2.41983, "8045": 2.45333, "8050": 2.44721, "8055": 2.42302, "8060": 2.44253, "8065": 2.46158, "8070": 2.4567, "8075": 2.46077, "8080": 2.44618, "8085": 2.44085, "8090": 2.42787, "8095": 2.42397, "8100": 2.43904, "8105": 2.49479, "8110": 2.43878, "8115": 2.58899, "8120": 2.49362, "8125": 2.47876, "8130": 2.45879, "8135": 2.4574, "8140": 2.44166, "8145": 2.42774, "8150": 2.42089, "8155": 2.48312, "8160": 2.45131, "8165": 2.43947, "8170": 2.43326, "8175": 2.42092, "8180": 2.4946, "8185": 2.42477, "8190": 2.46908, "8195": 2.45732, "8200": 2.44651, "8205": 2.44406, "8210": 2.43096, "8215": 2.44122, "8220": 2.43556, "8225": 2.41067, "8230": 2.44055, "8235": 2.46438, "8240": 2.42694, "8245": 2.44767, "8250": 2.44524, "8255": 2.43772, "8260": 2.43153, "8265": 2.42903, "8270": 2.4363, "8275": 2.44197, "8280": 2.39831, "8285": 2.4405, "8290": 2.48021, "8295": 2.44762, "8300": 2.45931, "8305": 2.40847, "8310": 2.43461, "8315": 2.45616, "8320": 2.40422, "8325": 2.39725, "8330": 2.43986, "8335": 2.44684, "8340": 2.49212, "8345": 2.44942, "8350": 2.45049, "8355": 2.40704, "8360": 2.40131, "8365": 2.45443, "8370": 2.45427, "8375": 2.42518, "8380": 2.41939, "8385": 2.42541, "8390": 2.4387, "8395": 2.44193, "8400": 2.44114, "8405": 2.49132, "8410": 2.4383, "8415": 2.43519, "8420": 2.41861, "8425": 2.44324, "8430": 2.46253, "8435": 2.40559, "8440": 2.45227, "8445": 2.45999, "8450": 2.40867, "8455": 2.46028, "8460": 2.45495, "8465": 2.43629, "8470": 2.40854, "8475": 2.47887, "8480": 2.40222, "8485": 2.41392, "8490": 2.46612, "8495": 2.43613, "8500": 2.44492, "8505": 2.40329, "8510": 2.40218, "8515": 2.42871, "8520": 2.42574, "8525": 2.49152, "8530": 2.3746, "8535": 2.40109, "8540": 2.48679, "8545": 2.3811, "8550": 2.43875, "8555": 2.4514, "8560": 2.47019, "8565": 2.42055, "8570": 2.43185, "8575": 2.44959, "8580": 2.44124, "8585": 2.42059, "8590": 2.4038, "8595": 2.42895, "8600": 2.41116, "8605": 2.49131, "8610": 2.42052, "8615": 2.38808, "8620": 2.45039, "8625": 2.42523, "8630": 2.45471, "8635": 2.4509, "8640": 2.43534, "8645": 2.47406, "8650": 2.42305, "8655": 2.45293, "8660": 2.45576, "8665": 2.38622, "8670": 2.41139, "8675": 2.42943, "8680": 2.44841, "8685": 2.43079, "8690": 2.41017, "8695": 2.44311, "8700": 2.43428, "8705": 2.42016, "8710": 2.42854, "8715": 2.44862, "8720": 2.47696, "8725": 2.41012, "8730": 2.39278, "8735": 2.43505, "8740": 2.43198, "8745": 2.39801, "8750": 2.43609, "8755": 2.42381, "8760": 2.40031, "8765": 2.43541, "8770": 2.40569, "8775": 2.43812, "8780": 2.42153, "8785": 2.47144, "8790": 2.42041, "8795": 2.41876, "8800": 2.41592, "8805": 2.40548, "8810": 2.41139, "8815": 2.47509, "8820": 2.45362, "8825": 2.4241, "8830": 2.38744, "8835": 2.42258, "8840": 2.39347, "8845": 2.42679, "8850": 2.43485, "8855": 2.4044, "8860": 2.42715, "8865": 2.42631, "8870": 2.43391, "8875": 2.44152, "8880": 2.41099, "8885": 2.39514, "8890": 2.44614, "8895": 2.42902, "8900": 2.41354, "8905": 2.40085, "8910": 2.4019, "8915": 2.4163, "8920": 2.43454, "8925": 2.46713, "8930": 2.41511, "8935": 2.40784, "8940": 2.38869, "8945": 2.39353, "8950": 2.41789, "8955": 2.39534, "8960": 2.43426, "8965": 2.41798, "8970": 2.40536, "8975": 2.47767, "8980": 2.44109, "8985": 2.37482, "8990": 2.41061, "8995": 2.416, "9000": 2.45568, "9005": 2.41279, "9010": 2.37662, "9015": 2.41141, "9020": 2.40089, "9025": 2.3701, "9030": 2.40026, "9035": 2.4243, "9040": 2.42079, "9045": 2.41805, "9050": 2.39505, "9055": 2.41785, "9060": 2.41922, "9065": 2.40527, "9070": 2.44454, "9075": 2.39395, "9080": 2.43398, "9085": 2.4136, "9090": 2.41293, "9095": 2.39793, "9100": 2.40135, "9105": 2.35782, "9110": 2.46451, "9115": 2.41499, "9120": 2.40368, "9125": 2.45804, "9130": 2.39387, "9135": 2.44878, "9140": 2.43562, "9145": 2.42684, "9150": 2.42505, "9155": 2.3752, "9160": 2.41724, "9165": 2.42569, "9170": 2.37359, "9175": 2.41857, "9180": 2.37803, "9185": 2.43942, "9190": 2.41281, "9195": 2.40662, "9200": 2.39186, "9205": 2.44999, "9210": 2.36248, "9215": 2.46363, "9220": 2.44779, "9225": 2.3828, "9230": 2.44575, "9235": 2.39772, "9240": 2.40182, "9245": 2.43796, "9250": 2.43806, "9255": 2.4326, "9260": 2.38813, "9265": 2.43977, "9270": 2.43657, "9275": 2.39535, "9280": 2.39074, "9285": 2.42225, "9290": 2.40437, "9295": 2.38603, "9300": 2.42495, "9305": 2.40579, "9310": 2.41555, "9315": 2.41153, "9320": 2.44493, "9325": 2.37049, "9330": 2.40434, "9335": 2.36191, "9340": 2.40835, "9345": 2.41458, "9350": 2.44039, "9355": 2.47763, "9360": 2.43745, "9365": 2.38821, "9370": 2.43648, "9375": 2.43331, "9380": 2.35346, "9385": 2.39958, "9390": 2.38109, "9395": 2.38731, "9400": 2.44471, "9405": 2.41259, "9410": 2.39756, "9415": 2.43759, "9420": 2.4441, "9425": 2.43656, "9430": 2.45071, "9435": 2.41453, "9440": 2.47761, "9445": 2.37622, "9450": 2.39383, "9455": 2.40249, "9460": 2.38597, "9465": 2.3775, "9470": 2.38205, "9475": 2.36454, "9480": 2.43551, "9485": 2.38642, "9490": 2.4204, "9495": 2.38165, "9500": 2.36325, "9505": 2.4296, "9510": 2.39916, "9515": 2.43096, "9520": 2.41792, "9525": 2.38898, "9530": 2.45385, "9535": 2.40151, "9540": 2.41839, "9545": 2.37813, "9550": 2.42143, "9555": 2.39054, "9560": 2.42191, "9565": 2.40523, "9570": 2.37157, "9575": 2.41109, "9580": 2.39564, "9585": 2.42353, "9590": 2.42924, "9595": 2.44777, "9600": 2.39117, "9605": 2.38431, "9610": 2.42142, "9615": 2.41558, "9620": 2.41413, "9625": 2.44723, "9630": 2.39712, "9635": 2.40396, "9640": 2.44817, "9645": 2.4109, "9650": 2.39894, "9655": 2.37366, "9660": 2.42329, "9665": 2.39029, "9670": 2.38274, "9675": 2.35662, "9680": 2.39869, "9685": 2.40199, "9690": 2.46804, "9695": 2.38133, "9700": 2.37698, "9705": 2.38453, "9710": 2.36554, "9715": 2.38868, "9720": 2.43552, "9725": 2.4413, "9730": 2.42919, "9735": 2.38684, "9740": 2.38077, "9745": 2.42676, "9750": 2.3991, "9755": 2.40788, "9760": 2.41084, "9765": 2.37036, "9770": 2.43675, "9775": 2.40145, "9780": 2.36196, "9785": 2.40085, "9790": 2.40714, "9795": 2.3593, "9800": 2.39629, "9805": 2.40561, "9810": 2.41066, "9815": 2.37884, "9820": 2.37671, "9825": 2.40364, "9830": 2.42194, "9835": 2.3861, "9840": 2.41457, "9845": 2.36502, "9850": 2.39824, "9855": 2.39496, "9860": 2.3972, "9865": 2.38197, "9870": 2.39342, "9875": 2.38398, "9880": 2.45319, "9885": 2.39313, "9890": 2.35399, "9895": 2.32116, "9900": 2.3962, "9905": 2.42494, "9910": 2.35642, "9915": 2.36473, "9920": 2.41154, "9925": 2.39863, "9930": 2.38182, "9935": 2.35063, "9940": 2.38377, "9945": 2.37842, "9950": 2.40342, "9955": 2.44928, "9960": 2.43108, "9965": 2.35851, "9970": 2.41017, "9975": 2.38564, "9980": 2.33084, "9985": 2.40772, "9990": 2.39761, "9995": 2.39543, "10000": 2.36621, "10005": 2.37213, "10010": 2.38256, "10015": 2.44495, "10020": 2.36326, "10025": 2.38851, "10030": 2.38817, "10035": 2.40993, "10040": 2.40515, "10045": 2.3831, "10050": 2.34965, "10055": 2.36805, "10060": 2.42146, "10065": 2.37528, "10070": 2.42235, "10075": 2.37088, "10080": 2.36211, "10085": 2.36918, "10090": 2.34573, "10095": 2.40221, "10100": 2.31408, "10105": 2.38253, "10110": 2.40897, "10115": 2.38736, "10120": 2.35801, "10125": 2.37033, "10130": 2.36037, "10135": 2.38382, "10140": 2.4139, "10145": 2.40714, "10150": 2.37532, "10155": 2.39536, "10160": 2.36205, "10165": 2.38369, "10170": 2.4236, "10175": 2.32447, "10180": 2.39651, "10185": 2.3824, "10190": 2.44396, "10195": 2.40416, "10200": 2.38955, "10205": 2.38797, "10210": 2.36805, "10215": 2.34261, "10220": 2.41843, "10225": 2.43079, "10230": 2.35627, "10235": 2.38764, "10240": 2.37226, "10245": 2.39117, "10250": 2.38838, "10255": 2.41316, "10260": 2.33469, "10265": 2.34846, "10270": 2.34979, "10275": 2.3717, "10280": 2.4513, "10285": 2.35906, "10290": 2.3861, "10295": 2.375, "10300": 2.36936, "10305": 2.41578, "10310": 2.38877, "10315": 2.36095, "10320": 2.36607, "10325": 2.36094, "10330": 2.41247, "10335": 2.36135, "10340": 2.41934, "10345": 2.36966, "10350": 2.35686, "10355": 2.39609, "10360": 2.37338, "10365": 2.36225, "10370": 2.34061, "10375": 2.3585, "10380": 2.41953, "10385": 2.40576, "10390": 2.38058, "10395": 2.35968, "10400": 2.37919, "10405": 2.34877, "10410": 2.3389, "10415": 2.41664, "10420": 2.37924, "10425": 2.32522, "10430": 2.35941, "10435": 2.37129, "10440": 2.3711, "10445": 2.35949, "10450": 2.36154, "10455": 2.38113, "10460": 2.38064, "10465": 2.30273, "10470": 2.3577, "10475": 2.37958, "10480": 2.36276, "10485": 2.36137, "10490": 2.41283, "10495": 2.36502, "10500": 2.36277, "10505": 2.37018, "10510": 2.38172, "10515": 2.37393, "10520": 2.40259, "10525": 2.39024, "10530": 2.39211, "10535": 2.35551, "10540": 2.40461, "10545": 2.35856, "10550": 2.37752, "10555": 2.35793, "10560": 2.34025, "10565": 2.37346, "10570": 2.37536, "10575": 2.3535, "10580": 2.37788, "10585": 2.36682, "10590": 2.37817, "10595": 2.37713, "10600": 2.33146, "10605": 2.3724, "10610": 2.36498, "10615": 2.36379, "10620": 2.34659, "10625": 2.41843, "10630": 2.36855, "10635": 2.32266, "10640": 2.36413, "10645": 2.42158, "10650": 2.36174, "10655": 2.30869, "10660": 2.34689, "10665": 2.39981, "10670": 2.31617, "10675": 2.41612, "10680": 2.35445, "10685": 2.28871, "10690": 2.38456, "10695": 2.33038, "10700": 2.38407, "10705": 2.38432, "10710": 2.34313, "10715": 2.3828, "10720": 2.32518, "10725": 2.35278, "10730": 2.34872, "10735": 2.35338, "10740": 2.31849, "10745": 2.33808, "10750": 2.33362, "10755": 2.4041, "10760": 2.36431, "10765": 2.33591, "10770": 2.36802, "10775": 2.38746, "10780": 2.36985, "10785": 2.39167, "10790": 2.34599, "10795": 2.38556, "10800": 2.32491, "10805": 2.39755, "10810": 2.37536, "10815": 2.35431, "10820": 2.34323, "10825": 2.37192, "10830": 2.33781, "10835": 2.3477, "10840": 2.32993, "10845": 2.38645, "10850": 2.33282, "10855": 2.36654, "10860": 2.33304, "10865": 2.32192, "10870": 2.32311, "10875": 2.30406, "10880": 2.39356, "10885": 2.40455, "10890": 2.36115, "10895": 2.37301, "10900": 2.33176, "10905": 2.31266, "10910": 2.40728, "10915": 2.37119, "10920": 2.37413, "10925": 2.36306, "10930": 2.31881, "10935": 2.36035, "10940": 2.35501, "10945": 2.34689, "10950": 2.36286, "10955": 2.3644, "10960": 2.30987, "10965": 2.3635, "10970": 2.35624, "10975": 2.40775, "10980": 2.37303, "10985": 2.3427, "10990": 2.39729, "10995": 2.36387, "11000": 2.33714, "11005": 2.36117, "11010": 2.34243, "11015": 2.32557, "11020": 2.3346, "11025": 2.36577, "11030": 2.34044, "11035": 2.31307, "11040": 2.31887, "11045": 2.31738, "11050": 2.31805, "11055": 2.28859, "11060": 2.33998, "11065": 2.31013, "11070": 2.39402, "11075": 2.32015, "11080": 2.35427, "11085": 2.33669, "11090": 2.34632, "11095": 2.37084, "11100": 2.32912, "11105": 2.31663, "11110": 2.36288, "11115": 2.37225, "11120": 2.38139, "11125": 2.31341, "11130": 2.34997, "11135": 2.3336, "11140": 2.37217, "11145": 2.35107, "11150": 2.39612, "11155": 2.34114, "11160": 2.3659, "11165": 2.36388, "11170": 2.34098, "11175": 2.33474, "11180": 2.37348, "11185": 2.31203, "11190": 2.27804, "11195": 2.32819, "11200": 2.34726, "11205": 2.36258, "11210": 2.33385, "11215": 2.31927, "11220": 2.34329, "11225": 2.37141, "11230": 2.36569, "11235": 2.32069, "11240": 2.34092, "11245": 2.35748, "11250": 2.3324, "11255": 2.33515, "11260": 2.35577, "11265": 2.38918, "11270": 2.28782, "11275": 2.31519, "11280": 2.36893, "11285": 2.29387, "11290": 2.34639, "11295": 2.3655, "11300": 2.38111, "11305": 2.33495, "11310": 2.32963, "11315": 2.29825, "11320": 2.30482, "11325": 2.31462, "11330": 2.35421, "11335": 2.33831, "11340": 2.30841, "11345": 2.31278, "11350": 2.29588, "11355": 2.3219, "11360": 2.35153, "11365": 2.29378, "11370": 2.35263, "11375": 2.32804, "11380": 2.34006, "11385": 2.34763, "11390": 2.33477, "11395": 2.28732, "11400": 2.30981, "11405": 2.35647, "11410": 2.35502, "11415": 2.38458, "11420": 2.35172, "11425": 2.30761, "11430": 2.36718, "11435": 2.36201, "11440": 2.34796, "11445": 2.36318, "11450": 2.32182, "11455": 2.30476, "11460": 2.35092, "11465": 2.34386, "11470": 2.37434, "11475": 2.31342, "11480": 2.32527, "11485": 2.30987, "11490": 2.34568, "11495": 2.406, "11500": 2.33937, "11505": 2.35014, "11510": 2.36223, "11515": 2.32176, "11520": 2.30507, "11525": 2.36152, "11530": 2.31469, "11535": 2.32196, "11540": 2.34627, "11545": 2.34321, "11550": 2.36438, "11555": 2.32533, "11560": 2.34981, "11565": 2.34125, "11570": 2.34916, "11575": 2.29628, "11580": 2.32931, "11585": 2.35173, "11590": 2.36158, "11595": 2.33454, "11600": 2.35704, "11605": 2.3235, "11610": 2.36089, "11615": 2.35899, "11620": 2.29569, "11625": 2.2757, "11630": 2.32782, "11635": 2.34204, "11640": 2.30488, "11645": 2.30751, "11650": 2.32628, "11655": 2.35114, "11660": 2.33566, "11665": 2.32994, "11670": 2.30002, "11675": 2.29666, "11680": 2.32542, "11685": 2.33637, "11690": 2.34433, "11695": 2.31688, "11700": 2.32535, "11705": 2.3009, "11710": 2.34479, "11715": 2.31575, "11720": 2.29844, "11725": 2.33988, "11730": 2.30403, "11735": 2.32822, "11740": 2.27122, "11745": 2.31714, "11750": 2.32793, "11755": 2.35133, "11760": 2.31357, "11765": 2.3378, "11770": 2.27597, "11775": 2.32591, "11780": 2.25511, "11785": 2.2973, "11790": 2.31403, "11795": 2.32024, "11800": 2.3345, "11805": 2.30403, "11810": 2.30398, "11815": 2.33078, "11820": 2.32015, "11825": 2.36083, "11830": 2.31663, "11835": 2.33741, "11840": 2.34081, "11845": 2.31727, "11850": 2.30496, "11855": 2.31403, "11860": 2.34333, "11865": 2.35836, "11870": 2.37861, "11875": 2.28155, "11880": 2.29163, "11885": 2.33553, "11890": 2.29241, "11895": 2.29059, "11900": 2.33401, "11905": 2.31769, "11910": 2.27783, "11915": 2.31082, "11920": 2.33519, "11925": 2.30272, "11930": 2.30681, "11935": 2.31569, "11940": 2.3175, "11945": 2.34208, "11950": 2.29773, "11955": 2.31327, "11960": 2.33576, "11965": 2.29584, "11970": 2.28204, "11975": 2.33575, "11980": 2.30612, "11985": 2.2776, "11990": 2.30416, "11995": 2.33013, "12000": 2.32323, "12005": 2.32565, "12010": 2.2884, "12015": 2.30861, "12020": 2.32922, "12025": 2.33525, "12030": 2.31246, "12035": 2.33617, "12040": 2.3154, "12045": 2.3126, "12050": 2.30835, "12055": 2.33352, "12060": 2.29764, "12065": 2.32975, "12070": 2.30319, "12075": 2.2775, "12080": 2.35063, "12085": 2.33812, "12090": 2.33359, "12095": 2.28176, "12100": 2.31543, "12105": 2.30903, "12110": 2.33029, "12115": 2.3036, "12120": 2.30606, "12125": 2.29484, "12130": 2.30409, "12135": 2.32842, "12140": 2.29591, "12145": 2.25622, "12150": 2.26125, "12155": 2.34249, "12160": 2.35771, "12165": 2.31914, "12170": 2.3336, "12175": 2.3412, "12180": 2.33054, "12185": 2.34135, "12190": 2.33375, "12195": 2.29767, "12200": 2.30036, "12205": 2.32225, "12210": 2.35697, "12215": 2.30437, "12220": 2.2987, "12225": 2.24241, "12230": 2.33348, "12235": 2.33945, "12240": 2.32345, "12245": 2.28764, "12250": 2.27397, "12255": 2.33706, "12260": 2.31368, "12265": 2.34287, "12270": 2.31292, "12275": 2.31361, "12280": 2.31869, "12285": 2.28631, "12290": 2.31074, "12295": 2.26654, "12300": 2.32931, "12305": 2.26821, "12310": 2.28768, "12315": 2.3543, "12320": 2.2963, "12325": 2.32045, "12330": 2.30113, "12335": 2.3194, "12340": 2.34117, "12345": 2.36885, "12350": 2.34318, "12355": 2.30683, "12360": 2.31344, "12365": 2.32933, "12370": 2.29273, "12375": 2.29957, "12380": 2.29184, "12385": 2.29061, "12390": 2.25018, "12395": 2.30421, "12400": 2.29905, "12405": 2.31088, "12410": 2.30419, "12415": 2.28306, "12420": 2.31729, "12425": 2.30099, "12430": 2.31571, "12435": 2.30048, "12440": 2.33123, "12445": 2.3202, "12450": 2.30745, "12455": 2.24018, "12460": 2.33488, "12465": 2.36363, "12470": 2.27626, "12475": 2.27276, "12480": 2.29139, "12485": 2.30632, "12490": 2.33128, "12495": 2.26961, "12500": 2.32122, "12505": 2.3351, "12510": 2.35582, "12515": 2.27062, "12520": 2.31971, "12525": 2.28653, "12530": 2.32054, "12535": 2.27138, "12540": 2.28491, "12545": 2.29049, "12550": 2.31572, "12555": 2.32333, "12560": 2.30023, "12565": 2.3353, "12570": 2.27829, "12575": 2.29941, "12580": 2.31153, "12585": 2.29201, "12590": 2.33455, "12595": 2.3227, "12600": 2.28167, "12605": 2.31996, "12610": 2.3631, "12615": 2.30567, "12620": 2.33322, "12625": 2.32935, "12630": 2.29885, "12635": 2.33561, "12640": 2.29568, "12645": 2.27902, "12650": 2.32556, "12655": 2.2647, "12660": 2.34199, "12665": 2.31843, "12670": 2.3097, "12675": 2.31886, "12680": 2.27525, "12685": 2.3664, "12690": 2.30452, "12695": 2.33199, "12700": 2.29244, "12705": 2.30628, "12710": 2.30837, "12715": 2.28749, "12720": "nan", "12725": "nan", "12730": "nan", "12735": "nan", "12740": "nan", "12745": "nan", "12750": "nan", "12755": "nan", "12760": "nan", "12765": "nan", "12770": "nan", "12775": "nan", "12780": "nan", "12785": "nan", "12790": "nan", "12795": "nan", "12800": "nan", "12805": "nan", "12810": "nan", "12815": "nan", "12820": "nan", "12825": "nan", "12830": "nan", "12835": "nan", "12840": "nan", "12845": "nan", "12850": "nan", "12855": "nan", "12860": "nan", "12865": "nan", "12870": "nan", "12875": "nan", "12880": "nan", "12885": "nan", "12890": "nan", "12895": "nan", "12900": "nan", "12905": "nan", "12910": "nan", "12915": "nan", "12920": "nan", "12925": "nan", "12930": "nan", "12935": "nan", "12940": "nan", "12945": "nan", "12950": "nan", "12955": "nan", "12960": "nan", "12965": "nan", "12970": "nan", "12975": "nan", "12980": "nan", "12985": "nan", "12990": "nan", "12995": "nan", "13000": "nan"}}, "num-zeros": {"start_step": 1, "end_step": 13000, "step_interval": 5, "values": {"1": 956236544.0, "5": 967337600.0, "10": 971388224.0, "15": 946439424.0, "20": 961330240.0, "25": 1083876480.0, "30": 1211133312.0, "35": 1297707520.0, "40": 1271785728.0, "45": 1175048064.0, "50": 1126729728.0, "55": 1083975424.0, "60": 1045060608.0, "65": 1026047360.0, "70": 995721280.0, "75": 986257152.0, "80": 1010241664.0, "85": 1006739968.0, "90": 988780736.0, "95": 959700032.0, "100": 971861632.0, "105": 980754624.0, "110": 977222528.0, "115": 978430848.0, "120": 961162432.0, "125": 942469184.0, "130": 977095104.0, "135": 966160128.0, "140": 963476928.0, "145": 976512384.0, "150": 921597184.0, "155": 968134336.0, "160": 956383232.0, "165": 959869952.0, "170": 974372224.0, "175": 949013120.0, "180": 946688448.0, "185": 972006784.0, "190": 969055488.0, "195": 985121664.0, "200": 945774592.0, "205": 958353792.0, "210": 979445248.0, "215": 967478208.0, "220": 956423424.0, "225": 962400768.0, "230": 948177792.0, "235": 965221120.0, "240": 966072192.0, "245": 969161216.0, "250": 974435968.0, "255": 925063296.0, "260": 965635968.0, "265": 970660352.0, "270": 959131264.0, "275": 954001216.0, "280": 963427648.0, "285": 945777408.0, "290": 974124544.0, "295": 966704640.0, "300": 967140096.0, "305": 964514048.0, "310": 940354048.0, "315": 967404800.0, "320": 969006080.0, "325": 980552832.0, "330": 972090752.0, "335": 946865984.0, "340": 966598784.0, "345": 973025856.0, "350": 973918720.0, "355": 963261696.0, "360": 948351680.0, "365": 964821248.0, "370": 962952704.0, "375": 958446848.0, "380": 947153280.0, "385": 955988608.0, "390": 945399616.0, "395": 970423552.0, "400": 979770112.0, "405": 968344320.0, "410": 970058752.0, "415": 953158528.0, "420": 943569920.0, "425": 954774144.0, "430": 962663232.0, "435": 977082240.0, "440": 954811392.0, "445": 971894272.0, "450": 963512576.0, "455": 973134720.0, "460": 983714688.0, "465": 945280512.0, "470": 942055616.0, "475": 967007104.0, "480": 966107264.0, "485": 976414528.0, "490": 962538880.0, "495": 945454464.0, "500": 964454656.0, "505": 986005440.0, "510": 965682944.0, "515": 943411584.0, "520": 945017408.0, "525": 971262208.0, "530": 971890688.0, "535": 979140352.0, "540": 969531264.0, "545": 954116608.0, "550": 951267584.0, "555": 987219456.0, "560": 960428288.0, "565": 966616320.0, "570": 975727488.0, "575": 927224960.0, "580": 970694528.0, "585": 961176064.0, "590": 972967040.0, "595": 963682816.0, "600": 937079168.0, "605": 951470208.0, "610": 963360768.0, "615": 970009728.0, "620": 976472192.0, "625": 949579776.0, "630": 954445504.0, "635": 986042816.0, "640": 980980992.0, "645": 955010560.0, "650": 958545664.0, "655": 951656640.0, "660": 961043712.0, "665": 967552000.0, "670": 962514304.0, "675": 968337536.0, "680": 965619200.0, "685": 962871040.0, "690": 961921088.0, "695": 954770368.0, "700": 970340608.0, "705": 945512640.0, "710": 943885440.0, "715": 973357568.0, "720": 968369856.0, "725": 978489984.0, "730": 952195008.0, "735": 948813952.0, "740": 955633408.0, "745": 975866880.0, "750": 981235072.0, "755": 962156608.0, "760": 951964800.0, "765": 967343616.0, "770": 976148096.0, "775": 970544000.0, "780": 977540928.0, "785": 931529024.0, "790": 960441536.0, "795": 964582016.0, "800": 967022848.0, "805": 962321024.0, "810": 940969344.0, "815": 949037568.0, "820": 953181440.0, "825": 954502400.0, "830": 976442240.0, "835": 956073344.0, "840": 948401920.0, "845": 965153024.0, "850": 966029248.0, "855": 960904384.0, "860": 976027200.0, "865": 938157824.0, "870": 966414016.0, "875": 972314880.0, "880": 963120896.0, "885": 967745600.0, "890": 949967872.0, "895": 960019072.0, "900": 974229696.0, "905": 963968256.0, "910": 958435072.0, "915": 956354560.0, "920": 943974592.0, "925": 960833728.0, "930": 978845952.0, "935": 971073664.0, "940": 960905792.0, "945": 945063040.0, "950": 957423360.0, "955": 979035520.0, "960": 983589248.0, "965": 966165824.0, "970": 951228672.0, "975": 961577344.0, "980": 968071040.0, "985": 968991872.0, "990": 984393024.0, "995": 953291264.0, "1000": 934780480.0, "1005": 960147328.0, "1010": 971538624.0, "1015": 985184896.0, "1020": 962780928.0, "1025": 935009408.0, "1030": 974679936.0, "1035": 964992384.0, "1040": 980464256.0, "1045": 960826496.0, "1050": 955197824.0, "1055": 957780352.0, "1060": 967748800.0, "1065": 967116352.0, "1070": 966600064.0, "1075": 950061696.0, "1080": 954508544.0, "1085": 967251712.0, "1090": 977132800.0, "1095": 961237632.0, "1100": 979613568.0, "1105": 953365120.0, "1110": 965954176.0, "1115": 966986944.0, "1120": 970350592.0, "1125": 965707776.0, "1130": 954942400.0, "1135": 965843328.0, "1140": 965176384.0, "1145": 970988224.0, "1150": 955556864.0, "1155": 930578432.0, "1160": 957774208.0, "1165": 978124736.0, "1170": 974299520.0, "1175": 973059648.0, "1180": 973083648.0, "1185": 947344640.0, "1190": 964793216.0, "1195": 953138560.0, "1200": 972843136.0, "1205": 988478656.0, "1210": 931126784.0, "1215": 968647040.0, "1220": 969160960.0, "1225": 975950656.0, "1230": 967331712.0, "1235": 943446912.0, "1240": 955853952.0, "1245": 981503488.0, "1250": 966111808.0, "1255": 973676032.0, "1260": 946497280.0, "1265": 963997824.0, "1270": 960489024.0, "1275": 973615104.0, "1280": 961112576.0, "1285": 957580480.0, "1290": 952528768.0, "1295": 971610240.0, "1300": 968862464.0, "1305": 963739136.0, "1310": 963334656.0, "1315": 943553408.0, "1320": 966307200.0, "1325": 989784960.0, "1330": 969508992.0, "1335": 972302464.0, "1340": 972269440.0, "1345": 960658304.0, "1350": 968639296.0, "1355": 955853312.0, "1360": 971822144.0, "1365": 960387584.0, "1370": 948791872.0, "1375": 973533376.0, "1380": 953470208.0, "1385": 969146880.0, "1390": 975720640.0, "1395": 931673984.0, "1400": 945854848.0, "1405": 976753536.0, "1410": 974510336.0, "1415": 967573760.0, "1420": 966747328.0, "1425": 937378560.0, "1430": 973916608.0, "1435": 978335552.0, "1440": 964178304.0, "1445": 958058240.0, "1450": 946147712.0, "1455": 983922304.0, "1460": 968651136.0, "1465": 948745152.0, "1470": 984243328.0, "1475": 943906048.0, "1480": 963975488.0, "1485": 957349376.0, "1490": 961261888.0, "1495": 980539648.0, "1500": 958332032.0, "1505": 942866816.0, "1510": 984180096.0, "1515": 959094528.0, "1520": 959105408.0, "1525": 952786816.0, "1530": 957741312.0, "1535": 949428928.0, "1540": 971088256.0, "1545": 963132352.0, "1550": 978666752.0, "1555": 952320512.0, "1560": 980089984.0, "1565": 967314048.0, "1570": 973844352.0, "1575": 975494912.0, "1580": 941862656.0, "1585": 970028352.0, "1590": 983822208.0, "1595": 948631616.0, "1600": 967442560.0, "1605": 952451328.0, "1610": 969616512.0, "1615": 983146496.0, "1620": 968019200.0, "1625": 970715776.0, "1630": 962887360.0, "1635": 942311936.0, "1640": 981612224.0, "1645": 973977856.0, "1650": 974188224.0, "1655": 967265024.0, "1660": 940687744.0, "1665": 961704448.0, "1670": 962902016.0, "1675": 971280896.0, "1680": 980879232.0, "1685": 944416192.0, "1690": 964688128.0, "1695": 965644992.0, "1700": 966342336.0, "1705": 985200000.0, "1710": 978354304.0, "1715": 943210880.0, "1720": 977089408.0, "1725": 965870208.0, "1730": 968968960.0, "1735": 965088000.0, "1740": 949713280.0, "1745": 970012352.0, "1750": 959681728.0, "1755": 960085440.0, "1760": 966381376.0, "1765": 951816192.0, "1770": 954665728.0, "1775": 973752064.0, "1780": 970534272.0, "1785": 968824960.0, "1790": 950235520.0, "1795": 945131072.0, "1800": 984666816.0, "1805": 987163520.0, "1810": 977766656.0, "1815": 948004480.0, "1820": 949209216.0, "1825": 978853632.0, "1830": 966362368.0, "1835": 964133632.0, "1840": 972320128.0, "1845": 935415808.0, "1850": 952497792.0, "1855": 980048640.0, "1860": 975866880.0, "1865": 958966528.0, "1870": 958949056.0, "1875": 932593408.0, "1880": 973574016.0, "1885": 978843264.0, "1890": 971358720.0, "1895": 959212288.0, "1900": 947394432.0, "1905": 981829952.0, "1910": 969126912.0, "1915": 970040704.0, "1920": 975597056.0, "1925": 960496512.0, "1930": 977922304.0, "1935": 963250432.0, "1940": 952460928.0, "1945": 981338176.0, "1950": 939172864.0, "1955": 960604416.0, "1960": 970031744.0, "1965": 981176000.0, "1970": 962045120.0, "1975": 952822016.0, "1980": 936847360.0, "1985": 975938432.0, "1990": 965965696.0, "1995": 962609920.0, "2000": 960553984.0, "2005": 954497728.0, "2010": 975579776.0, "2015": 991802112.0, "2020": 975433408.0, "2025": 974303936.0, "2030": 952084736.0, "2035": 967847680.0, "2040": 987457536.0, "2045": 976480064.0, "2050": 984702464.0, "2055": 942839488.0, "2060": 942593920.0, "2065": 966208768.0, "2070": 969622528.0, "2075": 980553536.0, "2080": 977598080.0, "2085": 939635968.0, "2090": 969872256.0, "2095": 961274880.0, "2100": 976719168.0, "2105": 972537920.0, "2110": 959901568.0, "2115": 956875264.0, "2120": 977482304.0, "2125": 962566784.0, "2130": 979618496.0, "2135": 950537408.0, "2140": 946996544.0, "2145": 962273920.0, "2150": 973404416.0, "2155": 972690944.0, "2160": 970314560.0, "2165": 948644160.0, "2170": 961541696.0, "2175": 969377216.0, "2180": 969329920.0, "2185": 947446592.0, "2190": 940480960.0, "2195": 986085952.0, "2200": 961861248.0, "2205": 978924672.0, "2210": 964100864.0, "2215": 963502336.0, "2220": 951311104.0, "2225": 969315776.0, "2230": 976331328.0, "2235": 974025920.0, "2240": 975493888.0, "2245": 960230784.0, "2250": 967640192.0, "2255": 969129984.0, "2260": 975065024.0, "2265": 968258688.0, "2270": 951744768.0, "2275": 962766848.0, "2280": 969640064.0, "2285": 971692992.0, "2290": 962889344.0, "2295": 931409280.0, "2300": 959906048.0, "2305": 970426560.0, "2310": 967444864.0, "2315": 970905792.0, "2320": 975590848.0, "2325": 938587264.0, "2330": 988438528.0, "2335": 977489408.0, "2340": 964596352.0, "2345": 964166528.0, "2350": 947555712.0, "2355": 977029568.0, "2360": 966899072.0, "2365": 977297728.0, "2370": 965072640.0, "2375": 953966272.0, "2380": 962918912.0, "2385": 967194496.0, "2390": 963077248.0, "2395": 974465792.0, "2400": 958410816.0, "2405": 968119552.0, "2410": 951586112.0, "2415": 965904256.0, "2420": 966516160.0, "2425": 959045632.0, "2430": 956685952.0, "2435": 961389184.0, "2440": 959755904.0, "2445": 970891392.0, "2450": 961996736.0, "2455": 922721216.0, "2460": 951953536.0, "2465": 955730432.0, "2470": 972570496.0, "2475": 973812992.0, "2480": 943895296.0, "2485": 944184064.0, "2490": 972411136.0, "2495": 974451712.0, "2500": 973910080.0, "2505": 958492032.0, "2510": 939510912.0, "2515": 979553728.0, "2520": 970473792.0, "2525": 964390784.0, "2530": 955799168.0, "2535": 936598144.0, "2540": 969027648.0, "2545": 970385024.0, "2550": 969462528.0, "2555": 969439040.0, "2560": 964978304.0, "2565": 959763712.0, "2570": 985176704.0, "2575": 957426112.0, "2580": 967424512.0, "2585": 966022400.0, "2590": 956354944.0, "2595": 981830400.0, "2600": 959530880.0, "2605": 962999168.0, "2610": 965972864.0, "2615": 951924992.0, "2620": 971241216.0, "2625": 976456064.0, "2630": 974409984.0, "2635": 948071296.0, "2640": 948137088.0, "2645": 963036736.0, "2650": 953982912.0, "2655": 977112448.0, "2660": 949622976.0, "2665": 953929024.0, "2670": 959064064.0, "2675": 979275904.0, "2680": 961395840.0, "2685": 970701952.0, "2690": 965222016.0, "2695": 943553536.0, "2700": 969425664.0, "2705": 978961792.0, "2710": 971810560.0, "2715": 990813952.0, "2720": 942649152.0, "2725": 967955328.0, "2730": 955466496.0, "2735": 970672704.0, "2740": 977921536.0, "2745": 932280000.0, "2750": 947856384.0, "2755": 956317184.0, "2760": 981697664.0, "2765": 966112192.0, "2770": 948914688.0, "2775": 935830272.0, "2780": 964777088.0, "2785": 969570176.0, "2790": 974273152.0, "2795": 966886144.0, "2800": 944388096.0, "2805": 964353920.0, "2810": 969610752.0, "2815": 975845248.0, "2820": 963081600.0, "2825": 937627392.0, "2830": 956738368.0, "2835": 986321024.0, "2840": 961756672.0, "2845": 967505920.0, "2850": 951714816.0, "2855": 962091520.0, "2860": 954242304.0, "2865": 955881216.0, "2870": 944662848.0, "2875": 974662784.0, "2880": 968199936.0, "2885": 981081984.0, "2890": 953454080.0, "2895": 957178304.0, "2900": 964989440.0, "2905": 931707648.0, "2910": 955730688.0, "2915": 979477120.0, "2920": 970492544.0, "2925": 964975680.0, "2930": 964046592.0, "2935": 940140416.0, "2940": 964912384.0, "2945": 989149952.0, "2950": 965209088.0, "2955": 965104256.0, "2960": 933161472.0, "2965": 968794496.0, "2970": 973034688.0, "2975": 958092288.0, "2980": 964497280.0, "2985": 937267584.0, "2990": 951254720.0, "2995": 978315264.0, "3000": 969275392.0, "3005": 974686080.0, "3010": 950235008.0, "3015": 943841024.0, "3020": 958440960.0, "3025": 975186560.0, "3030": 965018624.0, "3035": 963454464.0, "3040": 952131776.0, "3045": 989793472.0, "3050": 965545728.0, "3055": 982520704.0, "3060": 971227712.0, "3065": 943915648.0, "3070": 978409024.0, "3075": 975204992.0, "3080": 960992896.0, "3085": 962352128.0, "3090": 945953664.0, "3095": 938116032.0, "3100": 972929088.0, "3105": 961989120.0, "3110": 970657152.0, "3115": 963390528.0, "3120": 947116032.0, "3125": 972720640.0, "3130": 952973312.0, "3135": 966041920.0, "3140": 968488896.0, "3145": 937852288.0, "3150": 975009280.0, "3155": 976815488.0, "3160": 969629184.0, "3165": 982194944.0, "3170": 937959936.0, "3175": 953825280.0, "3180": 983810176.0, "3185": 965169536.0, "3190": 968482880.0, "3195": 950933888.0, "3200": 945101440.0, "3205": 959863232.0, "3210": 957486336.0, "3215": 958020096.0, "3220": 968129792.0, "3225": 935614144.0, "3230": 962588672.0, "3235": 975775488.0, "3240": 962621440.0, "3245": 981274368.0, "3250": 943260544.0, "3255": 954599424.0, "3260": 980363648.0, "3265": 963619968.0, "3270": 965162624.0, "3275": 959731072.0, "3280": 967046720.0, "3285": 982478144.0, "3290": 947689408.0, "3295": 966419840.0, "3300": 959165632.0, "3305": 949131008.0, "3310": 979510144.0, "3315": 964283008.0, "3320": 969207296.0, "3325": 956193280.0, "3330": 941167104.0, "3335": 964973184.0, "3340": 956900736.0, "3345": 972500224.0, "3350": 964576192.0, "3355": 943346176.0, "3360": 970037632.0, "3365": 969453952.0, "3370": 954774976.0, "3375": 958676288.0, "3380": 971463168.0, "3385": 947973376.0, "3390": 965793024.0, "3395": 978390016.0, "3400": 978127360.0, "3405": 976724032.0, "3410": 924193664.0, "3415": 955424256.0, "3420": 971820416.0, "3425": 977167488.0, "3430": 973837696.0, "3435": 936071680.0, "3440": 970502464.0, "3445": 957303232.0, "3450": 959840896.0, "3455": 963864256.0, "3460": 967881856.0, "3465": 931318976.0, "3470": 952348224.0, "3475": 973704384.0, "3480": 959738112.0, "3485": 979960640.0, "3490": 944670592.0, "3495": 953904576.0, "3500": 969330432.0, "3505": 964348416.0, "3510": 971222656.0, "3515": 955946112.0, "3520": 958733120.0, "3525": 971914240.0, "3530": 964124672.0, "3535": 983192192.0, "3540": 937489664.0, "3545": 944730496.0, "3550": 984462784.0, "3555": 978057984.0, "3560": 974372992.0, "3565": 968800128.0, "3570": 946694528.0, "3575": 976104640.0, "3580": 977494272.0, "3585": 954568832.0, "3590": 956425856.0, "3595": 951444800.0, "3600": 988996800.0, "3605": 962008448.0, "3610": 965055104.0, "3615": 974635648.0, "3620": 954888384.0, "3625": 939515392.0, "3630": 990148736.0, "3635": 971435712.0, "3640": 976025856.0, "3645": 961489664.0, "3650": 945804160.0, "3655": 965782784.0, "3660": 976208000.0, "3665": 964021248.0, "3670": 977431936.0, "3675": 943476096.0, "3680": 958185152.0, "3685": 964285312.0, "3690": 982093952.0, "3695": 963125248.0, "3700": 950570176.0, "3705": 947345792.0, "3710": 982355328.0, "3715": 972682240.0, "3720": 976138624.0, "3725": 964038272.0, "3730": 948851328.0, "3735": 967079424.0, "3740": 960973568.0, "3745": 969334272.0, "3750": 963949056.0, "3755": 953432832.0, "3760": 976641920.0, "3765": 979841280.0, "3770": 972360320.0, "3775": 972374720.0, "3780": 952585664.0, "3785": 960236800.0, "3790": 985598848.0, "3795": 969195392.0, "3800": 957877504.0, "3805": 972408192.0, "3810": 954517952.0, "3815": 974566528.0, "3820": 963019648.0, "3825": 962070528.0, "3830": 969394304.0, "3835": 934744960.0, "3840": 971255808.0, "3845": 986849792.0, "3850": 968873536.0, "3855": 965248640.0, "3860": 948057600.0, "3865": 975049216.0, "3870": 985098432.0, "3875": 983043072.0, "3880": 963615616.0, "3885": 953012224.0, "3890": 960296960.0, "3895": 960591616.0, "3900": 984926464.0, "3905": 976222592.0, "3910": 987360512.0, "3915": 946017792.0, "3920": 974867328.0, "3925": 961248384.0, "3930": 976790208.0, "3935": 978925824.0, "3940": 950292672.0, "3945": 960260864.0, "3950": 974185152.0, "3955": 972967808.0, "3960": 974078848.0, "3965": 950861696.0, "3970": 980692992.0, "3975": 960750208.0, "3980": 977519808.0, "3985": 962949440.0, "3990": 972755328.0, "3995": 953714176.0, "4000": 975011968.0, "4005": 971658752.0, "4010": 978420992.0, "4015": 971491968.0, "4020": 950311040.0, "4025": 968433024.0, "4030": 997935360.0, "4035": 978548864.0, "4040": 959813248.0, "4045": 939664448.0, "4050": 944718720.0, "4055": 980958720.0, "4060": 977682816.0, "4065": 975706624.0, "4070": 942152832.0, "4075": 945765376.0, "4080": 988765632.0, "4085": 962079872.0, "4090": 983356736.0, "4095": 986958720.0, "4100": 957224448.0, "4105": 954073984.0, "4110": 966488192.0, "4115": 976042432.0, "4120": 983531776.0, "4125": 960054336.0, "4130": 967278976.0, "4135": 971431296.0, "4140": 963171200.0, "4145": 956175616.0, "4150": 960332672.0, "4155": 946216576.0, "4160": 968451328.0, "4165": 970327424.0, "4170": 971946880.0, "4175": 955862272.0, "4180": 940997632.0, "4185": 968318592.0, "4190": 968003712.0, "4195": 989223168.0, "4200": 962678912.0, "4205": 960595072.0, "4210": 971851136.0, "4215": 974145536.0, "4220": 981144704.0, "4225": 975196160.0, "4230": 952703104.0, "4235": 958493312.0, "4240": 966824448.0, "4245": 961756800.0, "4250": 965845632.0, "4255": 958268032.0, "4260": 949545344.0, "4265": 964137856.0, "4270": 978408896.0, "4275": 975397504.0, "4280": 962672064.0, "4285": 951451136.0, "4290": 980029952.0, "4295": 968842816.0, "4300": 958299456.0, "4305": 966802048.0, "4310": 939613376.0, "4315": 949416704.0, "4320": 984522240.0, "4325": 982547712.0, "4330": 974746240.0, "4335": 949459840.0, "4340": 959478400.0, "4345": 956644096.0, "4350": 979960320.0, "4355": 968869888.0, "4360": 966313728.0, "4365": 941208704.0, "4370": 969495552.0, "4375": 972982336.0, "4380": 966147840.0, "4385": 972060544.0, "4390": 954141120.0, "4395": 951856512.0, "4400": 973629184.0, "4405": 972441728.0, "4410": 967908480.0, "4415": 958935232.0, "4420": 960777856.0, "4425": 976416000.0, "4430": 965905664.0, "4435": 975864704.0, "4440": 962282496.0, "4445": 954799872.0, "4450": 978294912.0, "4455": 960203776.0, "4460": 968579008.0, "4465": 968805760.0, "4470": 944127232.0, "4475": 951973056.0, "4480": 978824832.0, "4485": 968265728.0, "4490": 956975616.0, "4495": 938925248.0, "4500": 953215488.0, "4505": 977129344.0, "4510": 978700416.0, "4515": 962493568.0, "4520": 958825024.0, "4525": 958101760.0, "4530": 964428608.0, "4535": 976641024.0, "4540": 976846784.0, "4545": 970169408.0, "4550": 953223680.0, "4555": 959591040.0, "4560": 972497920.0, "4565": 973442560.0, "4570": 978909568.0, "4575": 957760000.0, "4580": 963142016.0, "4585": 957381888.0, "4590": 986530176.0, "4595": 960189824.0, "4600": 952267648.0, "4605": 959297664.0, "4610": 963698304.0, "4615": 957950912.0, "4620": 960255936.0, "4625": 973786624.0, "4630": 944507648.0, "4635": 977024896.0, "4640": 960310272.0, "4645": 981993856.0, "4650": 962400128.0, "4655": 939457792.0, "4660": 964001280.0, "4665": 962540544.0, "4670": 976671872.0, "4675": 963514432.0, "4680": 957488448.0, "4685": 949674432.0, "4690": 956962240.0, "4695": 969926912.0, "4700": 961214016.0, "4705": 970781312.0, "4710": 934493248.0, "4715": 970475008.0, "4720": 966361920.0, "4725": 980303616.0, "4730": 965826240.0, "4735": 937871360.0, "4740": 960124288.0, "4745": 975997376.0, "4750": 967957760.0, "4755": 984959744.0, "4760": 959123648.0, "4765": 955261056.0, "4770": 958621440.0, "4775": 991092608.0, "4780": 976856000.0, "4785": 967582336.0, "4790": 943756160.0, "4795": 955861760.0, "4800": 967745984.0, "4805": 976565888.0, "4810": 965160448.0, "4815": 957969408.0, "4820": 973993216.0, "4825": 961505920.0, "4830": 962638336.0, "4835": 972543936.0, "4840": 948913920.0, "4845": 965716608.0, "4850": 960305024.0, "4855": 964130624.0, "4860": 963051008.0, "4865": 967532352.0, "4870": 957207424.0, "4875": 983574528.0, "4880": 957085120.0, "4885": 977052480.0, "4890": 959740928.0, "4895": 942152192.0, "4900": 973700352.0, "4905": 975213952.0, "4910": 969220608.0, "4915": 970053248.0, "4920": 941169024.0, "4925": 954809344.0, "4930": 977034624.0, "4935": 963750016.0, "4940": 972586496.0, "4945": 960056448.0, "4950": 940793856.0, "4955": 968036480.0, "4960": 976800640.0, "4965": 961022464.0, "4970": 958665472.0, "4975": 933775168.0, "4980": 960824704.0, "4985": 963012160.0, "4990": 963588736.0, "4995": 986307968.0, "5000": 940776000.0, "5005": 968887552.0, "5010": 970307776.0, "5015": 965233408.0, "5020": 966702336.0, "5025": 949471744.0, "5030": 953490560.0, "5035": 967352704.0, "5040": 955689856.0, "5045": 969129856.0, "5050": 953417984.0, "5055": 954811520.0, "5060": 963029248.0, "5065": 952195072.0, "5070": 973597056.0, "5075": 978581632.0, "5080": 942828736.0, "5085": 965863040.0, "5090": 972857088.0, "5095": 964397952.0, "5100": 958318016.0, "5105": 965353728.0, "5110": 950401920.0, "5115": 972347520.0, "5120": 960424960.0, "5125": 969755008.0, "5130": 938795456.0, "5135": 943659008.0, "5140": 969888896.0, "5145": 968688768.0, "5150": 970601216.0, "5155": 972635008.0, "5160": 926551872.0, "5165": 961591552.0, "5170": 966873472.0, "5175": 966086400.0, "5180": 963656192.0, "5185": 930802688.0, "5190": 949852992.0, "5195": 972422016.0, "5200": 973758656.0, "5205": 968249664.0, "5210": 960527936.0, "5215": 928815680.0, "5220": 979174208.0, "5225": 984780416.0, "5230": 975060672.0, "5235": 975055232.0, "5240": 944314112.0, "5245": 970834048.0, "5250": 972426624.0, "5255": 966895296.0, "5260": 976679744.0, "5265": 942270592.0, "5270": 969202176.0, "5275": 970073344.0, "5280": 962825728.0, "5285": 964072064.0, "5290": 932501824.0, "5295": 951762944.0, "5300": 975588288.0, "5305": 951853504.0, "5310": 968040960.0, "5315": 955817472.0, "5320": 950921984.0, "5325": 973040384.0, "5330": 967843264.0, "5335": 967536384.0, "5340": 966503424.0, "5345": 962961344.0, "5350": 978937216.0, "5355": 972202560.0, "5360": 963892288.0, "5365": 965246464.0, "5370": 947824896.0, "5375": 948820864.0, "5380": 967210752.0, "5385": 980540416.0, "5390": 965312832.0, "5395": 955160000.0, "5400": 948286464.0, "5405": 974365952.0, "5410": 967844224.0, "5415": 976074688.0, "5420": 967431680.0, "5425": 937412096.0, "5430": 963919744.0, "5435": 971948544.0, "5440": 969128192.0, "5445": 957501184.0, "5450": 919431040.0, "5455": 952052800.0, "5460": 962315264.0, "5465": 978905088.0, "5470": 981031104.0, "5475": 941610304.0, "5480": 955755072.0, "5485": 964902912.0, "5490": 976036992.0, "5495": 962805120.0, "5500": 971218240.0, "5505": 957001728.0, "5510": 968610432.0, "5515": 945466624.0, "5520": 963181952.0, "5525": 975932480.0, "5530": 936635520.0, "5535": 970693824.0, "5540": 960295936.0, "5545": 972084800.0, "5550": 967898240.0, "5555": 955968960.0, "5560": 954520832.0, "5565": 968862848.0, "5570": 945186112.0, "5575": 960539584.0, "5580": 960563456.0, "5585": 959470720.0, "5590": 977668224.0, "5595": 975194496.0, "5600": 963009472.0, "5605": 964201216.0, "5610": 943157760.0, "5615": 966515904.0, "5620": 963224448.0, "5625": 982398656.0, "5630": 976073984.0, "5635": 957347520.0, "5640": 951455488.0, "5645": 967805568.0, "5650": 979181056.0, "5655": 983510912.0, "5660": 956493952.0, "5665": 953574656.0, "5670": 966097408.0, "5675": 967701184.0, "5680": 978598848.0, "5685": 962008576.0, "5690": 935908928.0, "5695": 963739648.0, "5700": 952464384.0, "5705": 974525376.0, "5710": 971341376.0, "5715": 946147648.0, "5720": 974991360.0, "5725": 967473664.0, "5730": 978561792.0, "5735": 964979712.0, "5740": 943573056.0, "5745": 971142016.0, "5750": 981951168.0, "5755": 956754944.0, "5760": 963695168.0, "5765": 957707648.0, "5770": 955747584.0, "5775": 970847104.0, "5780": 962811840.0, "5785": 970673664.0, "5790": 974652672.0, "5795": 949802368.0, "5800": 965999232.0, "5805": 968774272.0, "5810": 975986176.0, "5815": 970103936.0, "5820": 936377408.0, "5825": 969266816.0, "5830": 977614464.0, "5835": 974980224.0, "5840": 963061120.0, "5845": 968774464.0, "5850": 942897536.0, "5855": 975999104.0, "5860": 979537600.0, "5865": 978369280.0, "5870": 968714112.0, "5875": 942328320.0, "5880": 964508224.0, "5885": 974806656.0, "5890": 972671104.0, "5895": 965681920.0, "5900": 941482880.0, "5905": 961766528.0, "5910": 958568832.0, "5915": 968174464.0, "5920": 977468032.0, "5925": 959468800.0, "5930": 946750080.0, "5935": 952334656.0, "5940": 977662144.0, "5945": 984818560.0, "5950": 980689536.0, "5955": 935094464.0, "5960": 961735296.0, "5965": 965938176.0, "5970": 970612096.0, "5975": 961933888.0, "5980": 958344832.0, "5985": 964562816.0, "5990": 973676288.0, "5995": 955919488.0, "6000": 955633216.0, "6005": 961366784.0, "6010": 952707072.0, "6015": 974539328.0, "6020": 978265920.0, "6025": 972226688.0, "6030": 955311744.0, "6035": 946954368.0, "6040": 962641920.0, "6045": 983743552.0, "6050": 956515200.0, "6055": 963445888.0, "6060": 945767488.0, "6065": 958443776.0, "6070": 978390272.0, "6075": 977988416.0, "6080": 957530304.0, "6085": 947641408.0, "6090": 953645632.0, "6095": 964770560.0, "6100": 979886720.0, "6105": 971011904.0, "6110": 961816128.0, "6115": 943814016.0, "6120": 968573824.0, "6125": 960703232.0, "6130": 984040320.0, "6135": 960994432.0, "6140": 958720704.0, "6145": 971225728.0, "6150": 968493312.0, "6155": 974930688.0, "6160": 977216128.0, "6165": 952742848.0, "6170": 951129728.0, "6175": 963358592.0, "6180": 969549568.0, "6185": 966320320.0, "6190": 963731264.0, "6195": 947266752.0, "6200": 969378240.0, "6205": 967161728.0, "6210": 959339264.0, "6215": 973147776.0, "6220": 936374912.0, "6225": 978715520.0, "6230": 976146816.0, "6235": 971740992.0, "6240": 966006336.0, "6245": 956249728.0, "6250": 956449920.0, "6255": 973363584.0, "6260": 978804800.0, "6265": 974941952.0, "6270": 958855808.0, "6275": 963713152.0, "6280": 973141120.0, "6285": 966123136.0, "6290": 971040256.0, "6295": 987490560.0, "6300": 947640064.0, "6305": 964795456.0, "6310": 979017216.0, "6315": 978443392.0, "6320": 971724672.0, "6325": 923002624.0, "6330": 959366784.0, "6335": 974940864.0, "6340": 984824576.0, "6345": 966835456.0, "6350": 944574016.0, "6355": 957990272.0, "6360": 972711552.0, "6365": 972207744.0, "6370": 958966784.0, "6375": 967142144.0, "6380": 951399104.0, "6385": 973584896.0, "6390": 965514880.0, "6395": 975032064.0, "6400": 983945472.0, "6405": 944065408.0, "6410": 977178496.0, "6415": 971635776.0, "6420": 956726592.0, "6425": 960937728.0, "6430": 957525120.0, "6435": 960333440.0, "6440": 968713088.0, "6445": 973479168.0, "6450": 974637056.0, "6455": 962142208.0, "6460": 940994496.0, "6465": 974482944.0, "6470": 979911936.0, "6475": 960847808.0, "6480": 967532032.0, "6485": 948559616.0, "6490": 970748032.0, "6495": 988369024.0, "6500": 980468864.0, "6505": 972158336.0, "6510": 951648576.0, "6515": 957565440.0, "6520": 979061952.0, "6525": 978903424.0, "6530": 973271744.0, "6535": 967829056.0, "6540": 950159040.0, "6545": 966294144.0, "6550": 979335168.0, "6555": 967119872.0, "6560": 975391104.0, "6565": 949645696.0, "6570": 952068224.0, "6575": 962553728.0, "6580": 975679424.0, "6585": 979544832.0, "6590": 949212544.0, "6595": 961471616.0, "6600": 961353856.0, "6605": 961755520.0, "6610": 985212480.0, "6615": 959518336.0, "6620": 944576256.0, "6625": 971028736.0, "6630": 971564928.0, "6635": 964103936.0, "6640": 959857152.0, "6645": 951077504.0, "6650": 978674944.0, "6655": 965949440.0, "6660": 968814080.0, "6665": 969002112.0, "6670": 932999424.0, "6675": 970736128.0, "6680": 969016064.0, "6685": 958784384.0, "6690": 956215552.0, "6695": 955745920.0, "6700": 962135936.0, "6705": 979365824.0, "6710": 971098240.0, "6715": 966874944.0, "6720": 974162048.0, "6725": 941768192.0, "6730": 979399488.0, "6735": 994709376.0, "6740": 976356224.0, "6745": 974602752.0, "6750": 939272320.0, "6755": 977649344.0, "6760": 969757888.0, "6765": 978454848.0, "6770": 975513728.0, "6775": 943523520.0, "6780": 947283584.0, "6785": 975371712.0, "6790": 960607104.0, "6795": 976217984.0, "6800": 973344640.0, "6805": 946806016.0, "6810": 958265856.0, "6815": 970929792.0, "6820": 978086528.0, "6825": 969192704.0, "6830": 950435072.0, "6835": 981464192.0, "6840": 983022336.0, "6845": 948763840.0, "6850": 965465152.0, "6855": 954199552.0, "6860": 979123968.0, "6865": 983975808.0, "6870": 964842560.0, "6875": 978847808.0, "6880": 950371200.0, "6885": 958582016.0, "6890": 960484032.0, "6895": 965665280.0, "6900": 985370880.0, "6905": 968478592.0, "6910": 950097088.0, "6915": 971060736.0, "6920": 967166720.0, "6925": 965180672.0, "6930": 964715648.0, "6935": 952122112.0, "6940": 962920704.0, "6945": 986470144.0, "6950": 973350272.0, "6955": 964715136.0, "6960": 940248960.0, "6965": 974503680.0, "6970": 978554240.0, "6975": 985114880.0, "6980": 982851072.0, "6985": 959949376.0, "6990": 945298944.0, "6995": 987557120.0, "7000": 963329344.0, "7005": 962922240.0, "7010": 985144320.0, "7015": 945447424.0, "7020": 982884608.0, "7025": 968840640.0, "7030": 953537472.0, "7035": 982810432.0, "7040": 950520320.0, "7045": 956041600.0, "7050": 960403712.0, "7055": 963929728.0, "7060": 976999040.0, "7065": 968391296.0, "7070": 953547264.0, "7075": 956559360.0, "7080": 969124864.0, "7085": 965868800.0, "7090": 969521920.0, "7095": 960078592.0, "7100": 973651200.0, "7105": 973332672.0, "7110": 970254848.0, "7115": 958633088.0, "7120": 948865536.0, "7125": 963163584.0, "7130": 971421376.0, "7135": 964540096.0, "7140": 961681152.0, "7145": 930416448.0, "7150": 946280064.0, "7155": 991092864.0, "7160": 968442496.0, "7165": 956888320.0, "7170": 968275328.0, "7175": 955828224.0, "7180": 958441536.0, "7185": 984880256.0, "7190": 978735936.0, "7195": 973712000.0, "7200": 935905536.0, "7205": 957525760.0, "7210": 967114624.0, "7215": 969707264.0, "7220": 982219584.0, "7225": 928936768.0, "7230": 949560960.0, "7235": 967251712.0, "7240": 966968064.0, "7245": 967454976.0, "7250": 949502336.0, "7255": 957363968.0, "7260": 970114816.0, "7265": 974961664.0, "7270": 959874240.0, "7275": 959286784.0, "7280": 957121920.0, "7285": 977575808.0, "7290": 977249920.0, "7295": 962901120.0, "7300": 975499904.0, "7305": 964022528.0, "7310": 977515520.0, "7315": 966809600.0, "7320": 974552768.0, "7325": 966906752.0, "7330": 959690880.0, "7335": 964049280.0, "7340": 977556864.0, "7345": 967668224.0, "7350": 984804864.0, "7355": 959674816.0, "7360": 948842240.0, "7365": 972772864.0, "7370": 982593664.0, "7375": 963567424.0, "7380": 964284224.0, "7385": 948615488.0, "7390": 964192512.0, "7395": 958787008.0, "7400": 970242816.0, "7405": 988116736.0, "7410": 952423488.0, "7415": 950935744.0, "7420": 967472640.0, "7425": 982705664.0, "7430": 965871552.0, "7435": 973294080.0, "7440": 937228160.0, "7445": 969074752.0, "7450": 980608832.0, "7455": 971587712.0, "7460": 972749056.0, "7465": 939573760.0, "7470": 972011648.0, "7475": 958500480.0, "7480": 969529792.0, "7485": 961637568.0, "7490": 934760704.0, "7495": 957297216.0, "7500": 969548416.0, "7505": 970380928.0, "7510": 972589184.0, "7515": 979619840.0, "7520": 951796224.0, "7525": 970763840.0, "7530": 954655104.0, "7535": 971887616.0, "7540": 979952832.0, "7545": 959493248.0, "7550": 960599936.0, "7555": 960564352.0, "7560": 970209920.0, "7565": 955267200.0, "7570": 942667904.0, "7575": 966060032.0, "7580": 982740480.0, "7585": 979190784.0, "7590": 970155264.0, "7595": 950188416.0, "7600": 946546432.0, "7605": 982722432.0, "7610": 969487360.0, "7615": 988970624.0, "7620": 957136000.0, "7625": 941577856.0, "7630": 971809152.0, "7635": 984756608.0, "7640": 983937792.0, "7645": 968197120.0, "7650": 959357504.0, "7655": 962682368.0, "7660": 969198976.0, "7665": 978073088.0, "7670": 975279104.0, "7675": 975891840.0, "7680": 943167616.0, "7685": 960475136.0, "7690": 975843968.0, "7695": 982086400.0, "7700": 979923648.0, "7705": 940595776.0, "7710": 974811648.0, "7715": 979946496.0, "7720": 968216448.0, "7725": 960576640.0, "7730": 943583104.0, "7735": 968598400.0, "7740": 980697600.0, "7745": 964667008.0, "7750": 963965568.0, "7755": 960341056.0, "7760": 970667072.0, "7765": 971220096.0, "7770": 962730624.0, "7775": 981588800.0, "7780": 965073280.0, "7785": 959776384.0, "7790": 968147968.0, "7795": 969118208.0, "7800": 971586880.0, "7805": 968712128.0, "7810": 946156608.0, "7815": 963601664.0, "7820": 974369664.0, "7825": 963930944.0, "7830": 957420864.0, "7835": 949820864.0, "7840": 957576448.0, "7845": 954299264.0, "7850": 980140416.0, "7855": 987100288.0, "7860": 947203712.0, "7865": 949597632.0, "7870": 965653760.0, "7875": 976341632.0, "7880": 968749184.0, "7885": 969863296.0, "7890": 951979520.0, "7895": 974744576.0, "7900": 964075264.0, "7905": 964628544.0, "7910": 966224768.0, "7915": 943623808.0, "7920": 951251584.0, "7925": 969683840.0, "7930": 965018496.0, "7935": 984522112.0, "7940": 965260992.0, "7945": 950920512.0, "7950": 961778944.0, "7955": 980819072.0, "7960": 964107328.0, "7965": 952800768.0, "7970": 952096960.0, "7975": 969954944.0, "7980": 965058752.0, "7985": 959497728.0, "7990": 968288768.0, "7995": 947074368.0, "8000": 962595712.0, "8005": 980875264.0, "8010": 965703040.0, "8015": 982795648.0, "8020": 960636544.0, "8025": 965519616.0, "8030": 958643200.0, "8035": 975716096.0, "8040": 960827648.0, "8045": 948395264.0, "8050": 959831808.0, "8055": 979617792.0, "8060": 969592128.0, "8065": 958394752.0, "8070": 964066944.0, "8075": 942266240.0, "8080": 966035328.0, "8085": 966815936.0, "8090": 983700160.0, "8095": 988871424.0, "8100": 966531968.0, "8105": 944438272.0, "8110": 969326016.0, "8115": 985228672.0, "8120": 974833408.0, "8125": 964005120.0, "8130": 966272000.0, "8135": 967624576.0, "8140": 963686848.0, "8145": 994976768.0, "8150": 973166016.0, "8155": 938390528.0, "8160": 964462464.0, "8165": 972803200.0, "8170": 968497280.0, "8175": 961587008.0, "8180": 936029440.0, "8185": 962625536.0, "8190": 967799296.0, "8195": 977385088.0, "8200": 956367296.0, "8205": 960566528.0, "8210": 946495424.0, "8215": 982005248.0, "8220": 988443520.0, "8225": 966243584.0, "8230": 962552576.0, "8235": 934131712.0, "8240": 980267904.0, "8245": 976606848.0, "8250": 964327808.0, "8255": 977492864.0, "8260": 956833664.0, "8265": 982957440.0, "8270": 952836608.0, "8275": 974283968.0, "8280": 974906560.0, "8285": 953985664.0, "8290": 940194816.0, "8295": 981360128.0, "8300": 972952832.0, "8305": 978368320.0, "8310": 951095936.0, "8315": 937922048.0, "8320": 977484544.0, "8325": 967872768.0, "8330": 990116800.0, "8335": 975746048.0, "8340": 947366912.0, "8345": 970641408.0, "8350": 970082176.0, "8355": 975014080.0, "8360": 979651456.0, "8365": 932855680.0, "8370": 965537344.0, "8375": 979732736.0, "8380": 965482496.0, "8385": 972889472.0, "8390": 962502912.0, "8395": 951003840.0, "8400": 972739968.0, "8405": 951808384.0, "8410": 960912000.0, "8415": 965867904.0, "8420": 941925888.0, "8425": 968447872.0, "8430": 961416704.0, "8435": 966249344.0, "8440": 969510272.0, "8445": 952921344.0, "8450": 984742912.0, "8455": 990518400.0, "8460": 969086848.0, "8465": 967798656.0, "8470": 963598464.0, "8475": 942921920.0, "8480": 987605888.0, "8485": 979799936.0, "8490": 991849856.0, "8495": 971815552.0, "8500": 951760768.0, "8505": 982982848.0, "8510": 974371200.0, "8515": 969206912.0, "8520": 961827968.0, "8525": 944996096.0, "8530": 984721152.0, "8535": 978411520.0, "8540": 968342592.0, "8545": 969125440.0, "8550": 942408448.0, "8555": 971549056.0, "8560": 958775296.0, "8565": 975676160.0, "8570": 975305216.0, "8575": 971852992.0, "8580": 932583232.0, "8585": 966065856.0, "8590": 978933760.0, "8595": 979387904.0, "8600": 983792768.0, "8605": 958356416.0, "8610": 984069888.0, "8615": 978067776.0, "8620": 963535168.0, "8625": 979909120.0, "8630": 943580032.0, "8635": 961797632.0, "8640": 973745600.0, "8645": 970784128.0, "8650": 969289152.0, "8655": 970653440.0, "8660": 944484096.0, "8665": 986977728.0, "8670": 960353920.0, "8675": 974610176.0, "8680": 962718976.0, "8685": 956147136.0, "8690": 978612864.0, "8695": 969139072.0, "8700": 973135360.0, "8705": 973914176.0, "8710": 947435776.0, "8715": 973736320.0, "8720": 958622976.0, "8725": 978719488.0, "8730": 985894400.0, "8735": 952583040.0, "8740": 940201728.0, "8745": 987763456.0, "8750": 972207744.0, "8755": 971134720.0, "8760": 965569152.0, "8765": 934519872.0, "8770": 986656640.0, "8775": 969789440.0, "8780": 967920512.0, "8785": 962639488.0, "8790": 947921664.0, "8795": 969775296.0, "8800": 971220608.0, "8805": 973559168.0, "8810": 983161280.0, "8815": 951065856.0, "8820": 939478016.0, "8825": 964494336.0, "8830": 981089472.0, "8835": 971889408.0, "8840": 979835520.0, "8845": 951616384.0, "8850": 987153920.0, "8855": 971335296.0, "8860": 962222080.0, "8865": 957359360.0, "8870": 946242816.0, "8875": 968628096.0, "8880": 984173184.0, "8885": 971110144.0, "8890": 970299648.0, "8895": 952971136.0, "8900": 962246528.0, "8905": 977392000.0, "8910": 981876416.0, "8915": 981149952.0, "8920": 968258432.0, "8925": 940189184.0, "8930": 970787456.0, "8935": 963634560.0, "8940": 978025664.0, "8945": 982356352.0, "8950": 946274176.0, "8955": 972928128.0, "8960": 974032128.0, "8965": 973961216.0, "8970": 966361216.0, "8975": 937321600.0, "8980": 953099648.0, "8985": 977878528.0, "8990": 967166592.0, "8995": 980283904.0, "9000": 952421184.0, "9005": 950292544.0, "9010": 974935552.0, "9015": 982668672.0, "9020": 959278656.0, "9025": 979055040.0, "9030": 953936640.0, "9035": 968749312.0, "9040": 978270080.0, "9045": 968843136.0, "9050": 983417600.0, "9055": 947885952.0, "9060": 956699776.0, "9065": 970246528.0, "9070": 968015744.0, "9075": 981225856.0, "9080": 952541632.0, "9085": 971319168.0, "9090": 963789184.0, "9095": 968313984.0, "9100": 974584320.0, "9105": 960032896.0, "9110": 947321664.0, "9115": 956833728.0, "9120": 985899904.0, "9125": 963026176.0, "9130": 958457216.0, "9135": 951989056.0, "9140": 967565824.0, "9145": 977433728.0, "9150": 987305408.0, "9155": 976649408.0, "9160": 958050816.0, "9165": 950957248.0, "9170": 988702272.0, "9175": 971913280.0, "9180": 967854400.0, "9185": 955127680.0, "9190": 957263744.0, "9195": 966003584.0, "9200": 968856960.0, "9205": 967330048.0, "9210": 984179584.0, "9215": 931743808.0, "9220": 949808960.0, "9225": 971440256.0, "9230": 971281792.0, "9235": 971857152.0, "9240": 959917376.0, "9245": 963584128.0, "9250": 961416384.0, "9255": 983241472.0, "9260": 979566336.0, "9265": 953039104.0, "9270": 949474624.0, "9275": 978502016.0, "9280": 978025536.0, "9285": 962828800.0, "9290": 979390080.0, "9295": 958548480.0, "9300": 965876352.0, "9305": 969599232.0, "9310": 973283008.0, "9315": 976451392.0, "9320": 948304512.0, "9325": 979749696.0, "9330": 977926784.0, "9335": 975525504.0, "9340": 960336000.0, "9345": 943464832.0, "9350": 952835072.0, "9355": 962850048.0, "9360": 960675328.0, "9365": 983816320.0, "9370": 983035904.0, "9375": 942080896.0, "9380": 982540928.0, "9385": 985259136.0, "9390": 973406272.0, "9395": 978528128.0, "9400": 938038400.0, "9405": 968500672.0, "9410": 981791488.0, "9415": 991945472.0, "9420": 960625728.0, "9425": 956681216.0, "9430": 938695808.0, "9435": 974362368.0, "9440": 959727872.0, "9445": 973720576.0, "9450": 961877760.0, "9455": 946303872.0, "9460": 978086272.0, "9465": 988617984.0, "9470": 963615872.0, "9475": 983908608.0, "9480": 930854528.0, "9485": 987221248.0, "9490": 963974912.0, "9495": 972857088.0, "9500": 982392960.0, "9505": 970286080.0, "9510": 964873536.0, "9515": 957183296.0, "9520": 948641664.0, "9525": 965336064.0, "9530": 958567296.0, "9535": 950963840.0, "9540": 954501120.0, "9545": 979935296.0, "9550": 955384704.0, "9555": 953296192.0, "9560": 958726208.0, "9565": 969930112.0, "9570": 977751168.0, "9575": 958849792.0, "9580": 963257728.0, "9585": 946197184.0, "9590": 948135936.0, "9595": 967007808.0, "9600": 985117952.0, "9605": 985499648.0, "9610": 943959808.0, "9615": 952912128.0, "9620": 980920192.0, "9625": 978524736.0, "9630": 969671168.0, "9635": 974868544.0, "9640": 940772416.0, "9645": 962475008.0, "9650": 970857536.0, "9655": 987496960.0, "9660": 963394176.0, "9665": 950327872.0, "9670": 965817856.0, "9675": 963579264.0, "9680": 965384064.0, "9685": 986598272.0, "9690": 940596864.0, "9695": 950521728.0, "9700": 975714688.0, "9705": 972896256.0, "9710": 967299968.0, "9715": 971403392.0, "9720": 940613632.0, "9725": 966514816.0, "9730": 974099584.0, "9735": 974345792.0, "9740": 971516928.0, "9745": 951220736.0, "9750": 979370880.0, "9755": 970170432.0, "9760": 968237888.0, "9765": 963835520.0, "9770": 952652160.0, "9775": 956682880.0, "9780": 970721984.0, "9785": 958959232.0, "9790": 961043072.0, "9795": 958779200.0, "9800": 949918656.0, "9805": 962651200.0, "9810": 979093888.0, "9815": 978146816.0, "9820": 982841088.0, "9825": 939730944.0, "9830": 969614208.0, "9835": 973272832.0, "9840": 971945664.0, "9845": 967603328.0, "9850": 947232896.0, "9855": 956896512.0, "9860": 987801728.0, "9865": 970385664.0, "9870": 990310144.0, "9875": 957380096.0, "9880": 931362176.0, "9885": 963678464.0, "9890": 972811648.0, "9895": 984054016.0, "9900": 956595136.0, "9905": 939303808.0, "9910": 979107072.0, "9915": 973996800.0, "9920": 943946432.0, "9925": 963187328.0, "9930": 948020224.0, "9935": 960573120.0, "9940": 965856512.0, "9945": 958998016.0, "9950": 964584192.0, "9955": 943733120.0, "9960": 966844160.0, "9965": 983732096.0, "9970": 966840192.0, "9975": 964040640.0, "9980": 980881024.0, "9985": 942746240.0, "9990": 976134400.0, "9995": 982950848.0, "10000": 972073152.0, "10005": 970193472.0, "10010": 944380480.0, "10015": 983265344.0, "10020": 977865472.0, "10025": 979868544.0, "10030": 971490816.0, "10035": 946263296.0, "10040": 950534016.0, "10045": 977546880.0, "10050": 986017280.0, "10055": 990492800.0, "10060": 958996032.0, "10065": 947517312.0, "10070": 966895616.0, "10075": 979683904.0, "10080": 971953920.0, "10085": 974879744.0, "10090": 944216960.0, "10095": 962977344.0, "10100": 972381952.0, "10105": 976354432.0, "10110": 972128768.0, "10115": 948919680.0, "10120": 962852480.0, "10125": 974293120.0, "10130": 980737472.0, "10135": 972335104.0, "10140": 957843264.0, "10145": 934671872.0, "10150": 973965568.0, "10155": 970306112.0, "10160": 962491456.0, "10165": 975341248.0, "10170": 944624384.0, "10175": 979643712.0, "10180": 984008448.0, "10185": 978870144.0, "10190": 955877376.0, "10195": 937261120.0, "10200": 988253760.0, "10205": 973401856.0, "10210": 966901120.0, "10215": 976049664.0, "10220": 948799872.0, "10225": 950572096.0, "10230": 976120896.0, "10235": 954421632.0, "10240": 969850752.0, "10245": 962265472.0, "10250": 936756480.0, "10255": 979774976.0, "10260": 965000704.0, "10265": 967563712.0, "10270": 969297920.0, "10275": 935944256.0, "10280": 969526272.0, "10285": 996465152.0, "10290": 979762816.0, "10295": 981662912.0, "10300": 952271936.0, "10305": 972024256.0, "10310": 960359872.0, "10315": 971605760.0, "10320": 985354304.0, "10325": 983302336.0, "10330": 935148288.0, "10335": 976392064.0, "10340": 957603840.0, "10345": 973044352.0, "10350": 984707136.0, "10355": 942479296.0, "10360": 962279040.0, "10365": 973641856.0, "10370": 980432768.0, "10375": 970343296.0, "10380": 962080384.0, "10385": 955687296.0, "10390": 990783104.0, "10395": 965164608.0, "10400": 960470208.0, "10405": 950214848.0, "10410": 955491392.0, "10415": 975924736.0, "10420": 967248320.0, "10425": 969875328.0, "10430": 965126272.0, "10435": 962680768.0, "10440": 972024064.0, "10445": 972467456.0, "10450": 974949504.0, "10455": 965864704.0, "10460": 948726272.0, "10465": 971534464.0, "10470": 972756736.0, "10475": 979392128.0, "10480": 997292352.0, "10485": 949631936.0, "10490": 935104896.0, "10495": 969599424.0, "10500": 978688704.0, "10505": 959342784.0, "10510": 951008000.0, "10515": 954223744.0, "10520": 972150016.0, "10525": 969942528.0, "10530": 970425728.0, "10535": 986576256.0, "10540": 946829632.0, "10545": 970484032.0, "10550": 969371968.0, "10555": 959521856.0, "10560": 976274496.0, "10565": 960798208.0, "10570": 968688128.0, "10575": 973272576.0, "10580": 961017472.0, "10585": 973457024.0, "10590": 952053568.0, "10595": 956331776.0, "10600": 967935552.0, "10605": 986576256.0, "10610": 966417408.0, "10615": 976957568.0, "10620": 940933888.0, "10625": 965306432.0, "10630": 968022272.0, "10635": 973333888.0, "10640": 974664448.0, "10645": 948582400.0, "10650": 966388224.0, "10655": 985562624.0, "10660": 976682624.0, "10665": 967088256.0, "10670": 955226368.0, "10675": 934529920.0, "10680": 986153344.0, "10685": 991102656.0, "10690": 963886208.0, "10695": 971933632.0, "10700": 950091520.0, "10705": 978240128.0, "10710": 968317184.0, "10715": 967450432.0, "10720": 966357824.0, "10725": 944490816.0, "10730": 980318592.0, "10735": 961117952.0, "10740": 971283392.0, "10745": 984630528.0, "10750": 981762816.0, "10755": 945191296.0, "10760": 969882304.0, "10765": 972886400.0, "10770": 974268608.0, "10775": 959067392.0, "10780": 949520384.0, "10785": 953706304.0, "10790": 970157568.0, "10795": 960631552.0, "10800": 972050368.0, "10805": 951460864.0, "10810": 974235456.0, "10815": 959804160.0, "10820": 971302656.0, "10825": 967211072.0, "10830": 957016128.0, "10835": 963139136.0, "10840": 971035008.0, "10845": 964268160.0, "10850": 958162432.0, "10855": 967657344.0, "10860": 950849536.0, "10865": 964061696.0, "10870": 983627200.0, "10875": 982016640.0, "10880": 958659648.0, "10885": 954981888.0, "10890": 973122560.0, "10895": 973655744.0, "10900": 970546048.0, "10905": 965184256.0, "10910": 939048192.0, "10915": 960749824.0, "10920": 983653376.0, "10925": 970068160.0, "10930": 968771200.0, "10935": 963228480.0, "10940": 954249408.0, "10945": 964532608.0, "10950": 972466880.0, "10955": 966621248.0, "10960": 972285056.0, "10965": 966333184.0, "10970": 983572160.0, "10975": 965330496.0, "10980": 974669248.0, "10985": 986818496.0, "10990": 950797760.0, "10995": 963598784.0, "11000": 985495104.0, "11005": 978671168.0, "11010": 971614464.0, "11015": 970071232.0, "11020": 948195648.0, "11025": 960105088.0, "11030": 978168768.0, "11035": 976017024.0, "11040": 986523264.0, "11045": 956708480.0, "11050": 973395968.0, "11055": 974051968.0, "11060": 962164544.0, "11065": 985712768.0, "11070": 949791424.0, "11075": 976565888.0, "11080": 972315712.0, "11085": 967328576.0, "11090": 976399296.0, "11095": 946696448.0, "11100": 966199040.0, "11105": 974421504.0, "11110": 981198912.0, "11115": 968108160.0, "11120": 957518656.0, "11125": 956979840.0, "11130": 975786432.0, "11135": 979636544.0, "11140": 964944832.0, "11145": 966499008.0, "11150": 935518400.0, "11155": 976579008.0, "11160": 984367232.0, "11165": 982289792.0, "11170": 978113472.0, "11175": 958084864.0, "11180": 962589888.0, "11185": 972260672.0, "11190": 979666368.0, "11195": 985502784.0, "11200": 983014336.0, "11205": 942426240.0, "11210": 984802368.0, "11215": 967690816.0, "11220": 983476928.0, "11225": 961985728.0, "11230": 953398272.0, "11235": 981841280.0, "11240": 977805568.0, "11245": 966530176.0, "11250": 969466304.0, "11255": 960572544.0, "11260": 980096576.0, "11265": 963926720.0, "11270": 981695936.0, "11275": 968525888.0, "11280": 955905088.0, "11285": 953700224.0, "11290": 956489152.0, "11295": 968197568.0, "11300": 962513216.0, "11305": 958759872.0, "11310": 946304256.0, "11315": 983036096.0, "11320": 964828480.0, "11325": 980906304.0, "11330": 975476608.0, "11335": 952186816.0, "11340": 970596800.0, "11345": 969926080.0, "11350": 981628736.0, "11355": 981905088.0, "11360": 940723328.0, "11365": 970750592.0, "11370": 978978432.0, "11375": 975338432.0, "11380": 968256960.0, "11385": 958096384.0, "11390": 937948288.0, "11395": 977494080.0, "11400": 973515520.0, "11405": 961359424.0, "11410": 966143616.0, "11415": 929202368.0, "11420": 964768960.0, "11425": 981196352.0, "11430": 978636864.0, "11435": 970153280.0, "11440": 945072704.0, "11445": 975241024.0, "11450": 984735296.0, "11455": 971426176.0, "11460": 965182016.0, "11465": 960090176.0, "11470": 955191296.0, "11475": 972691072.0, "11480": 956542272.0, "11485": 977076864.0, "11490": 986332352.0, "11495": 959121344.0, "11500": 969424704.0, "11505": 964024640.0, "11510": 976702848.0, "11515": 977904064.0, "11520": 953963584.0, "11525": 976039360.0, "11530": 976686784.0, "11535": 979809792.0, "11540": 974141760.0, "11545": 953644288.0, "11550": 953295552.0, "11555": 981560640.0, "11560": 984532352.0, "11565": 965181312.0, "11570": 966487424.0, "11575": 950896832.0, "11580": 976062592.0, "11585": 977550784.0, "11590": 969314368.0, "11595": 976719232.0, "11600": 946047104.0, "11605": 973359168.0, "11610": 982457984.0, "11615": 972010048.0, "11620": 969363904.0, "11625": 949111040.0, "11630": 937578176.0, "11635": 973667008.0, "11640": 981259456.0, "11645": 980106048.0, "11650": 971758144.0, "11655": 956204288.0, "11660": 980712192.0, "11665": 958265664.0, "11670": 982618880.0, "11675": 972427200.0, "11680": 956445568.0, "11685": 982955712.0, "11690": 968614528.0, "11695": 968085632.0, "11700": 973819008.0, "11705": 956432640.0, "11710": 964584640.0, "11715": 983208448.0, "11720": 983636224.0, "11725": 965204032.0, "11730": 955695040.0, "11735": 942941376.0, "11740": 973829824.0, "11745": 971260672.0, "11750": 961624256.0, "11755": 963534976.0, "11760": 950291904.0, "11765": 983877632.0, "11770": 984810368.0, "11775": 975671936.0, "11780": 985430336.0, "11785": 947272512.0, "11790": 972444352.0, "11795": 970670464.0, "11800": 973251520.0, "11805": 986780480.0, "11810": 967591808.0, "11815": 955862848.0, "11820": 973765952.0, "11825": 970671296.0, "11830": 974883776.0, "11835": 961826368.0, "11840": 944467904.0, "11845": 980681344.0, "11850": 974601536.0, "11855": 977943744.0, "11860": 971600192.0, "11865": 938631104.0, "11870": 940150208.0, "11875": 990036736.0, "11880": 971605184.0, "11885": 962938432.0, "11890": 970330560.0, "11895": 965356416.0, "11900": 978791360.0, "11905": 961690240.0, "11910": 983653824.0, "11915": 989882688.0, "11920": 944918016.0, "11925": 994335296.0, "11930": 964853504.0, "11935": 963527104.0, "11940": 977089344.0, "11945": 944859968.0, "11950": 977822912.0, "11955": 979190400.0, "11960": 972201664.0, "11965": 976359488.0, "11970": 963048064.0, "11975": 963239936.0, "11980": 977609536.0, "11985": 953135424.0, "11990": 968692864.0, "11995": 965206144.0, "12000": 958650816.0, "12005": 974558720.0, "12010": 979488320.0, "12015": 972212928.0, "12020": 972972992.0, "12025": 934714048.0, "12030": 969111104.0, "12035": 984089664.0, "12040": 977507648.0, "12045": 981645056.0, "12050": 931286400.0, "12055": 938790208.0, "12060": 974349248.0, "12065": 965845696.0, "12070": 968239104.0, "12075": 949940224.0, "12080": 953231552.0, "12085": 972342592.0, "12090": 964042304.0, "12095": 963552832.0, "12100": 976701248.0, "12105": 950225216.0, "12110": 972205568.0, "12115": 968115136.0, "12120": 986359296.0, "12125": 980675264.0, "12130": 941589504.0, "12135": 955591040.0, "12140": 975756032.0, "12145": 979390528.0, "12150": 979435776.0, "12155": 961767936.0, "12160": 946323264.0, "12165": 968612864.0, "12170": 964129152.0, "12175": 967543936.0, "12180": 974942848.0, "12185": 952774592.0, "12190": 988261760.0, "12195": 970261312.0, "12200": 964981312.0, "12205": 968767232.0, "12210": 939144320.0, "12215": 996663488.0, "12220": 970037696.0, "12225": 979618880.0, "12230": 980198144.0, "12235": 950224576.0, "12240": 963441344.0, "12245": 965754240.0, "12250": 976793792.0, "12255": 968140288.0, "12260": 983346688.0, "12265": 931758592.0, "12270": 966457472.0, "12275": 979665408.0, "12280": 977629696.0, "12285": 970432320.0, "12290": 929095296.0, "12295": 976750144.0, "12300": 985683008.0, "12305": 969918016.0, "12310": 986105792.0, "12315": 936237952.0, "12320": 957828032.0, "12325": 966696000.0, "12330": 968270016.0, "12335": 963800896.0, "12340": 957433344.0, "12345": 944247872.0, "12350": 966701760.0, "12355": 975709440.0, "12360": 978832832.0, "12365": 964561280.0, "12370": 948990016.0, "12375": 963593536.0, "12380": 964789056.0, "12385": 973126912.0, "12390": 961501056.0, "12395": 961629824.0, "12400": 975307712.0, "12405": 976388032.0, "12410": 953154688.0, "12415": 962821184.0, "12420": 943854144.0, "12425": 948976640.0, "12430": 972157696.0, "12435": 969004352.0, "12440": 961893696.0, "12445": 951751744.0, "12450": 947021888.0, "12455": 981248448.0, "12460": 973989696.0, "12465": 954462784.0, "12470": 981144320.0, "12475": 958569728.0, "12480": 967084864.0, "12485": 978267776.0, "12490": 974168192.0, "12495": 969692160.0, "12500": 961573632.0, "12505": 943534528.0, "12510": 961013504.0, "12515": 969566080.0, "12520": 974194304.0, "12525": 972107840.0, "12530": 944608640.0, "12535": 976414272.0, "12540": 965917440.0, "12545": 972033856.0, "12550": 969671552.0, "12555": 941300736.0, "12560": 964469312.0, "12565": 947620608.0, "12570": 974443840.0, "12575": 963055232.0, "12580": 958104128.0, "12585": 964300352.0, "12590": 965924288.0, "12595": 978732672.0, "12600": 982093952.0, "12605": 949125824.0, "12610": 937745344.0, "12615": 962779264.0, "12620": 961201664.0, "12625": 966531136.0, "12630": 971046272.0, "12635": 962236416.0, "12640": 978541696.0, "12645": 969369920.0, "12650": 970161664.0, "12655": 964217216.0, "12660": 932390336.0, "12665": 956865664.0, "12670": 986180352.0, "12675": 965566464.0, "12680": 961126528.0, "12685": 951304256.0, "12690": 945491456.0, "12695": 978387648.0, "12700": 985277888.0, "12705": 958784640.0, "12710": 968294144.0, "12715": 956280512.0, "12720": "nan", "12725": "nan", "12730": "nan", "12735": "nan", "12740": "nan", "12745": "nan", "12750": "nan", "12755": "nan", "12760": "nan", "12765": "nan", "12770": "nan", "12775": "nan", "12780": "nan", "12785": "nan", "12790": "nan", "12795": "nan", "12800": "nan", "12805": "nan", "12810": "nan", "12815": "nan", "12820": "nan", "12825": "nan", "12830": "nan", "12835": "nan", "12840": "nan", "12845": "nan", "12850": "nan", "12855": "nan", "12860": "nan", "12865": "nan", "12870": "nan", "12875": "nan", "12880": "nan", "12885": "nan", "12890": "nan", "12895": "nan", "12900": "nan", "12905": "nan", "12910": "nan", "12915": "nan", "12920": "nan", "12925": "nan", "12930": "nan", "12935": "nan", "12940": "nan", "12945": "nan", "12950": "nan", "12955": "nan", "12960": "nan", "12965": "nan", "12970": "nan", "12975": "nan", "12980": "nan", "12985": "nan", "12990": "nan", "12995": "nan", "13000": "nan"}}, "mem-allocated-bytes": {"start_step": 1, "end_step": 13000, "step_interval": 5, "values": {"1": 12795811840.0, "5": 12795811840.0, "10": 12795811840.0, "15": 12795811840.0, "20": 12795811840.0, "25": 12795811840.0, "30": 12795811840.0, "35": 12795811840.0, "40": 12795811840.0, "45": 12795811840.0, "50": 12795811840.0, "55": 12795811840.0, "60": 12795811840.0, "65": 12795811840.0, "70": 12795811840.0, "75": 12795811840.0, "80": 12795811840.0, "85": 12795811840.0, "90": 12795811840.0, "95": 12795811840.0, "100": 12795811840.0, "105": 12795811840.0, "110": 12795811840.0, "115": 12795811840.0, "120": 12795811840.0, "125": 12795811840.0, "130": 12795811840.0, "135": 12795811840.0, "140": 12795811840.0, "145": 12795811840.0, "150": 12795811840.0, "155": 12795811840.0, "160": 12795811840.0, "165": 12795811840.0, "170": 12795811840.0, "175": 12795811840.0, "180": 12795811840.0, "185": 12795811840.0, "190": 12795811840.0, "195": 12795811840.0, "200": 12795811840.0, "205": 12795811840.0, "210": 12795811840.0, "215": 12795811840.0, "220": 12795811840.0, "225": 12795811840.0, "230": 12795811840.0, "235": 12795811840.0, "240": 12795811840.0, "245": 12795811840.0, "250": 12795811840.0, "255": 12795811840.0, "260": 12795811840.0, "265": 12795811840.0, "270": 12795811840.0, "275": 12795811840.0, "280": 12795811840.0, "285": 12795811840.0, "290": 12795811840.0, "295": 12795811840.0, "300": 12795811840.0, "305": 12795811840.0, "310": 12795811840.0, "315": 12795811840.0, "320": 12795811840.0, "325": 12795811840.0, "330": 12795811840.0, "335": 12795811840.0, "340": 12795811840.0, "345": 12795811840.0, "350": 12795811840.0, "355": 12795811840.0, "360": 12795811840.0, "365": 12795811840.0, "370": 12795811840.0, "375": 12795811840.0, "380": 12795811840.0, "385": 12795811840.0, "390": 12795811840.0, "395": 12795811840.0, "400": 12795811840.0, "405": 12795811840.0, "410": 12795811840.0, "415": 12795811840.0, "420": 12795811840.0, "425": 12795811840.0, "430": 12795811840.0, "435": 12795811840.0, "440": 12795811840.0, "445": 12795811840.0, "450": 12795811840.0, "455": 12795811840.0, "460": 12795811840.0, "465": 12795811840.0, "470": 12795811840.0, "475": 12795811840.0, "480": 12795811840.0, "485": 12795811840.0, "490": 12795811840.0, "495": 12795811840.0, "500": 12795811840.0, "505": 12795811840.0, "510": 12795811840.0, "515": 12795811840.0, "520": 12795811840.0, "525": 12795811840.0, "530": 12795811840.0, "535": 12795811840.0, "540": 12795811840.0, "545": 12795811840.0, "550": 12795811840.0, "555": 12795811840.0, "560": 12795811840.0, "565": 12795811840.0, "570": 12795811840.0, "575": 12795811840.0, "580": 12795811840.0, "585": 12795811840.0, "590": 12795811840.0, "595": 12795811840.0, "600": 12795811840.0, "605": 12795811840.0, "610": 12795811840.0, "615": 12795811840.0, "620": 12795811840.0, "625": 12795811840.0, "630": 12795811840.0, "635": 12795811840.0, "640": 12795811840.0, "645": 12795811840.0, "650": 12795811840.0, "655": 12795811840.0, "660": 12795811840.0, "665": 12795811840.0, "670": 12795811840.0, "675": 12795811840.0, "680": 12795811840.0, "685": 12795811840.0, "690": 12795811840.0, "695": 12795811840.0, "700": 12795811840.0, "705": 12795811840.0, "710": 12795811840.0, "715": 12795811840.0, "720": 12795811840.0, "725": 12795811840.0, "730": 12795811840.0, "735": 12795811840.0, "740": 12795811840.0, "745": 12795811840.0, "750": 12795811840.0, "755": 12795811840.0, "760": 12795811840.0, "765": 12795811840.0, "770": 12795811840.0, "775": 12795811840.0, "780": 12795811840.0, "785": 12795811840.0, "790": 12795811840.0, "795": 12795811840.0, "800": 12795811840.0, "805": 12795811840.0, "810": 12795811840.0, "815": 12795811840.0, "820": 12795811840.0, "825": 12795811840.0, "830": 12795811840.0, "835": 12795811840.0, "840": 12795811840.0, "845": 12795811840.0, "850": 12795811840.0, "855": 12795811840.0, "860": 12795811840.0, "865": 12795811840.0, "870": 12795811840.0, "875": 12795811840.0, "880": 12795811840.0, "885": 12795811840.0, "890": 12795811840.0, "895": 12795811840.0, "900": 12795811840.0, "905": 12795811840.0, "910": 12795811840.0, "915": 12795811840.0, "920": 12795811840.0, "925": 12795811840.0, "930": 12795811840.0, "935": 12795811840.0, "940": 12795811840.0, "945": 12795811840.0, "950": 12795811840.0, "955": 12795811840.0, "960": 12795811840.0, "965": 12795811840.0, "970": 12795811840.0, "975": 12795811840.0, "980": 12795811840.0, "985": 12795811840.0, "990": 12795811840.0, "995": 12795811840.0, "1000": 12795811840.0, "1005": 12795811840.0, "1010": 12795811840.0, "1015": 12795811840.0, "1020": 12795811840.0, "1025": 12795811840.0, "1030": 12795811840.0, "1035": 12795811840.0, "1040": 12795811840.0, "1045": 12795811840.0, "1050": 12795811840.0, "1055": 12795811840.0, "1060": 12795811840.0, "1065": 12795811840.0, "1070": 12795811840.0, "1075": 12795811840.0, "1080": 12795811840.0, "1085": 12795811840.0, "1090": 12795811840.0, "1095": 12795811840.0, "1100": 12795811840.0, "1105": 12795811840.0, "1110": 12795811840.0, "1115": 12795811840.0, "1120": 12795811840.0, "1125": 12795811840.0, "1130": 12795811840.0, "1135": 12795811840.0, "1140": 12795811840.0, "1145": 12795811840.0, "1150": 12795811840.0, "1155": 12795811840.0, "1160": 12795811840.0, "1165": 12795811840.0, "1170": 12795811840.0, "1175": 12795811840.0, "1180": 12795811840.0, "1185": 12795811840.0, "1190": 12795811840.0, "1195": 12795811840.0, "1200": 12795811840.0, "1205": 12795811840.0, "1210": 12795811840.0, "1215": 12795811840.0, "1220": 12795811840.0, "1225": 12795811840.0, "1230": 12795811840.0, "1235": 12795811840.0, "1240": 12795811840.0, "1245": 12795811840.0, "1250": 12795811840.0, "1255": 12795811840.0, "1260": 12795811840.0, "1265": 12795811840.0, "1270": 12795811840.0, "1275": 12795811840.0, "1280": 12795811840.0, "1285": 12795811840.0, "1290": 12795811840.0, "1295": 12795811840.0, "1300": 12795811840.0, "1305": 12795811840.0, "1310": 12795811840.0, "1315": 12795811840.0, "1320": 12795811840.0, "1325": 12795811840.0, "1330": 12795811840.0, "1335": 12795811840.0, "1340": 12795811840.0, "1345": 12795811840.0, "1350": 12795811840.0, "1355": 12795811840.0, "1360": 12795811840.0, "1365": 12795811840.0, "1370": 12795811840.0, "1375": 12795811840.0, "1380": 12795811840.0, "1385": 12795811840.0, "1390": 12795811840.0, "1395": 12795811840.0, "1400": 12795811840.0, "1405": 12795811840.0, "1410": 12795811840.0, "1415": 12795811840.0, "1420": 12795811840.0, "1425": 12795811840.0, "1430": 12795811840.0, "1435": 12795811840.0, "1440": 12795811840.0, "1445": 12795811840.0, "1450": 12795811840.0, "1455": 12795811840.0, "1460": 12795811840.0, "1465": 12795811840.0, "1470": 12795811840.0, "1475": 12795811840.0, "1480": 12795811840.0, "1485": 12795811840.0, "1490": 12795811840.0, "1495": 12795811840.0, "1500": 12795811840.0, "1505": 12795811840.0, "1510": 12795811840.0, "1515": 12795811840.0, "1520": 12795811840.0, "1525": 12795811840.0, "1530": 12795811840.0, "1535": 12795811840.0, "1540": 12795811840.0, "1545": 12795811840.0, "1550": 12795811840.0, "1555": 12795811840.0, "1560": 12795811840.0, "1565": 12795811840.0, "1570": 12795811840.0, "1575": 12795811840.0, "1580": 12795811840.0, "1585": 12795811840.0, "1590": 12795811840.0, "1595": 12795811840.0, "1600": 12795811840.0, "1605": 12795811840.0, "1610": 12795811840.0, "1615": 12795811840.0, "1620": 12795811840.0, "1625": 12795811840.0, "1630": 12795811840.0, "1635": 12795811840.0, "1640": 12795811840.0, "1645": 12795811840.0, "1650": 12795811840.0, "1655": 12795811840.0, "1660": 12795811840.0, "1665": 12795811840.0, "1670": 12795811840.0, "1675": 12795811840.0, "1680": 12795811840.0, "1685": 12795811840.0, "1690": 12795811840.0, "1695": 12795811840.0, "1700": 12795811840.0, "1705": 12795811840.0, "1710": 12795811840.0, "1715": 12795811840.0, "1720": 12795811840.0, "1725": 12795811840.0, "1730": 12795811840.0, "1735": 12795811840.0, "1740": 12795811840.0, "1745": 12795811840.0, "1750": 12795811840.0, "1755": 12795811840.0, "1760": 12795811840.0, "1765": 12795811840.0, "1770": 12795811840.0, "1775": 12795811840.0, "1780": 12795811840.0, "1785": 12795811840.0, "1790": 12795811840.0, "1795": 12795811840.0, "1800": 12795811840.0, "1805": 12795811840.0, "1810": 12795811840.0, "1815": 12795811840.0, "1820": 12795811840.0, "1825": 12795811840.0, "1830": 12795811840.0, "1835": 12795811840.0, "1840": 12795811840.0, "1845": 12795811840.0, "1850": 12795811840.0, "1855": 12795811840.0, "1860": 12795811840.0, "1865": 12795811840.0, "1870": 12795811840.0, "1875": 12795811840.0, "1880": 12795811840.0, "1885": 12795811840.0, "1890": 12795811840.0, "1895": 12795811840.0, "1900": 12795811840.0, "1905": 12795811840.0, "1910": 12795811840.0, "1915": 12795811840.0, "1920": 12795811840.0, "1925": 12795811840.0, "1930": 12795811840.0, "1935": 12795811840.0, "1940": 12795811840.0, "1945": 12795811840.0, "1950": 12795811840.0, "1955": 12795811840.0, "1960": 12795811840.0, "1965": 12795811840.0, "1970": 12795811840.0, "1975": 12795811840.0, "1980": 12795811840.0, "1985": 12795811840.0, "1990": 12795811840.0, "1995": 12795811840.0, "2000": 12795811840.0, "2005": 12795811840.0, "2010": 12795811840.0, "2015": 12795811840.0, "2020": 12795811840.0, "2025": 12795811840.0, "2030": 12795811840.0, "2035": 12795811840.0, "2040": 12795811840.0, "2045": 12795811840.0, "2050": 12795811840.0, "2055": 12795811840.0, "2060": 12795811840.0, "2065": 12795811840.0, "2070": 12795811840.0, "2075": 12795811840.0, "2080": 12795811840.0, "2085": 12795811840.0, "2090": 12795811840.0, "2095": 12795811840.0, "2100": 12795811840.0, "2105": 12795811840.0, "2110": 12795811840.0, "2115": 12795811840.0, "2120": 12795811840.0, "2125": 12795811840.0, "2130": 12795811840.0, "2135": 12795811840.0, "2140": 12795811840.0, "2145": 12795811840.0, "2150": 12795811840.0, "2155": 12795811840.0, "2160": 12795811840.0, "2165": 12795811840.0, "2170": 12795811840.0, "2175": 12795811840.0, "2180": 12795811840.0, "2185": 12795811840.0, "2190": 12795811840.0, "2195": 12795811840.0, "2200": 12795811840.0, "2205": 12795811840.0, "2210": 12795811840.0, "2215": 12795811840.0, "2220": 12795811840.0, "2225": 12795811840.0, "2230": 12795811840.0, "2235": 12795811840.0, "2240": 12795811840.0, "2245": 12795811840.0, "2250": 12795811840.0, "2255": 12795811840.0, "2260": 12795811840.0, "2265": 12795811840.0, "2270": 12795811840.0, "2275": 12795811840.0, "2280": 12795811840.0, "2285": 12795811840.0, "2290": 12795811840.0, "2295": 12795811840.0, "2300": 12795811840.0, "2305": 12795811840.0, "2310": 12795811840.0, "2315": 12795811840.0, "2320": 12795811840.0, "2325": 12795811840.0, "2330": 12795811840.0, "2335": 12795811840.0, "2340": 12795811840.0, "2345": 12795811840.0, "2350": 12795811840.0, "2355": 12795811840.0, "2360": 12795811840.0, "2365": 12795811840.0, "2370": 12795811840.0, "2375": 12795811840.0, "2380": 12795811840.0, "2385": 12795811840.0, "2390": 12795811840.0, "2395": 12795811840.0, "2400": 12795811840.0, "2405": 12795811840.0, "2410": 12795811840.0, "2415": 12795811840.0, "2420": 12795811840.0, "2425": 12795811840.0, "2430": 12795811840.0, "2435": 12795811840.0, "2440": 12795811840.0, "2445": 12795811840.0, "2450": 12795811840.0, "2455": 12795811840.0, "2460": 12795811840.0, "2465": 12795811840.0, "2470": 12795811840.0, "2475": 12795811840.0, "2480": 12795811840.0, "2485": 12795811840.0, "2490": 12795811840.0, "2495": 12795811840.0, "2500": 12795811840.0, "2505": 12795811840.0, "2510": 12795811840.0, "2515": 12795811840.0, "2520": 12795811840.0, "2525": 12795811840.0, "2530": 12795811840.0, "2535": 12795811840.0, "2540": 12795811840.0, "2545": 12795811840.0, "2550": 12795811840.0, "2555": 12795811840.0, "2560": 12795811840.0, "2565": 12795811840.0, "2570": 12795811840.0, "2575": 12795811840.0, "2580": 12795811840.0, "2585": 12795811840.0, "2590": 12795811840.0, "2595": 12795811840.0, "2600": 12795811840.0, "2605": 12795811840.0, "2610": 12795811840.0, "2615": 12795811840.0, "2620": 12795811840.0, "2625": 12795811840.0, "2630": 12795811840.0, "2635": 12795811840.0, "2640": 12795811840.0, "2645": 12795811840.0, "2650": 12795811840.0, "2655": 12795811840.0, "2660": 12795811840.0, "2665": 12795811840.0, "2670": 12795811840.0, "2675": 12795811840.0, "2680": 12795811840.0, "2685": 12795811840.0, "2690": 12795811840.0, "2695": 12795811840.0, "2700": 12795811840.0, "2705": 12795811840.0, "2710": 12795811840.0, "2715": 12795811840.0, "2720": 12795811840.0, "2725": 12795811840.0, "2730": 12795811840.0, "2735": 12795811840.0, "2740": 12795811840.0, "2745": 12795811840.0, "2750": 12795811840.0, "2755": 12795811840.0, "2760": 12795811840.0, "2765": 12795811840.0, "2770": 12795811840.0, "2775": 12795811840.0, "2780": 12795811840.0, "2785": 12795811840.0, "2790": 12795811840.0, "2795": 12795811840.0, "2800": 12795811840.0, "2805": 12795811840.0, "2810": 12795811840.0, "2815": 12795811840.0, "2820": 12795811840.0, "2825": 12795811840.0, "2830": 12795811840.0, "2835": 12795811840.0, "2840": 12795811840.0, "2845": 12795811840.0, "2850": 12795811840.0, "2855": 12795811840.0, "2860": 12795811840.0, "2865": 12795811840.0, "2870": 12795811840.0, "2875": 12795811840.0, "2880": 12795811840.0, "2885": 12795811840.0, "2890": 12795811840.0, "2895": 12795811840.0, "2900": 12795811840.0, "2905": 12795811840.0, "2910": 12795811840.0, "2915": 12795811840.0, "2920": 12795811840.0, "2925": 12795811840.0, "2930": 12795811840.0, "2935": 12795811840.0, "2940": 12795811840.0, "2945": 12795811840.0, "2950": 12795811840.0, "2955": 12795811840.0, "2960": 12795811840.0, "2965": 12795811840.0, "2970": 12795811840.0, "2975": 12795811840.0, "2980": 12795811840.0, "2985": 12795811840.0, "2990": 12795811840.0, "2995": 12795811840.0, "3000": 12795811840.0, "3005": 12795811840.0, "3010": 12795811840.0, "3015": 12795811840.0, "3020": 12795811840.0, "3025": 12795811840.0, "3030": 12795811840.0, "3035": 12795811840.0, "3040": 12795811840.0, "3045": 12795811840.0, "3050": 12795811840.0, "3055": 12795811840.0, "3060": 12795811840.0, "3065": 12795811840.0, "3070": 12795811840.0, "3075": 12795811840.0, "3080": 12795811840.0, "3085": 12795811840.0, "3090": 12795811840.0, "3095": 12795811840.0, "3100": 12795811840.0, "3105": 12795811840.0, "3110": 12795811840.0, "3115": 12795811840.0, "3120": 12795811840.0, "3125": 12795811840.0, "3130": 12795811840.0, "3135": 12795811840.0, "3140": 12795811840.0, "3145": 12795811840.0, "3150": 12795811840.0, "3155": 12795811840.0, "3160": 12795811840.0, "3165": 12795811840.0, "3170": 12795811840.0, "3175": 12795811840.0, "3180": 12795811840.0, "3185": 12795811840.0, "3190": 12795811840.0, "3195": 12795811840.0, "3200": 12795811840.0, "3205": 12795811840.0, "3210": 12795811840.0, "3215": 12795811840.0, "3220": 12795811840.0, "3225": 12795811840.0, "3230": 12795811840.0, "3235": 12795811840.0, "3240": 12795811840.0, "3245": 12795811840.0, "3250": 12795811840.0, "3255": 12795811840.0, "3260": 12795811840.0, "3265": 12795811840.0, "3270": 12795811840.0, "3275": 12795811840.0, "3280": 12795811840.0, "3285": 12795811840.0, "3290": 12795811840.0, "3295": 12795811840.0, "3300": 12795811840.0, "3305": 12795811840.0, "3310": 12795811840.0, "3315": 12795811840.0, "3320": 12795811840.0, "3325": 12795811840.0, "3330": 12795811840.0, "3335": 12795811840.0, "3340": 12795811840.0, "3345": 12795811840.0, "3350": 12795811840.0, "3355": 12795811840.0, "3360": 12795811840.0, "3365": 12795811840.0, "3370": 12795811840.0, "3375": 12795811840.0, "3380": 12795811840.0, "3385": 12795811840.0, "3390": 12795811840.0, "3395": 12795811840.0, "3400": 12795811840.0, "3405": 12795811840.0, "3410": 12795811840.0, "3415": 12795811840.0, "3420": 12795811840.0, "3425": 12795811840.0, "3430": 12795811840.0, "3435": 12795811840.0, "3440": 12795811840.0, "3445": 12795811840.0, "3450": 12795811840.0, "3455": 12795811840.0, "3460": 12795811840.0, "3465": 12795811840.0, "3470": 12795811840.0, "3475": 12795811840.0, "3480": 12795811840.0, "3485": 12795811840.0, "3490": 12795811840.0, "3495": 12795811840.0, "3500": 12795811840.0, "3505": 12795811840.0, "3510": 12795811840.0, "3515": 12795811840.0, "3520": 12795811840.0, "3525": 12795811840.0, "3530": 12795811840.0, "3535": 12795811840.0, "3540": 12795811840.0, "3545": 12795811840.0, "3550": 12795811840.0, "3555": 12795811840.0, "3560": 12795811840.0, "3565": 12795811840.0, "3570": 12795789312.0, "3575": 12795789312.0, "3580": 12795789312.0, "3585": 12795789312.0, "3590": 12795789312.0, "3595": 12795789312.0, "3600": 12795789312.0, "3605": 12795789312.0, "3610": 12795789312.0, "3615": 12795789312.0, "3620": 12795789312.0, "3625": 12795789312.0, "3630": 12795789312.0, "3635": 12795789312.0, "3640": 12795789312.0, "3645": 12795789312.0, "3650": 12795789312.0, "3655": 12795789312.0, "3660": 12795789312.0, "3665": 12795789312.0, "3670": 12795789312.0, "3675": 12795789312.0, "3680": 12795789312.0, "3685": 12795789312.0, "3690": 12795789312.0, "3695": 12795789312.0, "3700": 12795789312.0, "3705": 12795789312.0, "3710": 12795789312.0, "3715": 12795789312.0, "3720": 12795789312.0, "3725": 12795789312.0, "3730": 12795789312.0, "3735": 12795789312.0, "3740": 12795789312.0, "3745": 12795789312.0, "3750": 12795789312.0, "3755": 12795789312.0, "3760": 12795789312.0, "3765": 12795789312.0, "3770": 12795789312.0, "3775": 12795789312.0, "3780": 12795789312.0, "3785": 12795789312.0, "3790": 12795789312.0, "3795": 12795789312.0, "3800": 12795789312.0, "3805": 12795789312.0, "3810": 12795789312.0, "3815": 12795789312.0, "3820": 12795789312.0, "3825": 12795789312.0, "3830": 12795789312.0, "3835": 12795789312.0, "3840": 12795789312.0, "3845": 12795789312.0, "3850": 12795789312.0, "3855": 12795789312.0, "3860": 12795789312.0, "3865": 12795789312.0, "3870": 12795789312.0, "3875": 12795789312.0, "3880": 12795789312.0, "3885": 12795789312.0, "3890": 12795789312.0, "3895": 12795789312.0, "3900": 12795789312.0, "3905": 12795789312.0, "3910": 12795789312.0, "3915": 12795789312.0, "3920": 12795789312.0, "3925": 12795789312.0, "3930": 12795789312.0, "3935": 12795789312.0, "3940": 12795789312.0, "3945": 12795789312.0, "3950": 12795789312.0, "3955": 12795789312.0, "3960": 12795789312.0, "3965": 12795789312.0, "3970": 12795789312.0, "3975": 12795789312.0, "3980": 12795789312.0, "3985": 12795789312.0, "3990": 12795789312.0, "3995": 12795789312.0, "4000": 12795789312.0, "4005": 12795789312.0, "4010": 12795789312.0, "4015": 12795789312.0, "4020": 12795789312.0, "4025": 12795789312.0, "4030": 12795789312.0, "4035": 12795789312.0, "4040": 12795789312.0, "4045": 12795789312.0, "4050": 12795789312.0, "4055": 12795789312.0, "4060": 12795789312.0, "4065": 12795789312.0, "4070": 12795789312.0, "4075": 12795789312.0, "4080": 12795789312.0, "4085": 12795789312.0, "4090": 12795789312.0, "4095": 12795789312.0, "4100": 12795789312.0, "4105": 12795789312.0, "4110": 12795789312.0, "4115": 12795789312.0, "4120": 12795789312.0, "4125": 12795789312.0, "4130": 12795789312.0, "4135": 12795789312.0, "4140": 12795789312.0, "4145": 12795789312.0, "4150": 12795789312.0, "4155": 12795789312.0, "4160": 12795789312.0, "4165": 12795789312.0, "4170": 12795789312.0, "4175": 12795789312.0, "4180": 12795789312.0, "4185": 12795789312.0, "4190": 12795789312.0, "4195": 12795789312.0, "4200": 12795789312.0, "4205": 12795789312.0, "4210": 12795789312.0, "4215": 12795789312.0, "4220": 12795789312.0, "4225": 12795789312.0, "4230": 12795789312.0, "4235": 12795789312.0, "4240": 12795789312.0, "4245": 12795789312.0, "4250": 12795789312.0, "4255": 12795789312.0, "4260": 12795789312.0, "4265": 12795789312.0, "4270": 12795789312.0, "4275": 12795789312.0, "4280": 12795789312.0, "4285": 12795789312.0, "4290": 12795789312.0, "4295": 12795789312.0, "4300": 12795789312.0, "4305": 12795789312.0, "4310": 12795789312.0, "4315": 12795789312.0, "4320": 12795789312.0, "4325": 12795789312.0, "4330": 12795789312.0, "4335": 12795789312.0, "4340": 12795789312.0, "4345": 12795789312.0, "4350": 12795789312.0, "4355": 12795789312.0, "4360": 12795789312.0, "4365": 12795789312.0, "4370": 12795789312.0, "4375": 12795789312.0, "4380": 12795789312.0, "4385": 12795789312.0, "4390": 12795789312.0, "4395": 12795789312.0, "4400": 12795789312.0, "4405": 12795789312.0, "4410": 12795789312.0, "4415": 12795789312.0, "4420": 12795789312.0, "4425": 12795789312.0, "4430": 12795789312.0, "4435": 12795789312.0, "4440": 12795789312.0, "4445": 12795789312.0, "4450": 12795789312.0, "4455": 12795789312.0, "4460": 12795789312.0, "4465": 12795789312.0, "4470": 12795789312.0, "4475": 12795789312.0, "4480": 12795789312.0, "4485": 12795789312.0, "4490": 12795789312.0, "4495": 12795789312.0, "4500": 12795789312.0, "4505": 12795789312.0, "4510": 12795789312.0, "4515": 12795789312.0, "4520": 12795789312.0, "4525": 12795789312.0, "4530": 12795789312.0, "4535": 12795789312.0, "4540": 12795789312.0, "4545": 12795789312.0, "4550": 12795789312.0, "4555": 12795789312.0, "4560": 12795789312.0, "4565": 12795789312.0, "4570": 12795789312.0, "4575": 12795789312.0, "4580": 12795789312.0, "4585": 12795789312.0, "4590": 12795789312.0, "4595": 12795789312.0, "4600": 12795789312.0, "4605": 12795789312.0, "4610": 12795789312.0, "4615": 12795789312.0, "4620": 12795789312.0, "4625": 12795789312.0, "4630": 12795789312.0, "4635": 12795789312.0, "4640": 12795789312.0, "4645": 12795789312.0, "4650": 12795789312.0, "4655": 12795789312.0, "4660": 12795789312.0, "4665": 12795789312.0, "4670": 12795789312.0, "4675": 12795789312.0, "4680": 12795789312.0, "4685": 12795789312.0, "4690": 12795789312.0, "4695": 12795789312.0, "4700": 12795789312.0, "4705": 12795789312.0, "4710": 12795789312.0, "4715": 12795789312.0, "4720": 12795789312.0, "4725": 12795789312.0, "4730": 12795789312.0, "4735": 12795789312.0, "4740": 12795789312.0, "4745": 12795789312.0, "4750": 12795789312.0, "4755": 12795789312.0, "4760": 12795789312.0, "4765": 12795789312.0, "4770": 12795789312.0, "4775": 12795789312.0, "4780": 12795789312.0, "4785": 12795789312.0, "4790": 12795789312.0, "4795": 12795789312.0, "4800": 12795789312.0, "4805": 12795789312.0, "4810": 12795789312.0, "4815": 12795789312.0, "4820": 12795789312.0, "4825": 12795789312.0, "4830": 12795789312.0, "4835": 12795789312.0, "4840": 12795789312.0, "4845": 12795789312.0, "4850": 12795789312.0, "4855": 12795789312.0, "4860": 12795789312.0, "4865": 12795789312.0, "4870": 12795789312.0, "4875": 12795789312.0, "4880": 12795789312.0, "4885": 12795789312.0, "4890": 12795789312.0, "4895": 12795789312.0, "4900": 12795789312.0, "4905": 12795789312.0, "4910": 12795789312.0, "4915": 12795789312.0, "4920": 12795789312.0, "4925": 12795789312.0, "4930": 12795789312.0, "4935": 12795789312.0, "4940": 12795789312.0, "4945": 12795789312.0, "4950": 12795789312.0, "4955": 12795789312.0, "4960": 12795789312.0, "4965": 12795789312.0, "4970": 12795789312.0, "4975": 12795789312.0, "4980": 12795789312.0, "4985": 12795789312.0, "4990": 12795789312.0, "4995": 12795789312.0, "5000": 12795789312.0, "5005": 12795789312.0, "5010": 12795789312.0, "5015": 12795789312.0, "5020": 12795789312.0, "5025": 12795789312.0, "5030": 12795789312.0, "5035": 12795789312.0, "5040": 12795789312.0, "5045": 12795789312.0, "5050": 12795789312.0, "5055": 12795789312.0, "5060": 12795789312.0, "5065": 12795789312.0, "5070": 12795789312.0, "5075": 12795789312.0, "5080": 12795789312.0, "5085": 12795789312.0, "5090": 12795789312.0, "5095": 12795789312.0, "5100": 12795789312.0, "5105": 12795789312.0, "5110": 12795789312.0, "5115": 12795789312.0, "5120": 12795789312.0, "5125": 12795789312.0, "5130": 12795789312.0, "5135": 12795789312.0, "5140": 12795789312.0, "5145": 12795789312.0, "5150": 12795789312.0, "5155": 12795789312.0, "5160": 12795789312.0, "5165": 12795789312.0, "5170": 12795789312.0, "5175": 12795789312.0, "5180": 12795789312.0, "5185": 12795789312.0, "5190": 12795789312.0, "5195": 12795789312.0, "5200": 12795789312.0, "5205": 12795789312.0, "5210": 12795789312.0, "5215": 12795789312.0, "5220": 12795789312.0, "5225": 12795789312.0, "5230": 12795789312.0, "5235": 12795789312.0, "5240": 12795789312.0, "5245": 12795789312.0, "5250": 12795789312.0, "5255": 12795789312.0, "5260": 12795789312.0, "5265": 12795789312.0, "5270": 12795789312.0, "5275": 12795789312.0, "5280": 12795789312.0, "5285": 12795789312.0, "5290": 12795789312.0, "5295": 12795789312.0, "5300": 12795789312.0, "5305": 12795789312.0, "5310": 12795789312.0, "5315": 12795789312.0, "5320": 12795789312.0, "5325": 12795789312.0, "5330": 12795789312.0, "5335": 12795789312.0, "5340": 12795789312.0, "5345": 12795789312.0, "5350": 12795789312.0, "5355": 12795789312.0, "5360": 12795789312.0, "5365": 12795789312.0, "5370": 12795789312.0, "5375": 12795789312.0, "5380": 12795789312.0, "5385": 12795789312.0, "5390": 12795789312.0, "5395": 12795789312.0, "5400": 12795789312.0, "5405": 12795789312.0, "5410": 12795789312.0, "5415": 12795789312.0, "5420": 12795789312.0, "5425": 12795789312.0, "5430": 12795789312.0, "5435": 12795789312.0, "5440": 12795789312.0, "5445": 12795789312.0, "5450": 12795789312.0, "5455": 12795789312.0, "5460": 12795789312.0, "5465": 12795789312.0, "5470": 12795789312.0, "5475": 12795789312.0, "5480": 12795789312.0, "5485": 12795789312.0, "5490": 12795789312.0, "5495": 12795789312.0, "5500": 12795789312.0, "5505": 12795789312.0, "5510": 12795789312.0, "5515": 12795789312.0, "5520": 12795789312.0, "5525": 12795789312.0, "5530": 12795789312.0, "5535": 12795789312.0, "5540": 12795789312.0, "5545": 12795789312.0, "5550": 12795789312.0, "5555": 12795789312.0, "5560": 12795789312.0, "5565": 12795789312.0, "5570": 12795789312.0, "5575": 12795789312.0, "5580": 12795789312.0, "5585": 12795789312.0, "5590": 12795789312.0, "5595": 12795789312.0, "5600": 12795789312.0, "5605": 12795789312.0, "5610": 12795789312.0, "5615": 12795789312.0, "5620": 12795789312.0, "5625": 12795789312.0, "5630": 12795789312.0, "5635": 12795789312.0, "5640": 12795789312.0, "5645": 12795789312.0, "5650": 12795789312.0, "5655": 12795789312.0, "5660": 12795789312.0, "5665": 12795789312.0, "5670": 12795789312.0, "5675": 12795789312.0, "5680": 12795789312.0, "5685": 12795789312.0, "5690": 12795789312.0, "5695": 12795789312.0, "5700": 12795789312.0, "5705": 12795789312.0, "5710": 12795789312.0, "5715": 12795789312.0, "5720": 12795789312.0, "5725": 12795789312.0, "5730": 12795789312.0, "5735": 12795789312.0, "5740": 12795789312.0, "5745": 12795789312.0, "5750": 12795789312.0, "5755": 12795789312.0, "5760": 12795789312.0, "5765": 12795789312.0, "5770": 12795789312.0, "5775": 12795789312.0, "5780": 12795789312.0, "5785": 12795789312.0, "5790": 12795789312.0, "5795": 12795789312.0, "5800": 12795789312.0, "5805": 12795789312.0, "5810": 12795789312.0, "5815": 12795789312.0, "5820": 12795789312.0, "5825": 12795789312.0, "5830": 12795789312.0, "5835": 12795789312.0, "5840": 12795789312.0, "5845": 12795789312.0, "5850": 12795789312.0, "5855": 12795789312.0, "5860": 12795789312.0, "5865": 12795789312.0, "5870": 12795789312.0, "5875": 12795789312.0, "5880": 12795789312.0, "5885": 12795789312.0, "5890": 12795789312.0, "5895": 12795789312.0, "5900": 12795789312.0, "5905": 12795789312.0, "5910": 12795789312.0, "5915": 12795789312.0, "5920": 12795789312.0, "5925": 12795789312.0, "5930": 12795789312.0, "5935": 12795789312.0, "5940": 12795789312.0, "5945": 12795789312.0, "5950": 12795789312.0, "5955": 12795789312.0, "5960": 12795789312.0, "5965": 12795789312.0, "5970": 12795789312.0, "5975": 12795789312.0, "5980": 12795789312.0, "5985": 12795789312.0, "5990": 12795789312.0, "5995": 12795789312.0, "6000": 12795789312.0, "6005": 12795789312.0, "6010": 12795789312.0, "6015": 12795789312.0, "6020": 12795789312.0, "6025": 12795789312.0, "6030": 12795789312.0, "6035": 12795789312.0, "6040": 12795789312.0, "6045": 12795789312.0, "6050": 12795789312.0, "6055": 12795789312.0, "6060": 12795789312.0, "6065": 12795789312.0, "6070": 12795789312.0, "6075": 12795789312.0, "6080": 12795789312.0, "6085": 12795789312.0, "6090": 12795789312.0, "6095": 12795789312.0, "6100": 12795789312.0, "6105": 12795789312.0, "6110": 12795789312.0, "6115": 12795789312.0, "6120": 12795789312.0, "6125": 12795789312.0, "6130": 12795789312.0, "6135": 12795789312.0, "6140": 12795789312.0, "6145": 12795789312.0, "6150": 12795789312.0, "6155": 12795789312.0, "6160": 12795789312.0, "6165": 12795789312.0, "6170": 12795789312.0, "6175": 12795789312.0, "6180": 12795789312.0, "6185": 12795789312.0, "6190": 12795789312.0, "6195": 12795789312.0, "6200": 12795789312.0, "6205": 12795789312.0, "6210": 12795789312.0, "6215": 12795789312.0, "6220": 12795789312.0, "6225": 12795789312.0, "6230": 12795789312.0, "6235": 12795789312.0, "6240": 12795789312.0, "6245": 12795789312.0, "6250": 12795789312.0, "6255": 12795789312.0, "6260": 12795789312.0, "6265": 12795789312.0, "6270": 12795789312.0, "6275": 12795789312.0, "6280": 12795789312.0, "6285": 12795789312.0, "6290": 12795789312.0, "6295": 12795789312.0, "6300": 12795789312.0, "6305": 12795789312.0, "6310": 12795789312.0, "6315": 12795789312.0, "6320": 12795789312.0, "6325": 12795789312.0, "6330": 12795789312.0, "6335": 12795789312.0, "6340": 12795789312.0, "6345": 12795789312.0, "6350": 12795789312.0, "6355": 12795789312.0, "6360": 12795789312.0, "6365": 12795789312.0, "6370": 12795789312.0, "6375": 12795789312.0, "6380": 12795789312.0, "6385": 12795789312.0, "6390": 12795789312.0, "6395": 12795789312.0, "6400": 12795789312.0, "6405": 12795789312.0, "6410": 12795789312.0, "6415": 12795789312.0, "6420": 12795789312.0, "6425": 12795789312.0, "6430": 12795789312.0, "6435": 12795789312.0, "6440": 12795789312.0, "6445": 12795789312.0, "6450": 12795789312.0, "6455": 12795789312.0, "6460": 12795789312.0, "6465": 12795789312.0, "6470": 12795789312.0, "6475": 12795789312.0, "6480": 12795789312.0, "6485": 12795789312.0, "6490": 12795789312.0, "6495": 12795789312.0, "6500": 12795789312.0, "6505": 12795789312.0, "6510": 12795789312.0, "6515": 12795789312.0, "6520": 12795789312.0, "6525": 12795789312.0, "6530": 12795789312.0, "6535": 12795789312.0, "6540": 12795789312.0, "6545": 12795789312.0, "6550": 12795789312.0, "6555": 12795789312.0, "6560": 12795789312.0, "6565": 12795789312.0, "6570": 12795789312.0, "6575": 12795789312.0, "6580": 12795789312.0, "6585": 12795789312.0, "6590": 12795789312.0, "6595": 12795789312.0, "6600": 12795789312.0, "6605": 12795789312.0, "6610": 12795789312.0, "6615": 12795789312.0, "6620": 12795789312.0, "6625": 12795789312.0, "6630": 12795789312.0, "6635": 12795789312.0, "6640": 12795789312.0, "6645": 12795789312.0, "6650": 12795789312.0, "6655": 12795789312.0, "6660": 12795789312.0, "6665": 12795789312.0, "6670": 12795789312.0, "6675": 12795789312.0, "6680": 12795789312.0, "6685": 12795789312.0, "6690": 12795789312.0, "6695": 12795789312.0, "6700": 12795789312.0, "6705": 12795789312.0, "6710": 12795789312.0, "6715": 12795789312.0, "6720": 12795789312.0, "6725": 12795789312.0, "6730": 12795789312.0, "6735": 12795789312.0, "6740": 12795789312.0, "6745": 12795789312.0, "6750": 12795789312.0, "6755": 12795789312.0, "6760": 12795789312.0, "6765": 12795789312.0, "6770": 12795789312.0, "6775": 12795789312.0, "6780": 12795789312.0, "6785": 12795789312.0, "6790": 12795789312.0, "6795": 12795789312.0, "6800": 12795789312.0, "6805": 12795789312.0, "6810": 12795789312.0, "6815": 12795789312.0, "6820": 12795789312.0, "6825": 12795789312.0, "6830": 12795789312.0, "6835": 12795789312.0, "6840": 12795789312.0, "6845": 12795789312.0, "6850": 12795789312.0, "6855": 12795789312.0, "6860": 12795789312.0, "6865": 12795789312.0, "6870": 12795789312.0, "6875": 12795789312.0, "6880": 12795789312.0, "6885": 12795789312.0, "6890": 12795789312.0, "6895": 12795789312.0, "6900": 12795789312.0, "6905": 12795789312.0, "6910": 12795789312.0, "6915": 12795789312.0, "6920": 12795789312.0, "6925": 12795789312.0, "6930": 12795789312.0, "6935": 12795789312.0, "6940": 12795789312.0, "6945": 12795789312.0, "6950": 12795789312.0, "6955": 12795789312.0, "6960": 12795789312.0, "6965": 12795789312.0, "6970": 12795789312.0, "6975": 12795789312.0, "6980": 12795789312.0, "6985": 12795789312.0, "6990": 12795789312.0, "6995": 12795789312.0, "7000": 12795789312.0, "7005": 12795789312.0, "7010": 12795789312.0, "7015": 12795789312.0, "7020": 12795789312.0, "7025": 12795789312.0, "7030": 12795789312.0, "7035": 12795789312.0, "7040": 12795789312.0, "7045": 12795789312.0, "7050": 12795789312.0, "7055": 12795789312.0, "7060": 12795789312.0, "7065": 12795789312.0, "7070": 12795789312.0, "7075": 12795789312.0, "7080": 12795789312.0, "7085": 12795789312.0, "7090": 12795789312.0, "7095": 12795789312.0, "7100": 12795789312.0, "7105": 12795789312.0, "7110": 12795789312.0, "7115": 12795789312.0, "7120": 12795789312.0, "7125": 12795789312.0, "7130": 12795789312.0, "7135": 12795789312.0, "7140": 12795789312.0, "7145": 12795789312.0, "7150": 12795789312.0, "7155": 12795789312.0, "7160": 12795789312.0, "7165": 12795789312.0, "7170": 12795789312.0, "7175": 12795789312.0, "7180": 12795789312.0, "7185": 12795789312.0, "7190": 12795789312.0, "7195": 12795789312.0, "7200": 12795789312.0, "7205": 12795789312.0, "7210": 12795789312.0, "7215": 12795789312.0, "7220": 12795789312.0, "7225": 12795789312.0, "7230": 12795789312.0, "7235": 12795789312.0, "7240": 12795789312.0, "7245": 12795789312.0, "7250": 12795789312.0, "7255": 12795789312.0, "7260": 12795789312.0, "7265": 12795789312.0, "7270": 12795789312.0, "7275": 12795789312.0, "7280": 12795789312.0, "7285": 12795789312.0, "7290": 12795789312.0, "7295": 12795789312.0, "7300": 12795789312.0, "7305": 12795789312.0, "7310": 12795789312.0, "7315": 12795789312.0, "7320": 12795789312.0, "7325": 12795789312.0, "7330": 12795789312.0, "7335": 12795789312.0, "7340": 12795789312.0, "7345": 12795789312.0, "7350": 12795789312.0, "7355": 12795789312.0, "7360": 12795789312.0, "7365": 12795789312.0, "7370": 12795789312.0, "7375": 12795789312.0, "7380": 12795789312.0, "7385": 12795789312.0, "7390": 12795789312.0, "7395": 12795789312.0, "7400": 12795789312.0, "7405": 12795789312.0, "7410": 12795789312.0, "7415": 12795789312.0, "7420": 12795789312.0, "7425": 12795789312.0, "7430": 12795789312.0, "7435": 12795789312.0, "7440": 12795789312.0, "7445": 12795789312.0, "7450": 12795789312.0, "7455": 12795789312.0, "7460": 12795789312.0, "7465": 12795789312.0, "7470": 12795789312.0, "7475": 12795789312.0, "7480": 12795789312.0, "7485": 12795789312.0, "7490": 12795789312.0, "7495": 12795789312.0, "7500": 12795789312.0, "7505": 12795789312.0, "7510": 12795789312.0, "7515": 12795789312.0, "7520": 12795789312.0, "7525": 12795789312.0, "7530": 12795789312.0, "7535": 12795789312.0, "7540": 12795789312.0, "7545": 12795789312.0, "7550": 12795789312.0, "7555": 12795789312.0, "7560": 12795789312.0, "7565": 12795789312.0, "7570": 12795789312.0, "7575": 12795789312.0, "7580": 12795789312.0, "7585": 12795789312.0, "7590": 12795789312.0, "7595": 12795789312.0, "7600": 12795789312.0, "7605": 12795789312.0, "7610": 12795789312.0, "7615": 12795789312.0, "7620": 12795789312.0, "7625": 12795789312.0, "7630": 12795789312.0, "7635": 12795789312.0, "7640": 12795789312.0, "7645": 12795789312.0, "7650": 12795789312.0, "7655": 12795789312.0, "7660": 12795789312.0, "7665": 12795789312.0, "7670": 12795789312.0, "7675": 12795789312.0, "7680": 12795789312.0, "7685": 12795789312.0, "7690": 12795789312.0, "7695": 12795789312.0, "7700": 12795789312.0, "7705": 12795789312.0, "7710": 12795789312.0, "7715": 12795789312.0, "7720": 12795789312.0, "7725": 12795789312.0, "7730": 12795789312.0, "7735": 12795789312.0, "7740": 12795789312.0, "7745": 12795789312.0, "7750": 12795789312.0, "7755": 12795789312.0, "7760": 12795789312.0, "7765": 12795789312.0, "7770": 12795789312.0, "7775": 12795789312.0, "7780": 12795789312.0, "7785": 12795789312.0, "7790": 12795789312.0, "7795": 12795789312.0, "7800": 12795789312.0, "7805": 12795789312.0, "7810": 12795789312.0, "7815": 12795789312.0, "7820": 12795789312.0, "7825": 12795789312.0, "7830": 12795789312.0, "7835": 12795789312.0, "7840": 12795789312.0, "7845": 12795789312.0, "7850": 12795789312.0, "7855": 12795789312.0, "7860": 12795789312.0, "7865": 12795789312.0, "7870": 12795789312.0, "7875": 12795789312.0, "7880": 12795789312.0, "7885": 12795789312.0, "7890": 12795789312.0, "7895": 12795789312.0, "7900": 12795789312.0, "7905": 12795789312.0, "7910": 12795789312.0, "7915": 12795789312.0, "7920": 12795789312.0, "7925": 12795789312.0, "7930": 12795789312.0, "7935": 12795789312.0, "7940": 12795789312.0, "7945": 12795789312.0, "7950": 12795789312.0, "7955": 12795789312.0, "7960": 12795789312.0, "7965": 12795789312.0, "7970": 12795789312.0, "7975": 12795789312.0, "7980": 12795789312.0, "7985": 12795789312.0, "7990": 12795789312.0, "7995": 12795789312.0, "8000": 12795789312.0, "8005": 12795789312.0, "8010": 12795789312.0, "8015": 12795789312.0, "8020": 12795789312.0, "8025": 12795789312.0, "8030": 12795789312.0, "8035": 12795789312.0, "8040": 12795789312.0, "8045": 12795789312.0, "8050": 12795789312.0, "8055": 12795789312.0, "8060": 12795789312.0, "8065": 12795789312.0, "8070": 12795789312.0, "8075": 12795789312.0, "8080": 12795789312.0, "8085": 12795789312.0, "8090": 12795789312.0, "8095": 12795789312.0, "8100": 12795789312.0, "8105": 12795789312.0, "8110": 12795789312.0, "8115": 12795789312.0, "8120": 12795789312.0, "8125": 12795789312.0, "8130": 12795789312.0, "8135": 12795789312.0, "8140": 12795789312.0, "8145": 12795789312.0, "8150": 12795789312.0, "8155": 12795789312.0, "8160": 12795789312.0, "8165": 12795789312.0, "8170": 12795789312.0, "8175": 12795789312.0, "8180": 12795789312.0, "8185": 12795789312.0, "8190": 12795789312.0, "8195": 12795789312.0, "8200": 12795789312.0, "8205": 12795789312.0, "8210": 12795789312.0, "8215": 12795789312.0, "8220": 12795789312.0, "8225": 12795789312.0, "8230": 12795789312.0, "8235": 12795789312.0, "8240": 12795789312.0, "8245": 12795789312.0, "8250": 12795789312.0, "8255": 12795789312.0, "8260": 12795789312.0, "8265": 12795789312.0, "8270": 12795789312.0, "8275": 12795789312.0, "8280": 12795789312.0, "8285": 12795789312.0, "8290": 12795789312.0, "8295": 12795789312.0, "8300": 12795789312.0, "8305": 12795789312.0, "8310": 12795789312.0, "8315": 12795789312.0, "8320": 12795789312.0, "8325": 12795789312.0, "8330": 12795789312.0, "8335": 12795789312.0, "8340": 12795789312.0, "8345": 12795789312.0, "8350": 12795789312.0, "8355": 12795789312.0, "8360": 12795789312.0, "8365": 12795789312.0, "8370": 12795789312.0, "8375": 12795789312.0, "8380": 12795789312.0, "8385": 12795789312.0, "8390": 12795789312.0, "8395": 12795789312.0, "8400": 12795789312.0, "8405": 12795789312.0, "8410": 12795789312.0, "8415": 12795789312.0, "8420": 12795789312.0, "8425": 12795789312.0, "8430": 12795789312.0, "8435": 12795789312.0, "8440": 12795789312.0, "8445": 12795789312.0, "8450": 12795789312.0, "8455": 12795789312.0, "8460": 12795789312.0, "8465": 12795789312.0, "8470": 12795789312.0, "8475": 12795789312.0, "8480": 12795789312.0, "8485": 12795789312.0, "8490": 12795789312.0, "8495": 12795789312.0, "8500": 12795789312.0, "8505": 12795789312.0, "8510": 12795789312.0, "8515": 12795789312.0, "8520": 12795789312.0, "8525": 12795789312.0, "8530": 12795789312.0, "8535": 12795789312.0, "8540": 12795789312.0, "8545": 12795789312.0, "8550": 12795789312.0, "8555": 12795789312.0, "8560": 12795789312.0, "8565": 12795789312.0, "8570": 12795789312.0, "8575": 12795789312.0, "8580": 12795789312.0, "8585": 12795789312.0, "8590": 12795789312.0, "8595": 12795789312.0, "8600": 12795789312.0, "8605": 12795789312.0, "8610": 12795789312.0, "8615": 12795789312.0, "8620": 12795789312.0, "8625": 12795789312.0, "8630": 12795789312.0, "8635": 12795789312.0, "8640": 12795789312.0, "8645": 12795789312.0, "8650": 12795789312.0, "8655": 12795789312.0, "8660": 12795789312.0, "8665": 12795789312.0, "8670": 12795789312.0, "8675": 12795789312.0, "8680": 12795789312.0, "8685": 12795789312.0, "8690": 12795789312.0, "8695": 12795789312.0, "8700": 12795789312.0, "8705": 12795789312.0, "8710": 12795789312.0, "8715": 12795789312.0, "8720": 12795789312.0, "8725": 12795789312.0, "8730": 12795789312.0, "8735": 12795789312.0, "8740": 12795789312.0, "8745": 12795789312.0, "8750": 12795789312.0, "8755": 12795789312.0, "8760": 12795789312.0, "8765": 12795789312.0, "8770": 12795789312.0, "8775": 12795789312.0, "8780": 12795789312.0, "8785": 12795789312.0, "8790": 12795789312.0, "8795": 12795789312.0, "8800": 12795789312.0, "8805": 12795789312.0, "8810": 12795789312.0, "8815": 12795789312.0, "8820": 12795789312.0, "8825": 12795789312.0, "8830": 12795789312.0, "8835": 12795789312.0, "8840": 12795789312.0, "8845": 12795789312.0, "8850": 12795789312.0, "8855": 12795789312.0, "8860": 12795789312.0, "8865": 12795789312.0, "8870": 12795789312.0, "8875": 12795789312.0, "8880": 12795789312.0, "8885": 12795789312.0, "8890": 12795789312.0, "8895": 12795789312.0, "8900": 12795789312.0, "8905": 12795789312.0, "8910": 12795789312.0, "8915": 12795789312.0, "8920": 12795789312.0, "8925": 12795789312.0, "8930": 12795789312.0, "8935": 12795789312.0, "8940": 12795789312.0, "8945": 12795789312.0, "8950": 12795789312.0, "8955": 12795789312.0, "8960": 12795789312.0, "8965": 12795789312.0, "8970": 12795789312.0, "8975": 12795789312.0, "8980": 12795789312.0, "8985": 12795789312.0, "8990": 12795789312.0, "8995": 12795789312.0, "9000": 12795789312.0, "9005": 12795789312.0, "9010": 12795789312.0, "9015": 12795789312.0, "9020": 12795789312.0, "9025": 12795789312.0, "9030": 12795789312.0, "9035": 12795789312.0, "9040": 12795789312.0, "9045": 12795789312.0, "9050": 12795789312.0, "9055": 12795789312.0, "9060": 12795789312.0, "9065": 12795789312.0, "9070": 12795789312.0, "9075": 12795789312.0, "9080": 12795789312.0, "9085": 12795789312.0, "9090": 12795789312.0, "9095": 12795789312.0, "9100": 12795789312.0, "9105": 12795789312.0, "9110": 12795789312.0, "9115": 12795789312.0, "9120": 12795789312.0, "9125": 12795789312.0, "9130": 12795789312.0, "9135": 12795789312.0, "9140": 12795789312.0, "9145": 12795789312.0, "9150": 12795789312.0, "9155": 12795789312.0, "9160": 12795789312.0, "9165": 12795789312.0, "9170": 12795789312.0, "9175": 12795789312.0, "9180": 12795789312.0, "9185": 12795789312.0, "9190": 12795789312.0, "9195": 12795789312.0, "9200": 12795789312.0, "9205": 12795789312.0, "9210": 12795789312.0, "9215": 12795789312.0, "9220": 12795789312.0, "9225": 12795789312.0, "9230": 12795789312.0, "9235": 12795789312.0, "9240": 12795789312.0, "9245": 12795789312.0, "9250": 12795789312.0, "9255": 12795789312.0, "9260": 12795789312.0, "9265": 12795789312.0, "9270": 12795789312.0, "9275": 12795789312.0, "9280": 12795789312.0, "9285": 12795789312.0, "9290": 12795789312.0, "9295": 12795789312.0, "9300": 12795789312.0, "9305": 12795789312.0, "9310": 12795789312.0, "9315": 12795789312.0, "9320": 12795789312.0, "9325": 12795789312.0, "9330": 12795789312.0, "9335": 12795789312.0, "9340": 12795789312.0, "9345": 12795789312.0, "9350": 12795789312.0, "9355": 12795789312.0, "9360": 12795789312.0, "9365": 12795789312.0, "9370": 12795789312.0, "9375": 12795789312.0, "9380": 12795789312.0, "9385": 12795789312.0, "9390": 12795789312.0, "9395": 12795789312.0, "9400": 12795789312.0, "9405": 12795789312.0, "9410": 12795789312.0, "9415": 12795789312.0, "9420": 12795789312.0, "9425": 12795789312.0, "9430": 12795789312.0, "9435": 12795789312.0, "9440": 12795789312.0, "9445": 12795789312.0, "9450": 12795789312.0, "9455": 12795789312.0, "9460": 12795789312.0, "9465": 12795789312.0, "9470": 12795789312.0, "9475": 12795789312.0, "9480": 12795789312.0, "9485": 12795789312.0, "9490": 12795789312.0, "9495": 12795789312.0, "9500": 12795789312.0, "9505": 12795789312.0, "9510": 12795789312.0, "9515": 12795789312.0, "9520": 12795789312.0, "9525": 12795789312.0, "9530": 12795789312.0, "9535": 12795789312.0, "9540": 12795789312.0, "9545": 12795789312.0, "9550": 12795789312.0, "9555": 12795789312.0, "9560": 12795789312.0, "9565": 12795789312.0, "9570": 12795789312.0, "9575": 12795789312.0, "9580": 12795789312.0, "9585": 12795789312.0, "9590": 12795789312.0, "9595": 12795789312.0, "9600": 12795789312.0, "9605": 12795789312.0, "9610": 12795789312.0, "9615": 12795789312.0, "9620": 12795789312.0, "9625": 12795789312.0, "9630": 12795789312.0, "9635": 12795789312.0, "9640": 12795789312.0, "9645": 12795789312.0, "9650": 12795789312.0, "9655": 12795789312.0, "9660": 12795789312.0, "9665": 12795789312.0, "9670": 12795789312.0, "9675": 12795789312.0, "9680": 12795789312.0, "9685": 12795789312.0, "9690": 12795789312.0, "9695": 12795789312.0, "9700": 12795789312.0, "9705": 12795789312.0, "9710": 12795789312.0, "9715": 12795789312.0, "9720": 12795789312.0, "9725": 12795789312.0, "9730": 12795789312.0, "9735": 12795789312.0, "9740": 12795789312.0, "9745": 12795789312.0, "9750": 12795789312.0, "9755": 12795789312.0, "9760": 12795789312.0, "9765": 12795789312.0, "9770": 12795789312.0, "9775": 12795789312.0, "9780": 12795789312.0, "9785": 12795789312.0, "9790": 12795789312.0, "9795": 12795789312.0, "9800": 12795789312.0, "9805": 12795789312.0, "9810": 12795789312.0, "9815": 12795789312.0, "9820": 12795789312.0, "9825": 12795789312.0, "9830": 12795789312.0, "9835": 12795789312.0, "9840": 12795789312.0, "9845": 12795789312.0, "9850": 12795789312.0, "9855": 12795789312.0, "9860": 12795789312.0, "9865": 12795789312.0, "9870": 12795789312.0, "9875": 12795789312.0, "9880": 12795789312.0, "9885": 12795789312.0, "9890": 12795789312.0, "9895": 12795789312.0, "9900": 12795789312.0, "9905": 12795789312.0, "9910": 12795789312.0, "9915": 12795789312.0, "9920": 12795789312.0, "9925": 12795789312.0, "9930": 12795789312.0, "9935": 12795789312.0, "9940": 12795789312.0, "9945": 12795789312.0, "9950": 12795789312.0, "9955": 12795789312.0, "9960": 12795789312.0, "9965": 12795789312.0, "9970": 12795789312.0, "9975": 12795789312.0, "9980": 12795789312.0, "9985": 12795789312.0, "9990": 12795789312.0, "9995": 12795789312.0, "10000": 12795789312.0, "10005": 12795789312.0, "10010": 12795789312.0, "10015": 12795789312.0, "10020": 12795789312.0, "10025": 12795789312.0, "10030": 12795789312.0, "10035": 12795789312.0, "10040": 12795789312.0, "10045": 12795789312.0, "10050": 12795789312.0, "10055": 12795789312.0, "10060": 12795789312.0, "10065": 12795789312.0, "10070": 12795789312.0, "10075": 12795789312.0, "10080": 12795789312.0, "10085": 12795789312.0, "10090": 12795789312.0, "10095": 12795789312.0, "10100": 12795789312.0, "10105": 12795789312.0, "10110": 12795789312.0, "10115": 12795789312.0, "10120": 12795789312.0, "10125": 12795789312.0, "10130": 12795789312.0, "10135": 12795789312.0, "10140": 12795789312.0, "10145": 12795789312.0, "10150": 12795789312.0, "10155": 12795789312.0, "10160": 12795789312.0, "10165": 12795789312.0, "10170": 12795789312.0, "10175": 12795789312.0, "10180": 12795789312.0, "10185": 12795789312.0, "10190": 12795789312.0, "10195": 12795789312.0, "10200": 12795789312.0, "10205": 12795789312.0, "10210": 12795789312.0, "10215": 12795789312.0, "10220": 12795789312.0, "10225": 12795789312.0, "10230": 12795789312.0, "10235": 12795789312.0, "10240": 12795789312.0, "10245": 12795789312.0, "10250": 12795789312.0, "10255": 12795789312.0, "10260": 12795789312.0, "10265": 12795789312.0, "10270": 12795789312.0, "10275": 12795789312.0, "10280": 12795789312.0, "10285": 12795789312.0, "10290": 12795789312.0, "10295": 12795789312.0, "10300": 12795789312.0, "10305": 12795789312.0, "10310": 12795789312.0, "10315": 12795789312.0, "10320": 12795789312.0, "10325": 12795789312.0, "10330": 12795789312.0, "10335": 12795789312.0, "10340": 12795789312.0, "10345": 12795789312.0, "10350": 12795789312.0, "10355": 12795789312.0, "10360": 12795789312.0, "10365": 12795789312.0, "10370": 12795789312.0, "10375": 12795789312.0, "10380": 12795789312.0, "10385": 12795789312.0, "10390": 12795789312.0, "10395": 12795789312.0, "10400": 12795789312.0, "10405": 12795789312.0, "10410": 12795789312.0, "10415": 12795789312.0, "10420": 12795789312.0, "10425": 12795789312.0, "10430": 12795789312.0, "10435": 12795789312.0, "10440": 12795789312.0, "10445": 12795789312.0, "10450": 12795789312.0, "10455": 12795789312.0, "10460": 12795789312.0, "10465": 12795789312.0, "10470": 12795789312.0, "10475": 12795789312.0, "10480": 12795789312.0, "10485": 12795789312.0, "10490": 12795789312.0, "10495": 12795789312.0, "10500": 12795789312.0, "10505": 12795789312.0, "10510": 12795789312.0, "10515": 12795789312.0, "10520": 12795789312.0, "10525": 12795789312.0, "10530": 12795789312.0, "10535": 12795789312.0, "10540": 12795789312.0, "10545": 12795789312.0, "10550": 12795789312.0, "10555": 12795789312.0, "10560": 12795789312.0, "10565": 12795789312.0, "10570": 12795789312.0, "10575": 12795789312.0, "10580": 12795789312.0, "10585": 12795789312.0, "10590": 12795789312.0, "10595": 12795789312.0, "10600": 12795789312.0, "10605": 12795789312.0, "10610": 12795789312.0, "10615": 12795789312.0, "10620": 12795789312.0, "10625": 12795789312.0, "10630": 12795789312.0, "10635": 12795789312.0, "10640": 12795789312.0, "10645": 12795789312.0, "10650": 12795789312.0, "10655": 12795789312.0, "10660": 12795789312.0, "10665": 12795789312.0, "10670": 12795789312.0, "10675": 12795789312.0, "10680": 12795789312.0, "10685": 12795789312.0, "10690": 12795789312.0, "10695": 12795789312.0, "10700": 12795789312.0, "10705": 12795789312.0, "10710": 12795789312.0, "10715": 12795789312.0, "10720": 12795789312.0, "10725": 12795789312.0, "10730": 12795789312.0, "10735": 12795789312.0, "10740": 12795789312.0, "10745": 12795789312.0, "10750": 12795789312.0, "10755": 12795789312.0, "10760": 12795789312.0, "10765": 12795789312.0, "10770": 12795789312.0, "10775": 12795789312.0, "10780": 12795789312.0, "10785": 12795789312.0, "10790": 12795789312.0, "10795": 12795789312.0, "10800": 12795789312.0, "10805": 12795789312.0, "10810": 12795789312.0, "10815": 12795789312.0, "10820": 12795789312.0, "10825": 12795789312.0, "10830": 12795789312.0, "10835": 12795789312.0, "10840": 12795789312.0, "10845": 12795789312.0, "10850": 12795789312.0, "10855": 12795789312.0, "10860": 12795789312.0, "10865": 12795789312.0, "10870": 12795789312.0, "10875": 12795789312.0, "10880": 12795789312.0, "10885": 12795789312.0, "10890": 12795789312.0, "10895": 12795789312.0, "10900": 12795789312.0, "10905": 12795789312.0, "10910": 12795789312.0, "10915": 12795789312.0, "10920": 12795789312.0, "10925": 12795789312.0, "10930": 12795789312.0, "10935": 12795789312.0, "10940": 12795789312.0, "10945": 12795789312.0, "10950": 12795789312.0, "10955": 12795789312.0, "10960": 12795789312.0, "10965": 12795789312.0, "10970": 12795789312.0, "10975": 12795789312.0, "10980": 12795789312.0, "10985": 12795789312.0, "10990": 12795789312.0, "10995": 12795789312.0, "11000": 12795789312.0, "11005": 12795789312.0, "11010": 12795789312.0, "11015": 12795789312.0, "11020": 12795789312.0, "11025": 12795789312.0, "11030": 12795789312.0, "11035": 12795789312.0, "11040": 12795789312.0, "11045": 12795789312.0, "11050": 12795789312.0, "11055": 12795789312.0, "11060": 12795789312.0, "11065": 12795789312.0, "11070": 12795789312.0, "11075": 12795789312.0, "11080": 12795789312.0, "11085": 12795789312.0, "11090": 12795789312.0, "11095": 12795789312.0, "11100": 12795789312.0, "11105": 12795789312.0, "11110": 12795789312.0, "11115": 12795789312.0, "11120": 12795789312.0, "11125": 12795789312.0, "11130": 12795789312.0, "11135": 12795789312.0, "11140": 12795789312.0, "11145": 12795789312.0, "11150": 12795789312.0, "11155": 12795789312.0, "11160": 12795789312.0, "11165": 12795789312.0, "11170": 12795789312.0, "11175": 12795789312.0, "11180": 12795789312.0, "11185": 12795789312.0, "11190": 12795789312.0, "11195": 12795789312.0, "11200": 12795789312.0, "11205": 12795789312.0, "11210": 12795789312.0, "11215": 12795789312.0, "11220": 12795789312.0, "11225": 12795789312.0, "11230": 12795789312.0, "11235": 12795789312.0, "11240": 12795789312.0, "11245": 12795789312.0, "11250": 12795789312.0, "11255": 12795789312.0, "11260": 12795789312.0, "11265": 12795789312.0, "11270": 12795789312.0, "11275": 12795789312.0, "11280": 12795789312.0, "11285": 12795789312.0, "11290": 12795789312.0, "11295": 12795789312.0, "11300": 12795789312.0, "11305": 12795789312.0, "11310": 12795789312.0, "11315": 12795789312.0, "11320": 12795789312.0, "11325": 12795789312.0, "11330": 12795789312.0, "11335": 12795789312.0, "11340": 12795789312.0, "11345": 12795789312.0, "11350": 12795789312.0, "11355": 12795789312.0, "11360": 12795789312.0, "11365": 12795789312.0, "11370": 12795789312.0, "11375": 12795789312.0, "11380": 12795789312.0, "11385": 12795789312.0, "11390": 12795789312.0, "11395": 12795789312.0, "11400": 12795789312.0, "11405": 12795789312.0, "11410": 12795789312.0, "11415": 12795789312.0, "11420": 12795789312.0, "11425": 12795789312.0, "11430": 12795789312.0, "11435": 12795789312.0, "11440": 12795789312.0, "11445": 12795789312.0, "11450": 12795789312.0, "11455": 12795789312.0, "11460": 12795789312.0, "11465": 12795789312.0, "11470": 12795789312.0, "11475": 12795789312.0, "11480": 12795789312.0, "11485": 12795789312.0, "11490": 12795789312.0, "11495": 12795789312.0, "11500": 12795789312.0, "11505": 12795789312.0, "11510": 12795789312.0, "11515": 12795789312.0, "11520": 12795789312.0, "11525": 12795789312.0, "11530": 12795789312.0, "11535": 12795789312.0, "11540": 12795789312.0, "11545": 12795789312.0, "11550": 12795789312.0, "11555": 12795789312.0, "11560": 12795789312.0, "11565": 12795789312.0, "11570": 12795789312.0, "11575": 12795789312.0, "11580": 12795789312.0, "11585": 12795789312.0, "11590": 12795789312.0, "11595": 12795789312.0, "11600": 12795789312.0, "11605": 12795789312.0, "11610": 12795789312.0, "11615": 12795789312.0, "11620": 12795789312.0, "11625": 12795789312.0, "11630": 12795789312.0, "11635": 12795789312.0, "11640": 12795789312.0, "11645": 12795789312.0, "11650": 12795789312.0, "11655": 12795789312.0, "11660": 12795789312.0, "11665": 12795789312.0, "11670": 12795789312.0, "11675": 12795789312.0, "11680": 12795789312.0, "11685": 12795789312.0, "11690": 12795789312.0, "11695": 12795789312.0, "11700": 12795789312.0, "11705": 12795789312.0, "11710": 12795789312.0, "11715": 12795789312.0, "11720": 12795789312.0, "11725": 12795789312.0, "11730": 12795789312.0, "11735": 12795789312.0, "11740": 12795789312.0, "11745": 12795789312.0, "11750": 12795789312.0, "11755": 12795789312.0, "11760": 12795789312.0, "11765": 12795789312.0, "11770": 12795789312.0, "11775": 12795789312.0, "11780": 12795789312.0, "11785": 12795789312.0, "11790": 12795789312.0, "11795": 12795789312.0, "11800": 12795789312.0, "11805": 12795789312.0, "11810": 12795789312.0, "11815": 12795789312.0, "11820": 12795789312.0, "11825": 12795789312.0, "11830": 12795789312.0, "11835": 12795789312.0, "11840": 12795789312.0, "11845": 12795789312.0, "11850": 12795789312.0, "11855": 12795789312.0, "11860": 12795789312.0, "11865": 12795789312.0, "11870": 12795789312.0, "11875": 12795789312.0, "11880": 12795789312.0, "11885": 12795789312.0, "11890": 12795789312.0, "11895": 12795789312.0, "11900": 12795789312.0, "11905": 12795789312.0, "11910": 12795789312.0, "11915": 12795789312.0, "11920": 12795789312.0, "11925": 12795789312.0, "11930": 12795789312.0, "11935": 12795789312.0, "11940": 12795789312.0, "11945": 12795789312.0, "11950": 12795789312.0, "11955": 12795789312.0, "11960": 12795789312.0, "11965": 12795789312.0, "11970": 12795789312.0, "11975": 12795789312.0, "11980": 12795789312.0, "11985": 12795789312.0, "11990": 12795789312.0, "11995": 12795789312.0, "12000": 12795789312.0, "12005": 12795789312.0, "12010": 12795789312.0, "12015": 12795789312.0, "12020": 12795789312.0, "12025": 12795789312.0, "12030": 12795789312.0, "12035": 12795789312.0, "12040": 12795789312.0, "12045": 12795789312.0, "12050": 12795789312.0, "12055": 12795789312.0, "12060": 12795789312.0, "12065": 12795789312.0, "12070": 12795789312.0, "12075": 12795789312.0, "12080": 12795789312.0, "12085": 12795789312.0, "12090": 12795789312.0, "12095": 12795789312.0, "12100": 12795789312.0, "12105": 12795789312.0, "12110": 12795789312.0, "12115": 12795789312.0, "12120": 12795789312.0, "12125": 12795789312.0, "12130": 12795789312.0, "12135": 12795789312.0, "12140": 12795789312.0, "12145": 12795789312.0, "12150": 12795789312.0, "12155": 12795789312.0, "12160": 12795789312.0, "12165": 12795789312.0, "12170": 12795789312.0, "12175": 12795789312.0, "12180": 12795789312.0, "12185": 12795789312.0, "12190": 12795789312.0, "12195": 12795789312.0, "12200": 12795789312.0, "12205": 12795789312.0, "12210": 12795789312.0, "12215": 12795789312.0, "12220": 12795789312.0, "12225": 12795789312.0, "12230": 12795789312.0, "12235": 12795789312.0, "12240": 12795789312.0, "12245": 12795789312.0, "12250": 12795789312.0, "12255": 12795789312.0, "12260": 12795789312.0, "12265": 12795789312.0, "12270": 12795789312.0, "12275": 12795789312.0, "12280": 12795789312.0, "12285": 12795789312.0, "12290": 12795789312.0, "12295": 12795789312.0, "12300": 12795789312.0, "12305": 12795789312.0, "12310": 12795789312.0, "12315": 12795789312.0, "12320": 12795789312.0, "12325": 12795789312.0, "12330": 12795789312.0, "12335": 12795789312.0, "12340": 12795789312.0, "12345": 12795789312.0, "12350": 12795789312.0, "12355": 12795789312.0, "12360": 12795789312.0, "12365": 12795789312.0, "12370": 12795789312.0, "12375": 12795789312.0, "12380": 12795789312.0, "12385": 12795789312.0, "12390": 12795789312.0, "12395": 12795789312.0, "12400": 12795789312.0, "12405": 12795789312.0, "12410": 12795789312.0, "12415": 12795789312.0, "12420": 12795789312.0, "12425": 12795789312.0, "12430": 12795789312.0, "12435": 12795789312.0, "12440": 12795789312.0, "12445": 12795789312.0, "12450": 12795789312.0, "12455": 12795789312.0, "12460": 12795789312.0, "12465": 12795789312.0, "12470": 12795789312.0, "12475": 12795789312.0, "12480": 12795789312.0, "12485": 12795789312.0, "12490": 12795789312.0, "12495": 12795789312.0, "12500": 12795789312.0, "12505": 12795789312.0, "12510": 12795789312.0, "12515": 12795789312.0, "12520": 12795789312.0, "12525": 12795789312.0, "12530": 12795789312.0, "12535": 12795789312.0, "12540": 12795789312.0, "12545": 12795789312.0, "12550": 12795789312.0, "12555": 12795789312.0, "12560": 12795789312.0, "12565": 12795789312.0, "12570": 12795789312.0, "12575": 12795789312.0, "12580": 12795789312.0, "12585": 12795789312.0, "12590": 12795789312.0, "12595": 12795789312.0, "12600": 12795789312.0, "12605": 12795789312.0, "12610": 12795789312.0, "12615": 12795789312.0, "12620": 12795789312.0, "12625": 12795789312.0, "12630": 12795789312.0, "12635": 12795789312.0, "12640": 12795789312.0, "12645": 12795789312.0, "12650": 12795789312.0, "12655": 12795789312.0, "12660": 12795789312.0, "12665": 12795789312.0, "12670": 12795789312.0, "12675": 12795789312.0, "12680": 12795789312.0, "12685": 12795789312.0, "12690": 12795789312.0, "12695": 12795789312.0, "12700": 12795789312.0, "12705": 12795789312.0, "12710": 12795789312.0, "12715": 12795789312.0, "12720": "nan", "12725": "nan", "12730": "nan", "12735": "nan", "12740": "nan", "12745": "nan", "12750": "nan", "12755": "nan", "12760": "nan", "12765": "nan", "12770": "nan", "12775": "nan", "12780": "nan", "12785": "nan", "12790": "nan", "12795": "nan", "12800": "nan", "12805": "nan", "12810": "nan", "12815": "nan", "12820": "nan", "12825": "nan", "12830": "nan", "12835": "nan", "12840": "nan", "12845": "nan", "12850": "nan", "12855": "nan", "12860": "nan", "12865": "nan", "12870": "nan", "12875": "nan", "12880": "nan", "12885": "nan", "12890": "nan", "12895": "nan", "12900": "nan", "12905": "nan", "12910": "nan", "12915": "nan", "12920": "nan", "12925": "nan", "12930": "nan", "12935": "nan", "12940": "nan", "12945": "nan", "12950": "nan", "12955": "nan", "12960": "nan", "12965": "nan", "12970": "nan", "12975": "nan", "12980": "nan", "12985": "nan", "12990": "nan", "12995": "nan", "13000": "nan"}}, "mem-max-allocated-bytes": {"start_step": 1, "end_step": 13000, "step_interval": 5, "values": {"1": 27991298048.0, "5": 28489385984.0, "10": 28489385984.0, "15": 28489385984.0, "20": 28489385984.0, "25": 28489385984.0, "30": 28489385984.0, "35": 28489385984.0, "40": 28489385984.0, "45": 28489385984.0, "50": 28489385984.0, "55": 28489385984.0, "60": 28489385984.0, "65": 28489385984.0, "70": 28489385984.0, "75": 28489385984.0, "80": 28489385984.0, "85": 28489385984.0, "90": 28489385984.0, "95": 28489385984.0, "100": 28489385984.0, "105": 28489385984.0, "110": 28489385984.0, "115": 28489385984.0, "120": 28489385984.0, "125": 28489385984.0, "130": 28489385984.0, "135": 28489385984.0, "140": 28489385984.0, "145": 28489385984.0, "150": 28489385984.0, "155": 28489385984.0, "160": 28489385984.0, "165": 28489385984.0, "170": 28489385984.0, "175": 28489385984.0, "180": 28489385984.0, "185": 28489385984.0, "190": 28489385984.0, "195": 28489385984.0, "200": 28489385984.0, "205": 28489385984.0, "210": 28489385984.0, "215": 28489385984.0, "220": 28489385984.0, "225": 28489385984.0, "230": 28489385984.0, "235": 28489385984.0, "240": 28489385984.0, "245": 28489385984.0, "250": 28489385984.0, "255": 28489385984.0, "260": 28489385984.0, "265": 28489385984.0, "270": 28489385984.0, "275": 28489385984.0, "280": 28489385984.0, "285": 28489385984.0, "290": 28489385984.0, "295": 28489385984.0, "300": 28489385984.0, "305": 28489385984.0, "310": 28489385984.0, "315": 28489385984.0, "320": 28489385984.0, "325": 28489385984.0, "330": 28489385984.0, "335": 28489385984.0, "340": 28489385984.0, "345": 28489385984.0, "350": 28489385984.0, "355": 28489385984.0, "360": 28489385984.0, "365": 28489385984.0, "370": 28489385984.0, "375": 28489385984.0, "380": 28489385984.0, "385": 28489385984.0, "390": 28489385984.0, "395": 28489385984.0, "400": 28489385984.0, "405": 28489385984.0, "410": 28489385984.0, "415": 28489385984.0, "420": 28489385984.0, "425": 28489385984.0, "430": 28489385984.0, "435": 28489385984.0, "440": 28489385984.0, "445": 28489385984.0, "450": 28489385984.0, "455": 28489385984.0, "460": 28489385984.0, "465": 28489385984.0, "470": 28489385984.0, "475": 28489385984.0, "480": 28489385984.0, "485": 28489385984.0, "490": 28489385984.0, "495": 28489385984.0, "500": 28489385984.0, "505": 28489385984.0, "510": 28489385984.0, "515": 28489385984.0, "520": 28489385984.0, "525": 28489385984.0, "530": 28489385984.0, "535": 28489385984.0, "540": 28489385984.0, "545": 28489385984.0, "550": 28489385984.0, "555": 28489385984.0, "560": 28489385984.0, "565": 28489385984.0, "570": 28489385984.0, "575": 28489385984.0, "580": 28489385984.0, "585": 28489385984.0, "590": 28489385984.0, "595": 28489385984.0, "600": 28489385984.0, "605": 28489385984.0, "610": 28489385984.0, "615": 28489385984.0, "620": 28489385984.0, "625": 28489385984.0, "630": 28489385984.0, "635": 28489385984.0, "640": 28489385984.0, "645": 28489385984.0, "650": 28489385984.0, "655": 28489385984.0, "660": 28489385984.0, "665": 28489385984.0, "670": 28489385984.0, "675": 28489385984.0, "680": 28489385984.0, "685": 28489385984.0, "690": 28489385984.0, "695": 28489385984.0, "700": 28489385984.0, "705": 28489385984.0, "710": 28489385984.0, "715": 28489385984.0, "720": 28489385984.0, "725": 28489385984.0, "730": 28489385984.0, "735": 28489385984.0, "740": 28489385984.0, "745": 28489385984.0, "750": 28489385984.0, "755": 28489385984.0, "760": 28489385984.0, "765": 28489385984.0, "770": 28489385984.0, "775": 28489385984.0, "780": 28489385984.0, "785": 28489385984.0, "790": 28489385984.0, "795": 28489385984.0, "800": 28489385984.0, "805": 28489385984.0, "810": 28489385984.0, "815": 28489385984.0, "820": 28489385984.0, "825": 28489385984.0, "830": 28489385984.0, "835": 28489385984.0, "840": 28489385984.0, "845": 28489385984.0, "850": 28489385984.0, "855": 28489385984.0, "860": 28489385984.0, "865": 28489385984.0, "870": 28489385984.0, "875": 28489385984.0, "880": 28489385984.0, "885": 28489385984.0, "890": 28489385984.0, "895": 28489385984.0, "900": 28489385984.0, "905": 28489385984.0, "910": 28489385984.0, "915": 28489385984.0, "920": 28489385984.0, "925": 28489385984.0, "930": 28489385984.0, "935": 28489385984.0, "940": 28489385984.0, "945": 28489385984.0, "950": 28489385984.0, "955": 28489385984.0, "960": 28489385984.0, "965": 28489385984.0, "970": 28489385984.0, "975": 28489385984.0, "980": 28489385984.0, "985": 28489385984.0, "990": 28489385984.0, "995": 28489385984.0, "1000": 28489385984.0, "1005": 28489385984.0, "1010": 28489385984.0, "1015": 28489385984.0, "1020": 28489385984.0, "1025": 28489385984.0, "1030": 28489385984.0, "1035": 28489385984.0, "1040": 28489385984.0, "1045": 28489385984.0, "1050": 28489385984.0, "1055": 28489385984.0, "1060": 28489385984.0, "1065": 28489385984.0, "1070": 28489385984.0, "1075": 28489385984.0, "1080": 28489385984.0, "1085": 28489385984.0, "1090": 28489385984.0, "1095": 28489385984.0, "1100": 28489385984.0, "1105": 28489385984.0, "1110": 28489385984.0, "1115": 28489385984.0, "1120": 28489385984.0, "1125": 28489385984.0, "1130": 28489385984.0, "1135": 28489385984.0, "1140": 28489385984.0, "1145": 28489385984.0, "1150": 28489385984.0, "1155": 28489385984.0, "1160": 28489385984.0, "1165": 28489385984.0, "1170": 28489385984.0, "1175": 28489385984.0, "1180": 28489385984.0, "1185": 28489385984.0, "1190": 28489385984.0, "1195": 28489385984.0, "1200": 28489385984.0, "1205": 28489385984.0, "1210": 28489385984.0, "1215": 28489385984.0, "1220": 28489385984.0, "1225": 28489385984.0, "1230": 28489385984.0, "1235": 28489385984.0, "1240": 28489385984.0, "1245": 28489385984.0, "1250": 28489385984.0, "1255": 28489385984.0, "1260": 28489385984.0, "1265": 28489385984.0, "1270": 28489385984.0, "1275": 28489385984.0, "1280": 28489385984.0, "1285": 28489385984.0, "1290": 28489385984.0, "1295": 28489385984.0, "1300": 28489385984.0, "1305": 28489385984.0, "1310": 28489385984.0, "1315": 28489385984.0, "1320": 28489385984.0, "1325": 28489385984.0, "1330": 28489385984.0, "1335": 28489385984.0, "1340": 28489385984.0, "1345": 28489385984.0, "1350": 28489385984.0, "1355": 28489385984.0, "1360": 28489385984.0, "1365": 28489385984.0, "1370": 28489385984.0, "1375": 28489385984.0, "1380": 28489385984.0, "1385": 28489385984.0, "1390": 28489385984.0, "1395": 28489385984.0, "1400": 28489385984.0, "1405": 28489385984.0, "1410": 28489385984.0, "1415": 28489385984.0, "1420": 28489385984.0, "1425": 28489385984.0, "1430": 28489385984.0, "1435": 28489385984.0, "1440": 28489385984.0, "1445": 28489385984.0, "1450": 28489385984.0, "1455": 28489385984.0, "1460": 28489385984.0, "1465": 28489385984.0, "1470": 28489385984.0, "1475": 28489385984.0, "1480": 28489385984.0, "1485": 28489385984.0, "1490": 28489385984.0, "1495": 28489385984.0, "1500": 28489385984.0, "1505": 28489385984.0, "1510": 28489385984.0, "1515": 28489385984.0, "1520": 28489385984.0, "1525": 28489385984.0, "1530": 28489385984.0, "1535": 28489385984.0, "1540": 28489385984.0, "1545": 28489385984.0, "1550": 28489385984.0, "1555": 28489385984.0, "1560": 28489385984.0, "1565": 28489385984.0, "1570": 28489385984.0, "1575": 28489385984.0, "1580": 28489385984.0, "1585": 28489385984.0, "1590": 28489385984.0, "1595": 28489385984.0, "1600": 28489385984.0, "1605": 28489385984.0, "1610": 28489385984.0, "1615": 28489385984.0, "1620": 28489385984.0, "1625": 28489385984.0, "1630": 28489385984.0, "1635": 28489385984.0, "1640": 28489385984.0, "1645": 28489385984.0, "1650": 28489385984.0, "1655": 28489385984.0, "1660": 28489385984.0, "1665": 28489385984.0, "1670": 28489385984.0, "1675": 28489385984.0, "1680": 28489385984.0, "1685": 28489385984.0, "1690": 28489385984.0, "1695": 28489385984.0, "1700": 28489385984.0, "1705": 28489385984.0, "1710": 28489385984.0, "1715": 28489385984.0, "1720": 28489385984.0, "1725": 28489385984.0, "1730": 28489385984.0, "1735": 28489385984.0, "1740": 28489385984.0, "1745": 28489385984.0, "1750": 28489385984.0, "1755": 28489385984.0, "1760": 28489385984.0, "1765": 28489385984.0, "1770": 28489385984.0, "1775": 28489385984.0, "1780": 28489385984.0, "1785": 28489385984.0, "1790": 28489385984.0, "1795": 28489385984.0, "1800": 28489385984.0, "1805": 28489385984.0, "1810": 28489385984.0, "1815": 28489385984.0, "1820": 28489385984.0, "1825": 28489385984.0, "1830": 28489385984.0, "1835": 28489385984.0, "1840": 28489385984.0, "1845": 28489385984.0, "1850": 28489385984.0, "1855": 28489385984.0, "1860": 28489385984.0, "1865": 28489385984.0, "1870": 28489385984.0, "1875": 28489385984.0, "1880": 28489385984.0, "1885": 28489385984.0, "1890": 28489385984.0, "1895": 28489385984.0, "1900": 28489385984.0, "1905": 28489385984.0, "1910": 28489385984.0, "1915": 28489385984.0, "1920": 28489385984.0, "1925": 28489385984.0, "1930": 28489385984.0, "1935": 28489385984.0, "1940": 28489385984.0, "1945": 28489385984.0, "1950": 28489385984.0, "1955": 28489385984.0, "1960": 28489385984.0, "1965": 28489385984.0, "1970": 28489385984.0, "1975": 28489385984.0, "1980": 28489385984.0, "1985": 28489385984.0, "1990": 28489385984.0, "1995": 28489385984.0, "2000": 28489385984.0, "2005": 28489385984.0, "2010": 28489385984.0, "2015": 28489385984.0, "2020": 28489385984.0, "2025": 28489385984.0, "2030": 28489385984.0, "2035": 28489385984.0, "2040": 28489385984.0, "2045": 28489385984.0, "2050": 28489385984.0, "2055": 28489385984.0, "2060": 28489385984.0, "2065": 28489385984.0, "2070": 28489385984.0, "2075": 28489385984.0, "2080": 28489385984.0, "2085": 28489385984.0, "2090": 28489385984.0, "2095": 28489385984.0, "2100": 28489385984.0, "2105": 28489385984.0, "2110": 28489385984.0, "2115": 28489385984.0, "2120": 28489385984.0, "2125": 28489385984.0, "2130": 28489385984.0, "2135": 28489385984.0, "2140": 28489385984.0, "2145": 28489385984.0, "2150": 28489385984.0, "2155": 28489385984.0, "2160": 28489385984.0, "2165": 28489385984.0, "2170": 28489385984.0, "2175": 28489385984.0, "2180": 28489385984.0, "2185": 28489385984.0, "2190": 28489385984.0, "2195": 28489385984.0, "2200": 28489385984.0, "2205": 28489385984.0, "2210": 28489385984.0, "2215": 28489385984.0, "2220": 28489385984.0, "2225": 28489385984.0, "2230": 28489385984.0, "2235": 28489385984.0, "2240": 28489385984.0, "2245": 28489385984.0, "2250": 28489385984.0, "2255": 28489385984.0, "2260": 28489385984.0, "2265": 28489385984.0, "2270": 28489385984.0, "2275": 28489385984.0, "2280": 28489385984.0, "2285": 28489385984.0, "2290": 28489385984.0, "2295": 28489385984.0, "2300": 28489385984.0, "2305": 28489385984.0, "2310": 28489385984.0, "2315": 28489385984.0, "2320": 28489385984.0, "2325": 28489385984.0, "2330": 28489385984.0, "2335": 28489385984.0, "2340": 28489385984.0, "2345": 28489385984.0, "2350": 28489385984.0, "2355": 28489385984.0, "2360": 28489385984.0, "2365": 28489385984.0, "2370": 28489385984.0, "2375": 28489385984.0, "2380": 28489385984.0, "2385": 28489385984.0, "2390": 28489385984.0, "2395": 28489385984.0, "2400": 28489385984.0, "2405": 28489385984.0, "2410": 28489385984.0, "2415": 28489385984.0, "2420": 28489385984.0, "2425": 28489385984.0, "2430": 28489385984.0, "2435": 28489385984.0, "2440": 28489385984.0, "2445": 28489385984.0, "2450": 28489385984.0, "2455": 28489385984.0, "2460": 28489385984.0, "2465": 28489385984.0, "2470": 28489385984.0, "2475": 28489385984.0, "2480": 28489385984.0, "2485": 28489385984.0, "2490": 28489385984.0, "2495": 28489385984.0, "2500": 28489385984.0, "2505": 28489385984.0, "2510": 28489385984.0, "2515": 28489385984.0, "2520": 28489385984.0, "2525": 28489385984.0, "2530": 28489385984.0, "2535": 28489385984.0, "2540": 28489385984.0, "2545": 28489385984.0, "2550": 28489385984.0, "2555": 28489385984.0, "2560": 28489385984.0, "2565": 28489385984.0, "2570": 28489385984.0, "2575": 28489385984.0, "2580": 28489385984.0, "2585": 28489385984.0, "2590": 28489385984.0, "2595": 28489385984.0, "2600": 28489385984.0, "2605": 28489385984.0, "2610": 28489385984.0, "2615": 28489385984.0, "2620": 28489385984.0, "2625": 28489385984.0, "2630": 28489385984.0, "2635": 28489385984.0, "2640": 28489385984.0, "2645": 28489385984.0, "2650": 28489385984.0, "2655": 28489385984.0, "2660": 28489385984.0, "2665": 28489385984.0, "2670": 28489385984.0, "2675": 28489385984.0, "2680": 28489385984.0, "2685": 28489385984.0, "2690": 28489385984.0, "2695": 28489385984.0, "2700": 28489385984.0, "2705": 28489385984.0, "2710": 28489385984.0, "2715": 28489385984.0, "2720": 28489385984.0, "2725": 28489385984.0, "2730": 28489385984.0, "2735": 28489385984.0, "2740": 28489385984.0, "2745": 28489385984.0, "2750": 28489385984.0, "2755": 28489385984.0, "2760": 28489385984.0, "2765": 28489385984.0, "2770": 28489385984.0, "2775": 28489385984.0, "2780": 28489385984.0, "2785": 28489385984.0, "2790": 28489385984.0, "2795": 28489385984.0, "2800": 28489385984.0, "2805": 28489385984.0, "2810": 28489385984.0, "2815": 28489385984.0, "2820": 28489385984.0, "2825": 28489385984.0, "2830": 28489385984.0, "2835": 28489385984.0, "2840": 28489385984.0, "2845": 28489385984.0, "2850": 28489385984.0, "2855": 28489385984.0, "2860": 28489385984.0, "2865": 28489385984.0, "2870": 28489385984.0, "2875": 28489385984.0, "2880": 28489385984.0, "2885": 28489385984.0, "2890": 28489385984.0, "2895": 28489385984.0, "2900": 28489385984.0, "2905": 28489385984.0, "2910": 28489385984.0, "2915": 28489385984.0, "2920": 28489385984.0, "2925": 28489385984.0, "2930": 28489385984.0, "2935": 28489385984.0, "2940": 28489385984.0, "2945": 28489385984.0, "2950": 28489385984.0, "2955": 28489385984.0, "2960": 28489385984.0, "2965": 28489385984.0, "2970": 28489385984.0, "2975": 28489385984.0, "2980": 28489385984.0, "2985": 28489385984.0, "2990": 28489385984.0, "2995": 28489385984.0, "3000": 28489385984.0, "3005": 28489385984.0, "3010": 28489385984.0, "3015": 28489385984.0, "3020": 28489385984.0, "3025": 28489385984.0, "3030": 28489385984.0, "3035": 28489385984.0, "3040": 28489385984.0, "3045": 28489385984.0, "3050": 28489385984.0, "3055": 28489385984.0, "3060": 28489385984.0, "3065": 28489385984.0, "3070": 28489385984.0, "3075": 28489385984.0, "3080": 28489385984.0, "3085": 28489385984.0, "3090": 28489385984.0, "3095": 28489385984.0, "3100": 28489385984.0, "3105": 28489385984.0, "3110": 28489385984.0, "3115": 28489385984.0, "3120": 28489385984.0, "3125": 28489385984.0, "3130": 28489385984.0, "3135": 28489385984.0, "3140": 28489385984.0, "3145": 28489385984.0, "3150": 28489385984.0, "3155": 28489385984.0, "3160": 28489385984.0, "3165": 28489385984.0, "3170": 28489385984.0, "3175": 28489385984.0, "3180": 28489385984.0, "3185": 28489385984.0, "3190": 28489385984.0, "3195": 28489385984.0, "3200": 28489385984.0, "3205": 28489385984.0, "3210": 28489385984.0, "3215": 28489385984.0, "3220": 28489385984.0, "3225": 28489385984.0, "3230": 28489385984.0, "3235": 28489385984.0, "3240": 28489385984.0, "3245": 28489385984.0, "3250": 28489385984.0, "3255": 28489385984.0, "3260": 28489385984.0, "3265": 28489385984.0, "3270": 28489385984.0, "3275": 28489385984.0, "3280": 28489385984.0, "3285": 28489385984.0, "3290": 28489385984.0, "3295": 28489385984.0, "3300": 28489385984.0, "3305": 28489385984.0, "3310": 28489385984.0, "3315": 28489385984.0, "3320": 28489385984.0, "3325": 28489385984.0, "3330": 28489385984.0, "3335": 28489385984.0, "3340": 28489385984.0, "3345": 28489385984.0, "3350": 28489385984.0, "3355": 28489385984.0, "3360": 28489385984.0, "3365": 28489385984.0, "3370": 28489385984.0, "3375": 28489385984.0, "3380": 28489385984.0, "3385": 28489385984.0, "3390": 28489385984.0, "3395": 28489385984.0, "3400": 28489385984.0, "3405": 28489385984.0, "3410": 28489385984.0, "3415": 28489385984.0, "3420": 28489385984.0, "3425": 28489385984.0, "3430": 28489385984.0, "3435": 28489385984.0, "3440": 28489385984.0, "3445": 28489385984.0, "3450": 28489385984.0, "3455": 28489385984.0, "3460": 28489385984.0, "3465": 28489385984.0, "3470": 28489385984.0, "3475": 28489385984.0, "3480": 28489385984.0, "3485": 28489385984.0, "3490": 28489385984.0, "3495": 28489385984.0, "3500": 28489385984.0, "3505": 28489385984.0, "3510": 28489385984.0, "3515": 28489385984.0, "3520": 28489385984.0, "3525": 28489385984.0, "3530": 28489385984.0, "3535": 28489385984.0, "3540": 28489385984.0, "3545": 28489385984.0, "3550": 28489385984.0, "3555": 28489385984.0, "3560": 28489385984.0, "3565": 28489385984.0, "3570": 28489568256.0, "3575": 28489568256.0, "3580": 28489568256.0, "3585": 28489568256.0, "3590": 28489568256.0, "3595": 28489568256.0, "3600": 28489568256.0, "3605": 28489568256.0, "3610": 28489568256.0, "3615": 28489568256.0, "3620": 28489568256.0, "3625": 28489568256.0, "3630": 28489568256.0, "3635": 28489568256.0, "3640": 28489568256.0, "3645": 28489568256.0, "3650": 28489568256.0, "3655": 28489568256.0, "3660": 28489568256.0, "3665": 28489568256.0, "3670": 28489568256.0, "3675": 28489568256.0, "3680": 28489568256.0, "3685": 28489568256.0, "3690": 28489568256.0, "3695": 28489568256.0, "3700": 28489568256.0, "3705": 28489568256.0, "3710": 28489568256.0, "3715": 28489568256.0, "3720": 28489568256.0, "3725": 28489568256.0, "3730": 28489568256.0, "3735": 28489568256.0, "3740": 28489568256.0, "3745": 28489568256.0, "3750": 28489568256.0, "3755": 28489568256.0, "3760": 28489568256.0, "3765": 28489568256.0, "3770": 28489568256.0, "3775": 28489568256.0, "3780": 28489568256.0, "3785": 28489568256.0, "3790": 28489568256.0, "3795": 28489568256.0, "3800": 28489568256.0, "3805": 28489568256.0, "3810": 28489568256.0, "3815": 28489568256.0, "3820": 28489568256.0, "3825": 28489568256.0, "3830": 28489568256.0, "3835": 28489568256.0, "3840": 28489568256.0, "3845": 28489568256.0, "3850": 28489568256.0, "3855": 28489568256.0, "3860": 28489568256.0, "3865": 28489568256.0, "3870": 28489568256.0, "3875": 28489568256.0, "3880": 28489568256.0, "3885": 28489568256.0, "3890": 28489568256.0, "3895": 28489568256.0, "3900": 28489568256.0, "3905": 28489568256.0, "3910": 28489568256.0, "3915": 28489568256.0, "3920": 28489568256.0, "3925": 28489568256.0, "3930": 28489568256.0, "3935": 28489568256.0, "3940": 28489568256.0, "3945": 28489568256.0, "3950": 28489568256.0, "3955": 28489568256.0, "3960": 28489568256.0, "3965": 28489568256.0, "3970": 28489568256.0, "3975": 28489568256.0, "3980": 28489568256.0, "3985": 28489568256.0, "3990": 28489568256.0, "3995": 28489568256.0, "4000": 28489568256.0, "4005": 28489568256.0, "4010": 28489568256.0, "4015": 28489568256.0, "4020": 28489568256.0, "4025": 28489568256.0, "4030": 28489568256.0, "4035": 28489568256.0, "4040": 28489568256.0, "4045": 28489568256.0, "4050": 28489568256.0, "4055": 28489568256.0, "4060": 28489568256.0, "4065": 28489568256.0, "4070": 28489568256.0, "4075": 28489568256.0, "4080": 28489568256.0, "4085": 28489568256.0, "4090": 28489568256.0, "4095": 28489568256.0, "4100": 28489568256.0, "4105": 28489568256.0, "4110": 28489568256.0, "4115": 28489568256.0, "4120": 28489568256.0, "4125": 28489568256.0, "4130": 28489568256.0, "4135": 28489568256.0, "4140": 28489568256.0, "4145": 28489568256.0, "4150": 28489568256.0, "4155": 28489568256.0, "4160": 28489568256.0, "4165": 28489568256.0, "4170": 28489568256.0, "4175": 28489568256.0, "4180": 28489568256.0, "4185": 28489568256.0, "4190": 28489568256.0, "4195": 28489568256.0, "4200": 28489568256.0, "4205": 28489568256.0, "4210": 28489568256.0, "4215": 28489568256.0, "4220": 28489568256.0, "4225": 28489568256.0, "4230": 28489568256.0, "4235": 28489568256.0, "4240": 28489568256.0, "4245": 28489568256.0, "4250": 28489568256.0, "4255": 28489568256.0, "4260": 28489568256.0, "4265": 28489568256.0, "4270": 28489568256.0, "4275": 28489568256.0, "4280": 28489568256.0, "4285": 28489568256.0, "4290": 28489568256.0, "4295": 28489568256.0, "4300": 28489568256.0, "4305": 28489568256.0, "4310": 28489568256.0, "4315": 28489568256.0, "4320": 28489568256.0, "4325": 28489568256.0, "4330": 28489568256.0, "4335": 28489568256.0, "4340": 28489568256.0, "4345": 28489568256.0, "4350": 28489568256.0, "4355": 28489568256.0, "4360": 28489568256.0, "4365": 28489568256.0, "4370": 28489568256.0, "4375": 28489568256.0, "4380": 28489568256.0, "4385": 28489568256.0, "4390": 28489568256.0, "4395": 28489568256.0, "4400": 28489568256.0, "4405": 28489568256.0, "4410": 28489568256.0, "4415": 28489568256.0, "4420": 28489568256.0, "4425": 28489568256.0, "4430": 28489568256.0, "4435": 28489568256.0, "4440": 28489568256.0, "4445": 28489568256.0, "4450": 28489568256.0, "4455": 28489568256.0, "4460": 28489568256.0, "4465": 28489568256.0, "4470": 28489568256.0, "4475": 28489568256.0, "4480": 28489568256.0, "4485": 28489568256.0, "4490": 28489568256.0, "4495": 28489568256.0, "4500": 28489568256.0, "4505": 28489568256.0, "4510": 28489568256.0, "4515": 28489568256.0, "4520": 28489568256.0, "4525": 28489568256.0, "4530": 28489568256.0, "4535": 28489568256.0, "4540": 28489568256.0, "4545": 28489568256.0, "4550": 28489568256.0, "4555": 28489568256.0, "4560": 28489568256.0, "4565": 28489568256.0, "4570": 28489568256.0, "4575": 28489568256.0, "4580": 28489568256.0, "4585": 28489568256.0, "4590": 28489568256.0, "4595": 28489568256.0, "4600": 28489568256.0, "4605": 28489568256.0, "4610": 28489568256.0, "4615": 28489568256.0, "4620": 28489568256.0, "4625": 28489568256.0, "4630": 28489568256.0, "4635": 28489568256.0, "4640": 28489568256.0, "4645": 28489568256.0, "4650": 28489568256.0, "4655": 28489568256.0, "4660": 28489568256.0, "4665": 28489568256.0, "4670": 28489568256.0, "4675": 28489568256.0, "4680": 28489568256.0, "4685": 28489568256.0, "4690": 28489568256.0, "4695": 28489568256.0, "4700": 28489568256.0, "4705": 28489568256.0, "4710": 28489568256.0, "4715": 28489568256.0, "4720": 28489568256.0, "4725": 28489568256.0, "4730": 28489568256.0, "4735": 28489568256.0, "4740": 28489568256.0, "4745": 28489568256.0, "4750": 28489568256.0, "4755": 28489568256.0, "4760": 28489568256.0, "4765": 28489568256.0, "4770": 28489568256.0, "4775": 28489568256.0, "4780": 28489568256.0, "4785": 28489568256.0, "4790": 28489568256.0, "4795": 28489568256.0, "4800": 28489568256.0, "4805": 28489568256.0, "4810": 28489568256.0, "4815": 28489568256.0, "4820": 28489568256.0, "4825": 28489568256.0, "4830": 28489568256.0, "4835": 28489568256.0, "4840": 28489568256.0, "4845": 28489568256.0, "4850": 28489568256.0, "4855": 28489568256.0, "4860": 28489568256.0, "4865": 28489568256.0, "4870": 28489568256.0, "4875": 28489568256.0, "4880": 28489568256.0, "4885": 28489568256.0, "4890": 28489568256.0, "4895": 28489568256.0, "4900": 28489568256.0, "4905": 28489568256.0, "4910": 28489568256.0, "4915": 28489568256.0, "4920": 28489568256.0, "4925": 28489568256.0, "4930": 28489568256.0, "4935": 28489568256.0, "4940": 28489568256.0, "4945": 28489568256.0, "4950": 28489568256.0, "4955": 28489568256.0, "4960": 28489568256.0, "4965": 28489568256.0, "4970": 28489568256.0, "4975": 28489568256.0, "4980": 28489568256.0, "4985": 28489568256.0, "4990": 28489568256.0, "4995": 28489568256.0, "5000": 28489568256.0, "5005": 28489568256.0, "5010": 28489568256.0, "5015": 28489568256.0, "5020": 28489568256.0, "5025": 28489568256.0, "5030": 28489568256.0, "5035": 28489568256.0, "5040": 28489568256.0, "5045": 28489568256.0, "5050": 28489568256.0, "5055": 28489568256.0, "5060": 28489568256.0, "5065": 28489568256.0, "5070": 28489568256.0, "5075": 28489568256.0, "5080": 28489568256.0, "5085": 28489568256.0, "5090": 28489568256.0, "5095": 28489568256.0, "5100": 28489568256.0, "5105": 28489568256.0, "5110": 28489568256.0, "5115": 28489568256.0, "5120": 28489568256.0, "5125": 28489568256.0, "5130": 28489568256.0, "5135": 28489568256.0, "5140": 28489568256.0, "5145": 28489568256.0, "5150": 28489568256.0, "5155": 28489568256.0, "5160": 28489568256.0, "5165": 28489568256.0, "5170": 28489568256.0, "5175": 28489568256.0, "5180": 28489568256.0, "5185": 28489568256.0, "5190": 28489568256.0, "5195": 28489568256.0, "5200": 28489568256.0, "5205": 28489568256.0, "5210": 28489568256.0, "5215": 28489568256.0, "5220": 28489568256.0, "5225": 28489568256.0, "5230": 28489568256.0, "5235": 28489568256.0, "5240": 28489568256.0, "5245": 28489568256.0, "5250": 28489568256.0, "5255": 28489568256.0, "5260": 28489568256.0, "5265": 28489568256.0, "5270": 28489568256.0, "5275": 28489568256.0, "5280": 28489568256.0, "5285": 28489568256.0, "5290": 28489568256.0, "5295": 28489568256.0, "5300": 28489568256.0, "5305": 28489568256.0, "5310": 28489568256.0, "5315": 28489568256.0, "5320": 28489568256.0, "5325": 28489568256.0, "5330": 28489568256.0, "5335": 28489568256.0, "5340": 28489568256.0, "5345": 28489568256.0, "5350": 28489568256.0, "5355": 28489568256.0, "5360": 28489568256.0, "5365": 28489568256.0, "5370": 28489568256.0, "5375": 28489568256.0, "5380": 28489568256.0, "5385": 28489568256.0, "5390": 28489568256.0, "5395": 28489568256.0, "5400": 28489568256.0, "5405": 28489568256.0, "5410": 28489568256.0, "5415": 28489568256.0, "5420": 28489568256.0, "5425": 28489568256.0, "5430": 28489568256.0, "5435": 28489568256.0, "5440": 28489568256.0, "5445": 28489568256.0, "5450": 28489568256.0, "5455": 28489568256.0, "5460": 28489568256.0, "5465": 28489568256.0, "5470": 28489568256.0, "5475": 28489568256.0, "5480": 28489568256.0, "5485": 28489568256.0, "5490": 28489568256.0, "5495": 28489568256.0, "5500": 28489568256.0, "5505": 28489568256.0, "5510": 28489568256.0, "5515": 28489568256.0, "5520": 28489568256.0, "5525": 28489568256.0, "5530": 28489568256.0, "5535": 28489568256.0, "5540": 28489568256.0, "5545": 28489568256.0, "5550": 28489568256.0, "5555": 28489568256.0, "5560": 28489568256.0, "5565": 28489568256.0, "5570": 28489568256.0, "5575": 28489568256.0, "5580": 28489568256.0, "5585": 28489568256.0, "5590": 28489568256.0, "5595": 28489568256.0, "5600": 28489568256.0, "5605": 28489568256.0, "5610": 28489568256.0, "5615": 28489568256.0, "5620": 28489568256.0, "5625": 28489568256.0, "5630": 28489568256.0, "5635": 28489568256.0, "5640": 28489568256.0, "5645": 28489568256.0, "5650": 28489568256.0, "5655": 28489568256.0, "5660": 28489568256.0, "5665": 28489568256.0, "5670": 28489568256.0, "5675": 28489568256.0, "5680": 28489568256.0, "5685": 28489568256.0, "5690": 28489568256.0, "5695": 28489568256.0, "5700": 28489568256.0, "5705": 28489568256.0, "5710": 28489568256.0, "5715": 28489568256.0, "5720": 28489568256.0, "5725": 28489568256.0, "5730": 28489568256.0, "5735": 28489568256.0, "5740": 28489568256.0, "5745": 28489568256.0, "5750": 28489568256.0, "5755": 28489568256.0, "5760": 28489568256.0, "5765": 28489568256.0, "5770": 28489568256.0, "5775": 28489568256.0, "5780": 28489568256.0, "5785": 28489568256.0, "5790": 28489568256.0, "5795": 28489568256.0, "5800": 28489568256.0, "5805": 28489568256.0, "5810": 28489568256.0, "5815": 28489568256.0, "5820": 28489568256.0, "5825": 28489568256.0, "5830": 28489568256.0, "5835": 28489568256.0, "5840": 28489568256.0, "5845": 28489568256.0, "5850": 28489568256.0, "5855": 28489568256.0, "5860": 28489568256.0, "5865": 28489568256.0, "5870": 28489568256.0, "5875": 28489568256.0, "5880": 28489568256.0, "5885": 28489568256.0, "5890": 28489568256.0, "5895": 28489568256.0, "5900": 28489568256.0, "5905": 28489568256.0, "5910": 28489568256.0, "5915": 28489568256.0, "5920": 28489568256.0, "5925": 28489568256.0, "5930": 28489568256.0, "5935": 28489568256.0, "5940": 28489568256.0, "5945": 28489568256.0, "5950": 28489568256.0, "5955": 28489568256.0, "5960": 28489568256.0, "5965": 28489568256.0, "5970": 28489568256.0, "5975": 28489568256.0, "5980": 28489568256.0, "5985": 28489568256.0, "5990": 28489568256.0, "5995": 28489568256.0, "6000": 28489568256.0, "6005": 28489568256.0, "6010": 28489568256.0, "6015": 28489568256.0, "6020": 28489568256.0, "6025": 28489568256.0, "6030": 28489568256.0, "6035": 28489568256.0, "6040": 28489568256.0, "6045": 28489568256.0, "6050": 28489568256.0, "6055": 28489568256.0, "6060": 28489568256.0, "6065": 28489568256.0, "6070": 28489568256.0, "6075": 28489568256.0, "6080": 28489568256.0, "6085": 28489568256.0, "6090": 28489568256.0, "6095": 28489568256.0, "6100": 28489568256.0, "6105": 28489568256.0, "6110": 28489568256.0, "6115": 28489568256.0, "6120": 28489568256.0, "6125": 28489568256.0, "6130": 28489568256.0, "6135": 28489568256.0, "6140": 28489568256.0, "6145": 28489568256.0, "6150": 28489568256.0, "6155": 28489568256.0, "6160": 28489568256.0, "6165": 28489568256.0, "6170": 28489568256.0, "6175": 28489568256.0, "6180": 28489568256.0, "6185": 28489568256.0, "6190": 28489568256.0, "6195": 28489568256.0, "6200": 28489568256.0, "6205": 28489568256.0, "6210": 28489568256.0, "6215": 28489568256.0, "6220": 28489568256.0, "6225": 28489568256.0, "6230": 28489568256.0, "6235": 28489568256.0, "6240": 28489568256.0, "6245": 28489568256.0, "6250": 28489568256.0, "6255": 28489568256.0, "6260": 28489568256.0, "6265": 28489568256.0, "6270": 28489568256.0, "6275": 28489568256.0, "6280": 28489568256.0, "6285": 28489568256.0, "6290": 28489568256.0, "6295": 28489568256.0, "6300": 28489568256.0, "6305": 28489568256.0, "6310": 28489568256.0, "6315": 28489568256.0, "6320": 28489568256.0, "6325": 28489568256.0, "6330": 28489568256.0, "6335": 28489568256.0, "6340": 28489568256.0, "6345": 28489568256.0, "6350": 28489568256.0, "6355": 28489568256.0, "6360": 28489568256.0, "6365": 28489568256.0, "6370": 28489568256.0, "6375": 28489568256.0, "6380": 28489568256.0, "6385": 28489568256.0, "6390": 28489568256.0, "6395": 28489568256.0, "6400": 28489568256.0, "6405": 28489568256.0, "6410": 28489568256.0, "6415": 28489568256.0, "6420": 28489568256.0, "6425": 28489568256.0, "6430": 28489568256.0, "6435": 28489568256.0, "6440": 28489568256.0, "6445": 28489568256.0, "6450": 28489568256.0, "6455": 28489568256.0, "6460": 28489568256.0, "6465": 28489568256.0, "6470": 28489568256.0, "6475": 28489568256.0, "6480": 28489568256.0, "6485": 28489568256.0, "6490": 28489568256.0, "6495": 28489568256.0, "6500": 28489568256.0, "6505": 28489568256.0, "6510": 28489568256.0, "6515": 28489568256.0, "6520": 28489568256.0, "6525": 28489568256.0, "6530": 28489568256.0, "6535": 28489568256.0, "6540": 28489568256.0, "6545": 28489568256.0, "6550": 28489568256.0, "6555": 28489568256.0, "6560": 28489568256.0, "6565": 28489568256.0, "6570": 28489568256.0, "6575": 28489568256.0, "6580": 28489568256.0, "6585": 28489568256.0, "6590": 28489568256.0, "6595": 28489568256.0, "6600": 28489568256.0, "6605": 28489568256.0, "6610": 28489568256.0, "6615": 28489568256.0, "6620": 28489568256.0, "6625": 28489568256.0, "6630": 28489568256.0, "6635": 28489568256.0, "6640": 28489568256.0, "6645": 28489568256.0, "6650": 28489568256.0, "6655": 28489568256.0, "6660": 28489568256.0, "6665": 28489568256.0, "6670": 28489568256.0, "6675": 28489568256.0, "6680": 28489568256.0, "6685": 28489568256.0, "6690": 28489568256.0, "6695": 28489568256.0, "6700": 28489568256.0, "6705": 28489568256.0, "6710": 28489568256.0, "6715": 28489568256.0, "6720": 28489568256.0, "6725": 28489568256.0, "6730": 28489568256.0, "6735": 28489568256.0, "6740": 28489568256.0, "6745": 28489568256.0, "6750": 28489568256.0, "6755": 28489568256.0, "6760": 28489568256.0, "6765": 28489568256.0, "6770": 28489568256.0, "6775": 28489568256.0, "6780": 28489568256.0, "6785": 28489568256.0, "6790": 28489568256.0, "6795": 28489568256.0, "6800": 28489568256.0, "6805": 28489568256.0, "6810": 28489568256.0, "6815": 28489568256.0, "6820": 28489568256.0, "6825": 28489568256.0, "6830": 28489568256.0, "6835": 28489568256.0, "6840": 28489568256.0, "6845": 28489568256.0, "6850": 28489568256.0, "6855": 28489568256.0, "6860": 28489568256.0, "6865": 28489568256.0, "6870": 28489568256.0, "6875": 28489568256.0, "6880": 28489568256.0, "6885": 28489568256.0, "6890": 28489568256.0, "6895": 28489568256.0, "6900": 28489568256.0, "6905": 28489568256.0, "6910": 28489568256.0, "6915": 28489568256.0, "6920": 28489568256.0, "6925": 28489568256.0, "6930": 28489568256.0, "6935": 28489568256.0, "6940": 28489568256.0, "6945": 28489568256.0, "6950": 28489568256.0, "6955": 28489568256.0, "6960": 28489568256.0, "6965": 28489568256.0, "6970": 28489568256.0, "6975": 28489568256.0, "6980": 28489568256.0, "6985": 28489568256.0, "6990": 28489568256.0, "6995": 28489568256.0, "7000": 28489568256.0, "7005": 28489568256.0, "7010": 28489568256.0, "7015": 28489568256.0, "7020": 28489568256.0, "7025": 28489568256.0, "7030": 28489568256.0, "7035": 28489568256.0, "7040": 28489568256.0, "7045": 28489568256.0, "7050": 28489568256.0, "7055": 28489568256.0, "7060": 28489568256.0, "7065": 28489568256.0, "7070": 28489568256.0, "7075": 28489568256.0, "7080": 28489568256.0, "7085": 28489568256.0, "7090": 28489568256.0, "7095": 28489568256.0, "7100": 28489568256.0, "7105": 28489568256.0, "7110": 28489568256.0, "7115": 28489568256.0, "7120": 28489568256.0, "7125": 28489568256.0, "7130": 28489568256.0, "7135": 28489568256.0, "7140": 28489568256.0, "7145": 28489568256.0, "7150": 28489568256.0, "7155": 28489568256.0, "7160": 28489568256.0, "7165": 28489568256.0, "7170": 28489568256.0, "7175": 28489568256.0, "7180": 28489568256.0, "7185": 28489568256.0, "7190": 28489568256.0, "7195": 28489568256.0, "7200": 28489568256.0, "7205": 28489568256.0, "7210": 28489568256.0, "7215": 28489568256.0, "7220": 28489568256.0, "7225": 28489568256.0, "7230": 28489568256.0, "7235": 28489568256.0, "7240": 28489568256.0, "7245": 28489568256.0, "7250": 28489568256.0, "7255": 28489568256.0, "7260": 28489568256.0, "7265": 28489568256.0, "7270": 28489568256.0, "7275": 28489568256.0, "7280": 28489568256.0, "7285": 28489568256.0, "7290": 28489568256.0, "7295": 28489568256.0, "7300": 28489568256.0, "7305": 28489568256.0, "7310": 28489568256.0, "7315": 28489568256.0, "7320": 28489568256.0, "7325": 28489568256.0, "7330": 28489568256.0, "7335": 28489568256.0, "7340": 28489568256.0, "7345": 28489568256.0, "7350": 28489568256.0, "7355": 28489568256.0, "7360": 28489568256.0, "7365": 28489568256.0, "7370": 28489568256.0, "7375": 28489568256.0, "7380": 28489568256.0, "7385": 28489568256.0, "7390": 28489568256.0, "7395": 28489568256.0, "7400": 28489568256.0, "7405": 28489568256.0, "7410": 28489568256.0, "7415": 28489568256.0, "7420": 28489568256.0, "7425": 28489568256.0, "7430": 28489568256.0, "7435": 28489568256.0, "7440": 28489568256.0, "7445": 28489568256.0, "7450": 28489568256.0, "7455": 28489568256.0, "7460": 28489568256.0, "7465": 28489568256.0, "7470": 28489568256.0, "7475": 28489568256.0, "7480": 28489568256.0, "7485": 28489568256.0, "7490": 28489568256.0, "7495": 28489568256.0, "7500": 28489568256.0, "7505": 28489568256.0, "7510": 28489568256.0, "7515": 28489568256.0, "7520": 28489568256.0, "7525": 28489568256.0, "7530": 28489568256.0, "7535": 28489568256.0, "7540": 28489568256.0, "7545": 28489568256.0, "7550": 28489568256.0, "7555": 28489568256.0, "7560": 28489568256.0, "7565": 28489568256.0, "7570": 28489568256.0, "7575": 28489568256.0, "7580": 28489568256.0, "7585": 28489568256.0, "7590": 28489568256.0, "7595": 28489568256.0, "7600": 28489568256.0, "7605": 28489568256.0, "7610": 28489568256.0, "7615": 28489568256.0, "7620": 28489568256.0, "7625": 28489568256.0, "7630": 28489568256.0, "7635": 28489568256.0, "7640": 28489568256.0, "7645": 28489568256.0, "7650": 28489568256.0, "7655": 28489568256.0, "7660": 28489568256.0, "7665": 28489568256.0, "7670": 28489568256.0, "7675": 28489568256.0, "7680": 28489568256.0, "7685": 28489568256.0, "7690": 28489568256.0, "7695": 28489568256.0, "7700": 28489568256.0, "7705": 28489568256.0, "7710": 28489568256.0, "7715": 28489568256.0, "7720": 28489568256.0, "7725": 28489568256.0, "7730": 28489568256.0, "7735": 28489568256.0, "7740": 28489568256.0, "7745": 28489568256.0, "7750": 28489568256.0, "7755": 28489568256.0, "7760": 28489568256.0, "7765": 28489568256.0, "7770": 28489568256.0, "7775": 28489568256.0, "7780": 28489568256.0, "7785": 28489568256.0, "7790": 28489568256.0, "7795": 28489568256.0, "7800": 28489568256.0, "7805": 28489568256.0, "7810": 28489568256.0, "7815": 28489568256.0, "7820": 28489568256.0, "7825": 28489568256.0, "7830": 28489568256.0, "7835": 28489568256.0, "7840": 28489568256.0, "7845": 28489568256.0, "7850": 28489568256.0, "7855": 28489568256.0, "7860": 28489568256.0, "7865": 28489568256.0, "7870": 28489568256.0, "7875": 28489568256.0, "7880": 28489568256.0, "7885": 28489568256.0, "7890": 28489568256.0, "7895": 28489568256.0, "7900": 28489568256.0, "7905": 28489568256.0, "7910": 28489568256.0, "7915": 28489568256.0, "7920": 28489568256.0, "7925": 28489568256.0, "7930": 28489568256.0, "7935": 28489568256.0, "7940": 28489568256.0, "7945": 28489568256.0, "7950": 28489568256.0, "7955": 28489568256.0, "7960": 28489568256.0, "7965": 28489568256.0, "7970": 28489568256.0, "7975": 28489568256.0, "7980": 28489568256.0, "7985": 28489568256.0, "7990": 28489568256.0, "7995": 28489568256.0, "8000": 28489568256.0, "8005": 28489568256.0, "8010": 28489568256.0, "8015": 28489568256.0, "8020": 28489568256.0, "8025": 28489568256.0, "8030": 28489568256.0, "8035": 28489568256.0, "8040": 28489568256.0, "8045": 28489568256.0, "8050": 28489568256.0, "8055": 28489568256.0, "8060": 28489568256.0, "8065": 28489568256.0, "8070": 28489568256.0, "8075": 28489568256.0, "8080": 28489568256.0, "8085": 28489568256.0, "8090": 28489568256.0, "8095": 28489568256.0, "8100": 28489568256.0, "8105": 28489568256.0, "8110": 28489568256.0, "8115": 28489568256.0, "8120": 28489568256.0, "8125": 28489568256.0, "8130": 28489568256.0, "8135": 28489568256.0, "8140": 28489568256.0, "8145": 28489568256.0, "8150": 28489568256.0, "8155": 28489568256.0, "8160": 28489568256.0, "8165": 28489568256.0, "8170": 28489568256.0, "8175": 28489568256.0, "8180": 28489568256.0, "8185": 28489568256.0, "8190": 28489568256.0, "8195": 28489568256.0, "8200": 28489568256.0, "8205": 28489568256.0, "8210": 28489568256.0, "8215": 28489568256.0, "8220": 28489568256.0, "8225": 28489568256.0, "8230": 28489568256.0, "8235": 28489568256.0, "8240": 28489568256.0, "8245": 28489568256.0, "8250": 28489568256.0, "8255": 28489568256.0, "8260": 28489568256.0, "8265": 28489568256.0, "8270": 28489568256.0, "8275": 28489568256.0, "8280": 28489568256.0, "8285": 28489568256.0, "8290": 28489568256.0, "8295": 28489568256.0, "8300": 28489568256.0, "8305": 28489568256.0, "8310": 28489568256.0, "8315": 28489568256.0, "8320": 28489568256.0, "8325": 28489568256.0, "8330": 28489568256.0, "8335": 28489568256.0, "8340": 28489568256.0, "8345": 28489568256.0, "8350": 28489568256.0, "8355": 28489568256.0, "8360": 28489568256.0, "8365": 28489568256.0, "8370": 28489568256.0, "8375": 28489568256.0, "8380": 28489568256.0, "8385": 28489568256.0, "8390": 28489568256.0, "8395": 28489568256.0, "8400": 28489568256.0, "8405": 28489568256.0, "8410": 28489568256.0, "8415": 28489568256.0, "8420": 28489568256.0, "8425": 28489568256.0, "8430": 28489568256.0, "8435": 28489568256.0, "8440": 28489568256.0, "8445": 28489568256.0, "8450": 28489568256.0, "8455": 28489568256.0, "8460": 28489568256.0, "8465": 28489568256.0, "8470": 28489568256.0, "8475": 28489568256.0, "8480": 28489568256.0, "8485": 28489568256.0, "8490": 28489568256.0, "8495": 28489568256.0, "8500": 28489568256.0, "8505": 28489568256.0, "8510": 28489568256.0, "8515": 28489568256.0, "8520": 28489568256.0, "8525": 28489568256.0, "8530": 28489568256.0, "8535": 28489568256.0, "8540": 28489568256.0, "8545": 28489568256.0, "8550": 28489568256.0, "8555": 28489568256.0, "8560": 28489568256.0, "8565": 28489568256.0, "8570": 28489568256.0, "8575": 28489568256.0, "8580": 28489568256.0, "8585": 28489568256.0, "8590": 28489568256.0, "8595": 28489568256.0, "8600": 28489568256.0, "8605": 28489568256.0, "8610": 28489568256.0, "8615": 28489568256.0, "8620": 28489568256.0, "8625": 28489568256.0, "8630": 28489568256.0, "8635": 28489568256.0, "8640": 28489568256.0, "8645": 28489568256.0, "8650": 28489568256.0, "8655": 28489568256.0, "8660": 28489568256.0, "8665": 28489568256.0, "8670": 28489568256.0, "8675": 28489568256.0, "8680": 28489568256.0, "8685": 28489568256.0, "8690": 28489568256.0, "8695": 28489568256.0, "8700": 28489568256.0, "8705": 28489568256.0, "8710": 28489568256.0, "8715": 28489568256.0, "8720": 28489568256.0, "8725": 28489568256.0, "8730": 28489568256.0, "8735": 28489568256.0, "8740": 28489568256.0, "8745": 28489568256.0, "8750": 28489568256.0, "8755": 28489568256.0, "8760": 28489568256.0, "8765": 28489568256.0, "8770": 28489568256.0, "8775": 28489568256.0, "8780": 28489568256.0, "8785": 28489568256.0, "8790": 28489568256.0, "8795": 28489568256.0, "8800": 28489568256.0, "8805": 28489568256.0, "8810": 28489568256.0, "8815": 28489568256.0, "8820": 28489568256.0, "8825": 28489568256.0, "8830": 28489568256.0, "8835": 28489568256.0, "8840": 28489568256.0, "8845": 28489568256.0, "8850": 28489568256.0, "8855": 28489568256.0, "8860": 28489568256.0, "8865": 28489568256.0, "8870": 28489568256.0, "8875": 28489568256.0, "8880": 28489568256.0, "8885": 28489568256.0, "8890": 28489568256.0, "8895": 28489568256.0, "8900": 28489568256.0, "8905": 28489568256.0, "8910": 28489568256.0, "8915": 28489568256.0, "8920": 28489568256.0, "8925": 28489568256.0, "8930": 28489568256.0, "8935": 28489568256.0, "8940": 28489568256.0, "8945": 28489568256.0, "8950": 28489568256.0, "8955": 28489568256.0, "8960": 28489568256.0, "8965": 28489568256.0, "8970": 28489568256.0, "8975": 28489568256.0, "8980": 28489568256.0, "8985": 28489568256.0, "8990": 28489568256.0, "8995": 28489568256.0, "9000": 28489568256.0, "9005": 28489568256.0, "9010": 28489568256.0, "9015": 28489568256.0, "9020": 28489568256.0, "9025": 28489568256.0, "9030": 28489568256.0, "9035": 28489568256.0, "9040": 28489568256.0, "9045": 28489568256.0, "9050": 28489568256.0, "9055": 28489568256.0, "9060": 28489568256.0, "9065": 28489568256.0, "9070": 28489568256.0, "9075": 28489568256.0, "9080": 28489568256.0, "9085": 28489568256.0, "9090": 28489568256.0, "9095": 28489568256.0, "9100": 28489568256.0, "9105": 28489568256.0, "9110": 28489568256.0, "9115": 28489568256.0, "9120": 28489568256.0, "9125": 28489568256.0, "9130": 28489568256.0, "9135": 28489568256.0, "9140": 28489568256.0, "9145": 28489568256.0, "9150": 28489568256.0, "9155": 28489568256.0, "9160": 28489568256.0, "9165": 28489568256.0, "9170": 28489568256.0, "9175": 28489568256.0, "9180": 28489568256.0, "9185": 28489568256.0, "9190": 28489568256.0, "9195": 28489568256.0, "9200": 28489568256.0, "9205": 28489568256.0, "9210": 28489568256.0, "9215": 28489568256.0, "9220": 28489568256.0, "9225": 28489568256.0, "9230": 28489568256.0, "9235": 28489568256.0, "9240": 28489568256.0, "9245": 28489568256.0, "9250": 28489568256.0, "9255": 28489568256.0, "9260": 28489568256.0, "9265": 28489568256.0, "9270": 28489568256.0, "9275": 28489568256.0, "9280": 28489568256.0, "9285": 28489568256.0, "9290": 28489568256.0, "9295": 28489568256.0, "9300": 28489568256.0, "9305": 28489568256.0, "9310": 28489568256.0, "9315": 28489568256.0, "9320": 28489568256.0, "9325": 28489568256.0, "9330": 28489568256.0, "9335": 28489568256.0, "9340": 28489568256.0, "9345": 28489568256.0, "9350": 28489568256.0, "9355": 28489568256.0, "9360": 28489568256.0, "9365": 28489568256.0, "9370": 28489568256.0, "9375": 28489568256.0, "9380": 28489568256.0, "9385": 28489568256.0, "9390": 28489568256.0, "9395": 28489568256.0, "9400": 28489568256.0, "9405": 28489568256.0, "9410": 28489568256.0, "9415": 28489568256.0, "9420": 28489568256.0, "9425": 28489568256.0, "9430": 28489568256.0, "9435": 28489568256.0, "9440": 28489568256.0, "9445": 28489568256.0, "9450": 28489568256.0, "9455": 28489568256.0, "9460": 28489568256.0, "9465": 28489568256.0, "9470": 28489568256.0, "9475": 28489568256.0, "9480": 28489568256.0, "9485": 28489568256.0, "9490": 28489568256.0, "9495": 28489568256.0, "9500": 28489568256.0, "9505": 28489568256.0, "9510": 28489568256.0, "9515": 28489568256.0, "9520": 28489568256.0, "9525": 28489568256.0, "9530": 28489568256.0, "9535": 28489568256.0, "9540": 28489568256.0, "9545": 28489568256.0, "9550": 28489568256.0, "9555": 28489568256.0, "9560": 28489568256.0, "9565": 28489568256.0, "9570": 28489568256.0, "9575": 28489568256.0, "9580": 28489568256.0, "9585": 28489568256.0, "9590": 28489568256.0, "9595": 28489568256.0, "9600": 28489568256.0, "9605": 28489568256.0, "9610": 28489568256.0, "9615": 28489568256.0, "9620": 28489568256.0, "9625": 28489568256.0, "9630": 28489568256.0, "9635": 28489568256.0, "9640": 28489568256.0, "9645": 28489568256.0, "9650": 28489568256.0, "9655": 28489568256.0, "9660": 28489568256.0, "9665": 28489568256.0, "9670": 28489568256.0, "9675": 28489568256.0, "9680": 28489568256.0, "9685": 28489568256.0, "9690": 28489568256.0, "9695": 28489568256.0, "9700": 28489568256.0, "9705": 28489568256.0, "9710": 28489568256.0, "9715": 28489568256.0, "9720": 28489568256.0, "9725": 28489568256.0, "9730": 28489568256.0, "9735": 28489568256.0, "9740": 28489568256.0, "9745": 28489568256.0, "9750": 28489568256.0, "9755": 28489568256.0, "9760": 28489568256.0, "9765": 28489568256.0, "9770": 28489568256.0, "9775": 28489568256.0, "9780": 28489568256.0, "9785": 28489568256.0, "9790": 28489568256.0, "9795": 28489568256.0, "9800": 28489568256.0, "9805": 28489568256.0, "9810": 28489568256.0, "9815": 28489568256.0, "9820": 28489568256.0, "9825": 28489568256.0, "9830": 28489568256.0, "9835": 28489568256.0, "9840": 28489568256.0, "9845": 28489568256.0, "9850": 28489568256.0, "9855": 28489568256.0, "9860": 28489568256.0, "9865": 28489568256.0, "9870": 28489568256.0, "9875": 28489568256.0, "9880": 28489568256.0, "9885": 28489568256.0, "9890": 28489568256.0, "9895": 28489568256.0, "9900": 28489568256.0, "9905": 28489568256.0, "9910": 28489568256.0, "9915": 28489568256.0, "9920": 28489568256.0, "9925": 28489568256.0, "9930": 28489568256.0, "9935": 28489568256.0, "9940": 28489568256.0, "9945": 28489568256.0, "9950": 28489568256.0, "9955": 28489568256.0, "9960": 28489568256.0, "9965": 28489568256.0, "9970": 28489568256.0, "9975": 28489568256.0, "9980": 28489568256.0, "9985": 28489568256.0, "9990": 28489568256.0, "9995": 28489568256.0, "10000": 28489568256.0, "10005": 28489568256.0, "10010": 28489568256.0, "10015": 28489568256.0, "10020": 28489568256.0, "10025": 28489568256.0, "10030": 28489568256.0, "10035": 28489568256.0, "10040": 28489568256.0, "10045": 28489568256.0, "10050": 28489568256.0, "10055": 28489568256.0, "10060": 28489568256.0, "10065": 28489568256.0, "10070": 28489568256.0, "10075": 28489568256.0, "10080": 28489568256.0, "10085": 28489568256.0, "10090": 28489568256.0, "10095": 28489568256.0, "10100": 28489568256.0, "10105": 28489568256.0, "10110": 28489568256.0, "10115": 28489568256.0, "10120": 28489568256.0, "10125": 28489568256.0, "10130": 28489568256.0, "10135": 28489568256.0, "10140": 28489568256.0, "10145": 28489568256.0, "10150": 28489568256.0, "10155": 28489568256.0, "10160": 28489568256.0, "10165": 28489568256.0, "10170": 28489568256.0, "10175": 28489568256.0, "10180": 28489568256.0, "10185": 28489568256.0, "10190": 28489568256.0, "10195": 28489568256.0, "10200": 28489568256.0, "10205": 28489568256.0, "10210": 28489568256.0, "10215": 28489568256.0, "10220": 28489568256.0, "10225": 28489568256.0, "10230": 28489568256.0, "10235": 28489568256.0, "10240": 28489568256.0, "10245": 28489568256.0, "10250": 28489568256.0, "10255": 28489568256.0, "10260": 28489568256.0, "10265": 28489568256.0, "10270": 28489568256.0, "10275": 28489568256.0, "10280": 28489568256.0, "10285": 28489568256.0, "10290": 28489568256.0, "10295": 28489568256.0, "10300": 28489568256.0, "10305": 28489568256.0, "10310": 28489568256.0, "10315": 28489568256.0, "10320": 28489568256.0, "10325": 28489568256.0, "10330": 28489568256.0, "10335": 28489568256.0, "10340": 28489568256.0, "10345": 28489568256.0, "10350": 28489568256.0, "10355": 28489568256.0, "10360": 28489568256.0, "10365": 28489568256.0, "10370": 28489568256.0, "10375": 28489568256.0, "10380": 28489568256.0, "10385": 28489568256.0, "10390": 28489568256.0, "10395": 28489568256.0, "10400": 28489568256.0, "10405": 28489568256.0, "10410": 28489568256.0, "10415": 28489568256.0, "10420": 28489568256.0, "10425": 28489568256.0, "10430": 28489568256.0, "10435": 28489568256.0, "10440": 28489568256.0, "10445": 28489568256.0, "10450": 28489568256.0, "10455": 28489568256.0, "10460": 28489568256.0, "10465": 28489568256.0, "10470": 28489568256.0, "10475": 28489568256.0, "10480": 28489568256.0, "10485": 28489568256.0, "10490": 28489568256.0, "10495": 28489568256.0, "10500": 28489568256.0, "10505": 28489568256.0, "10510": 28489568256.0, "10515": 28489568256.0, "10520": 28489568256.0, "10525": 28489568256.0, "10530": 28489568256.0, "10535": 28489568256.0, "10540": 28489568256.0, "10545": 28489568256.0, "10550": 28489568256.0, "10555": 28489568256.0, "10560": 28489568256.0, "10565": 28489568256.0, "10570": 28489568256.0, "10575": 28489568256.0, "10580": 28489568256.0, "10585": 28489568256.0, "10590": 28489568256.0, "10595": 28489568256.0, "10600": 28489568256.0, "10605": 28489568256.0, "10610": 28489568256.0, "10615": 28489568256.0, "10620": 28489568256.0, "10625": 28489568256.0, "10630": 28489568256.0, "10635": 28489568256.0, "10640": 28489568256.0, "10645": 28489568256.0, "10650": 28489568256.0, "10655": 28489568256.0, "10660": 28489568256.0, "10665": 28489568256.0, "10670": 28489568256.0, "10675": 28489568256.0, "10680": 28489568256.0, "10685": 28489568256.0, "10690": 28489568256.0, "10695": 28489568256.0, "10700": 28489568256.0, "10705": 28489568256.0, "10710": 28489568256.0, "10715": 28489568256.0, "10720": 28489568256.0, "10725": 28489568256.0, "10730": 28489568256.0, "10735": 28489568256.0, "10740": 28489568256.0, "10745": 28489568256.0, "10750": 28489568256.0, "10755": 28489568256.0, "10760": 28489568256.0, "10765": 28489568256.0, "10770": 28489568256.0, "10775": 28489568256.0, "10780": 28489568256.0, "10785": 28489568256.0, "10790": 28489568256.0, "10795": 28489568256.0, "10800": 28489568256.0, "10805": 28489568256.0, "10810": 28489568256.0, "10815": 28489568256.0, "10820": 28489568256.0, "10825": 28489568256.0, "10830": 28489568256.0, "10835": 28489568256.0, "10840": 28489568256.0, "10845": 28489568256.0, "10850": 28489568256.0, "10855": 28489568256.0, "10860": 28489568256.0, "10865": 28489568256.0, "10870": 28489568256.0, "10875": 28489568256.0, "10880": 28489568256.0, "10885": 28489568256.0, "10890": 28489568256.0, "10895": 28489568256.0, "10900": 28489568256.0, "10905": 28489568256.0, "10910": 28489568256.0, "10915": 28489568256.0, "10920": 28489568256.0, "10925": 28489568256.0, "10930": 28489568256.0, "10935": 28489568256.0, "10940": 28489568256.0, "10945": 28489568256.0, "10950": 28489568256.0, "10955": 28489568256.0, "10960": 28489568256.0, "10965": 28489568256.0, "10970": 28489568256.0, "10975": 28489568256.0, "10980": 28489568256.0, "10985": 28489568256.0, "10990": 28489568256.0, "10995": 28489568256.0, "11000": 28489568256.0, "11005": 28489568256.0, "11010": 28489568256.0, "11015": 28489568256.0, "11020": 28489568256.0, "11025": 28489568256.0, "11030": 28489568256.0, "11035": 28489568256.0, "11040": 28489568256.0, "11045": 28489568256.0, "11050": 28489568256.0, "11055": 28489568256.0, "11060": 28489568256.0, "11065": 28489568256.0, "11070": 28489568256.0, "11075": 28489568256.0, "11080": 28489568256.0, "11085": 28489568256.0, "11090": 28489568256.0, "11095": 28489568256.0, "11100": 28489568256.0, "11105": 28489568256.0, "11110": 28489568256.0, "11115": 28489568256.0, "11120": 28489568256.0, "11125": 28489568256.0, "11130": 28489568256.0, "11135": 28489568256.0, "11140": 28489568256.0, "11145": 28489568256.0, "11150": 28489568256.0, "11155": 28489568256.0, "11160": 28489568256.0, "11165": 28489568256.0, "11170": 28489568256.0, "11175": 28489568256.0, "11180": 28489568256.0, "11185": 28489568256.0, "11190": 28489568256.0, "11195": 28489568256.0, "11200": 28489568256.0, "11205": 28489568256.0, "11210": 28489568256.0, "11215": 28489568256.0, "11220": 28489568256.0, "11225": 28489568256.0, "11230": 28489568256.0, "11235": 28489568256.0, "11240": 28489568256.0, "11245": 28489568256.0, "11250": 28489568256.0, "11255": 28489568256.0, "11260": 28489568256.0, "11265": 28489568256.0, "11270": 28489568256.0, "11275": 28489568256.0, "11280": 28489568256.0, "11285": 28489568256.0, "11290": 28489568256.0, "11295": 28489568256.0, "11300": 28489568256.0, "11305": 28489568256.0, "11310": 28489568256.0, "11315": 28489568256.0, "11320": 28489568256.0, "11325": 28489568256.0, "11330": 28489568256.0, "11335": 28489568256.0, "11340": 28489568256.0, "11345": 28489568256.0, "11350": 28489568256.0, "11355": 28489568256.0, "11360": 28489568256.0, "11365": 28489568256.0, "11370": 28489568256.0, "11375": 28489568256.0, "11380": 28489568256.0, "11385": 28489568256.0, "11390": 28489568256.0, "11395": 28489568256.0, "11400": 28489568256.0, "11405": 28489568256.0, "11410": 28489568256.0, "11415": 28489568256.0, "11420": 28489568256.0, "11425": 28489568256.0, "11430": 28489568256.0, "11435": 28489568256.0, "11440": 28489568256.0, "11445": 28489568256.0, "11450": 28489568256.0, "11455": 28489568256.0, "11460": 28489568256.0, "11465": 28489568256.0, "11470": 28489568256.0, "11475": 28489568256.0, "11480": 28489568256.0, "11485": 28489568256.0, "11490": 28489568256.0, "11495": 28489568256.0, "11500": 28489568256.0, "11505": 28489568256.0, "11510": 28489568256.0, "11515": 28489568256.0, "11520": 28489568256.0, "11525": 28489568256.0, "11530": 28489568256.0, "11535": 28489568256.0, "11540": 28489568256.0, "11545": 28489568256.0, "11550": 28489568256.0, "11555": 28489568256.0, "11560": 28489568256.0, "11565": 28489568256.0, "11570": 28489568256.0, "11575": 28489568256.0, "11580": 28489568256.0, "11585": 28489568256.0, "11590": 28489568256.0, "11595": 28489568256.0, "11600": 28489568256.0, "11605": 28489568256.0, "11610": 28489568256.0, "11615": 28489568256.0, "11620": 28489568256.0, "11625": 28489568256.0, "11630": 28489568256.0, "11635": 28489568256.0, "11640": 28489568256.0, "11645": 28489568256.0, "11650": 28489568256.0, "11655": 28489568256.0, "11660": 28489568256.0, "11665": 28489568256.0, "11670": 28489568256.0, "11675": 28489568256.0, "11680": 28489568256.0, "11685": 28489568256.0, "11690": 28489568256.0, "11695": 28489568256.0, "11700": 28489568256.0, "11705": 28489568256.0, "11710": 28489568256.0, "11715": 28489568256.0, "11720": 28489568256.0, "11725": 28489568256.0, "11730": 28489568256.0, "11735": 28489568256.0, "11740": 28489568256.0, "11745": 28489568256.0, "11750": 28489568256.0, "11755": 28489568256.0, "11760": 28489568256.0, "11765": 28489568256.0, "11770": 28489568256.0, "11775": 28489568256.0, "11780": 28489568256.0, "11785": 28489568256.0, "11790": 28489568256.0, "11795": 28489568256.0, "11800": 28489568256.0, "11805": 28489568256.0, "11810": 28489568256.0, "11815": 28489568256.0, "11820": 28489568256.0, "11825": 28489568256.0, "11830": 28489568256.0, "11835": 28489568256.0, "11840": 28489568256.0, "11845": 28489568256.0, "11850": 28489568256.0, "11855": 28489568256.0, "11860": 28489568256.0, "11865": 28489568256.0, "11870": 28489568256.0, "11875": 28489568256.0, "11880": 28489568256.0, "11885": 28489568256.0, "11890": 28489568256.0, "11895": 28489568256.0, "11900": 28489568256.0, "11905": 28489568256.0, "11910": 28489568256.0, "11915": 28489568256.0, "11920": 28489568256.0, "11925": 28489568256.0, "11930": 28489568256.0, "11935": 28489568256.0, "11940": 28489568256.0, "11945": 28489568256.0, "11950": 28489568256.0, "11955": 28489568256.0, "11960": 28489568256.0, "11965": 28489568256.0, "11970": 28489568256.0, "11975": 28489568256.0, "11980": 28489568256.0, "11985": 28489568256.0, "11990": 28489568256.0, "11995": 28489568256.0, "12000": 28489568256.0, "12005": 28489568256.0, "12010": 28489568256.0, "12015": 28489568256.0, "12020": 28489568256.0, "12025": 28489568256.0, "12030": 28489568256.0, "12035": 28489568256.0, "12040": 28489568256.0, "12045": 28489568256.0, "12050": 28489568256.0, "12055": 28489568256.0, "12060": 28489568256.0, "12065": 28489568256.0, "12070": 28489568256.0, "12075": 28489568256.0, "12080": 28489568256.0, "12085": 28489568256.0, "12090": 28489568256.0, "12095": 28489568256.0, "12100": 28489568256.0, "12105": 28489568256.0, "12110": 28489568256.0, "12115": 28489568256.0, "12120": 28489568256.0, "12125": 28489568256.0, "12130": 28489568256.0, "12135": 28489568256.0, "12140": 28489568256.0, "12145": 28489568256.0, "12150": 28489568256.0, "12155": 28489568256.0, "12160": 28489568256.0, "12165": 28489568256.0, "12170": 28489568256.0, "12175": 28489568256.0, "12180": 28489568256.0, "12185": 28489568256.0, "12190": 28489568256.0, "12195": 28489568256.0, "12200": 28489568256.0, "12205": 28489568256.0, "12210": 28489568256.0, "12215": 28489568256.0, "12220": 28489568256.0, "12225": 28489568256.0, "12230": 28489568256.0, "12235": 28489568256.0, "12240": 28489568256.0, "12245": 28489568256.0, "12250": 28489568256.0, "12255": 28489568256.0, "12260": 28489568256.0, "12265": 28489568256.0, "12270": 28489568256.0, "12275": 28489568256.0, "12280": 28489568256.0, "12285": 28489568256.0, "12290": 28489568256.0, "12295": 28489568256.0, "12300": 28489568256.0, "12305": 28489568256.0, "12310": 28489568256.0, "12315": 28489568256.0, "12320": 28489568256.0, "12325": 28489568256.0, "12330": 28489568256.0, "12335": 28489568256.0, "12340": 28489568256.0, "12345": 28489568256.0, "12350": 28489568256.0, "12355": 28489568256.0, "12360": 28489568256.0, "12365": 28489568256.0, "12370": 28489568256.0, "12375": 28489568256.0, "12380": 28489568256.0, "12385": 28489568256.0, "12390": 28489568256.0, "12395": 28489568256.0, "12400": 28489568256.0, "12405": 28489568256.0, "12410": 28489568256.0, "12415": 28489568256.0, "12420": 28489568256.0, "12425": 28489568256.0, "12430": 28489568256.0, "12435": 28489568256.0, "12440": 28489568256.0, "12445": 28489568256.0, "12450": 28489568256.0, "12455": 28489568256.0, "12460": 28489568256.0, "12465": 28489568256.0, "12470": 28489568256.0, "12475": 28489568256.0, "12480": 28489568256.0, "12485": 28489568256.0, "12490": 28489568256.0, "12495": 28489568256.0, "12500": 28489568256.0, "12505": 28489568256.0, "12510": 28489568256.0, "12515": 28489568256.0, "12520": 28489568256.0, "12525": 28489568256.0, "12530": 28489568256.0, "12535": 28489568256.0, "12540": 28489568256.0, "12545": 28489568256.0, "12550": 28489568256.0, "12555": 28489568256.0, "12560": 28489568256.0, "12565": 28489568256.0, "12570": 28489568256.0, "12575": 28489568256.0, "12580": 28489568256.0, "12585": 28489568256.0, "12590": 28489568256.0, "12595": 28489568256.0, "12600": 28489568256.0, "12605": 28489568256.0, "12610": 28489568256.0, "12615": 28489568256.0, "12620": 28489568256.0, "12625": 28489568256.0, "12630": 28489568256.0, "12635": 28489568256.0, "12640": 28489568256.0, "12645": 28489568256.0, "12650": 28489568256.0, "12655": 28489568256.0, "12660": 28489568256.0, "12665": 28489568256.0, "12670": 28489568256.0, "12675": 28489568256.0, "12680": 28489568256.0, "12685": 28489568256.0, "12690": 28489568256.0, "12695": 28489568256.0, "12700": 28489568256.0, "12705": 28489568256.0, "12710": 28489568256.0, "12715": 28489568256.0, "12720": "nan", "12725": "nan", "12730": "nan", "12735": "nan", "12740": "nan", "12745": "nan", "12750": "nan", "12755": "nan", "12760": "nan", "12765": "nan", "12770": "nan", "12775": "nan", "12780": "nan", "12785": "nan", "12790": "nan", "12795": "nan", "12800": "nan", "12805": "nan", "12810": "nan", "12815": "nan", "12820": "nan", "12825": "nan", "12830": "nan", "12835": "nan", "12840": "nan", "12845": "nan", "12850": "nan", "12855": "nan", "12860": "nan", "12865": "nan", "12870": "nan", "12875": "nan", "12880": "nan", "12885": "nan", "12890": "nan", "12895": "nan", "12900": "nan", "12905": "nan", "12910": "nan", "12915": "nan", "12920": "nan", "12925": "nan", "12930": "nan", "12935": "nan", "12940": "nan", "12945": "nan", "12950": "nan", "12955": "nan", "12960": "nan", "12965": "nan", "12970": "nan", "12975": "nan", "12980": "nan", "12985": "nan", "12990": "nan", "12995": "nan", "13000": "nan"}}, "iteration-time": {"start_step": 1, "end_step": 13000, "step_interval": 5, "values": {"1": "nan", "5": "nan", "10": "nan", "15": "nan", "20": "nan", "25": "nan", "30": "nan", "35": "nan", "40": "nan", "45": "nan", "50": "nan", "55": "nan", "60": "nan", "65": "nan", "70": "nan", "75": "nan", "80": "nan", "85": "nan", "90": "nan", "95": "nan", "100": 3.5554, "105": "nan", "110": "nan", "115": "nan", "120": "nan", "125": "nan", "130": "nan", "135": "nan", "140": "nan", "145": "nan", "150": "nan", "155": "nan", "160": "nan", "165": "nan", "170": "nan", "175": "nan", "180": "nan", "185": "nan", "190": "nan", "195": "nan", "200": 3.47185, "205": "nan", "210": "nan", "215": "nan", "220": "nan", "225": "nan", "230": "nan", "235": "nan", "240": "nan", "245": "nan", "250": "nan", "255": "nan", "260": "nan", "265": "nan", "270": "nan", "275": "nan", "280": "nan", "285": "nan", "290": "nan", "295": "nan", "300": 3.45692, "305": "nan", "310": "nan", "315": "nan", "320": "nan", "325": "nan", "330": "nan", "335": "nan", "340": "nan", "345": "nan", "350": "nan", "355": "nan", "360": "nan", "365": "nan", "370": "nan", "375": "nan", "380": "nan", "385": "nan", "390": "nan", "395": "nan", "400": 3.45471, "405": "nan", "410": "nan", "415": "nan", "420": "nan", "425": "nan", "430": "nan", "435": "nan", "440": "nan", "445": "nan", "450": "nan", "455": "nan", "460": "nan", "465": "nan", "470": "nan", "475": "nan", "480": "nan", "485": "nan", "490": "nan", "495": "nan", "500": 3.45467, "505": "nan", "510": "nan", "515": "nan", "520": "nan", "525": "nan", "530": "nan", "535": "nan", "540": "nan", "545": "nan", "550": "nan", "555": "nan", "560": "nan", "565": "nan", "570": "nan", "575": "nan", "580": "nan", "585": "nan", "590": "nan", "595": "nan", "600": 3.4543, "605": "nan", "610": "nan", "615": "nan", "620": "nan", "625": "nan", "630": "nan", "635": "nan", "640": "nan", "645": "nan", "650": "nan", "655": "nan", "660": "nan", "665": "nan", "670": "nan", "675": "nan", "680": "nan", "685": "nan", "690": "nan", "695": "nan", "700": 3.45264, "705": "nan", "710": "nan", "715": "nan", "720": "nan", "725": "nan", "730": "nan", "735": "nan", "740": "nan", "745": "nan", "750": "nan", "755": "nan", "760": "nan", "765": "nan", "770": "nan", "775": "nan", "780": "nan", "785": "nan", "790": "nan", "795": "nan", "800": 3.45125, "805": "nan", "810": "nan", "815": "nan", "820": "nan", "825": "nan", "830": "nan", "835": "nan", "840": "nan", "845": "nan", "850": "nan", "855": "nan", "860": "nan", "865": "nan", "870": "nan", "875": "nan", "880": "nan", "885": "nan", "890": "nan", "895": "nan", "900": 3.44668, "905": "nan", "910": "nan", "915": "nan", "920": "nan", "925": "nan", "930": "nan", "935": "nan", "940": "nan", "945": "nan", "950": "nan", "955": "nan", "960": "nan", "965": "nan", "970": "nan", "975": "nan", "980": "nan", "985": "nan", "990": "nan", "995": "nan", "1000": 3.44035, "1005": "nan", "1010": "nan", "1015": "nan", "1020": "nan", "1025": "nan", "1030": "nan", "1035": "nan", "1040": "nan", "1045": "nan", "1050": "nan", "1055": "nan", "1060": "nan", "1065": "nan", "1070": "nan", "1075": "nan", "1080": "nan", "1085": "nan", "1090": "nan", "1095": "nan", "1100": 3.43442, "1105": "nan", "1110": "nan", "1115": "nan", "1120": "nan", "1125": "nan", "1130": "nan", "1135": "nan", "1140": "nan", "1145": "nan", "1150": "nan", "1155": "nan", "1160": "nan", "1165": "nan", "1170": "nan", "1175": "nan", "1180": "nan", "1185": "nan", "1190": "nan", "1195": "nan", "1200": 3.4306, "1205": "nan", "1210": "nan", "1215": "nan", "1220": "nan", "1225": "nan", "1230": "nan", "1235": "nan", "1240": "nan", "1245": "nan", "1250": "nan", "1255": "nan", "1260": "nan", "1265": "nan", "1270": "nan", "1275": "nan", "1280": "nan", "1285": "nan", "1290": "nan", "1295": "nan", "1300": 3.42464, "1305": "nan", "1310": "nan", "1315": "nan", "1320": "nan", "1325": "nan", "1330": "nan", "1335": "nan", "1340": "nan", "1345": "nan", "1350": "nan", "1355": "nan", "1360": "nan", "1365": "nan", "1370": "nan", "1375": "nan", "1380": "nan", "1385": "nan", "1390": "nan", "1395": "nan", "1400": 3.42155, "1405": "nan", "1410": "nan", "1415": "nan", "1420": "nan", "1425": "nan", "1430": "nan", "1435": "nan", "1440": "nan", "1445": "nan", "1450": "nan", "1455": "nan", "1460": "nan", "1465": "nan", "1470": "nan", "1475": "nan", "1480": "nan", "1485": "nan", "1490": "nan", "1495": "nan", "1500": 3.4201, "1505": "nan", "1510": "nan", "1515": "nan", "1520": "nan", "1525": "nan", "1530": "nan", "1535": "nan", "1540": "nan", "1545": "nan", "1550": "nan", "1555": "nan", "1560": "nan", "1565": "nan", "1570": "nan", "1575": "nan", "1580": "nan", "1585": "nan", "1590": "nan", "1595": "nan", "1600": 3.41703, "1605": "nan", "1610": "nan", "1615": "nan", "1620": "nan", "1625": "nan", "1630": "nan", "1635": "nan", "1640": "nan", "1645": "nan", "1650": "nan", "1655": "nan", "1660": "nan", "1665": "nan", "1670": "nan", "1675": "nan", "1680": "nan", "1685": "nan", "1690": "nan", "1695": "nan", "1700": 3.41482, "1705": "nan", "1710": "nan", "1715": "nan", "1720": "nan", "1725": "nan", "1730": "nan", "1735": "nan", "1740": "nan", "1745": "nan", "1750": "nan", "1755": "nan", "1760": "nan", "1765": "nan", "1770": "nan", "1775": "nan", "1780": "nan", "1785": "nan", "1790": "nan", "1795": "nan", "1800": 3.41352, "1805": "nan", "1810": "nan", "1815": "nan", "1820": "nan", "1825": "nan", "1830": "nan", "1835": "nan", "1840": "nan", "1845": "nan", "1850": "nan", "1855": "nan", "1860": "nan", "1865": "nan", "1870": "nan", "1875": "nan", "1880": "nan", "1885": "nan", "1890": "nan", "1895": "nan", "1900": 3.4128, "1905": "nan", "1910": "nan", "1915": "nan", "1920": "nan", "1925": "nan", "1930": "nan", "1935": "nan", "1940": "nan", "1945": "nan", "1950": "nan", "1955": "nan", "1960": "nan", "1965": "nan", "1970": "nan", "1975": "nan", "1980": "nan", "1985": "nan", "1990": "nan", "1995": "nan", "2000": 3.40994, "2005": "nan", "2010": "nan", "2015": "nan", "2020": "nan", "2025": "nan", "2030": "nan", "2035": "nan", "2040": "nan", "2045": "nan", "2050": "nan", "2055": "nan", "2060": "nan", "2065": "nan", "2070": "nan", "2075": "nan", "2080": "nan", "2085": "nan", "2090": "nan", "2095": "nan", "2100": 3.40878, "2105": "nan", "2110": "nan", "2115": "nan", "2120": "nan", "2125": "nan", "2130": "nan", "2135": "nan", "2140": "nan", "2145": "nan", "2150": "nan", "2155": "nan", "2160": "nan", "2165": "nan", "2170": "nan", "2175": "nan", "2180": "nan", "2185": "nan", "2190": "nan", "2195": "nan", "2200": 3.40862, "2205": "nan", "2210": "nan", "2215": "nan", "2220": "nan", "2225": "nan", "2230": "nan", "2235": "nan", "2240": "nan", "2245": "nan", "2250": "nan", "2255": "nan", "2260": "nan", "2265": "nan", "2270": "nan", "2275": "nan", "2280": "nan", "2285": "nan", "2290": "nan", "2295": "nan", "2300": 3.41459, "2305": "nan", "2310": "nan", "2315": "nan", "2320": "nan", "2325": "nan", "2330": "nan", "2335": "nan", "2340": "nan", "2345": "nan", "2350": "nan", "2355": "nan", "2360": "nan", "2365": "nan", "2370": "nan", "2375": "nan", "2380": "nan", "2385": "nan", "2390": "nan", "2395": "nan", "2400": 3.40662, "2405": "nan", "2410": "nan", "2415": "nan", "2420": "nan", "2425": "nan", "2430": "nan", "2435": "nan", "2440": "nan", "2445": "nan", "2450": "nan", "2455": "nan", "2460": "nan", "2465": "nan", "2470": "nan", "2475": "nan", "2480": "nan", "2485": "nan", "2490": "nan", "2495": "nan", "2500": 3.40543, "2505": "nan", "2510": "nan", "2515": "nan", "2520": "nan", "2525": "nan", "2530": "nan", "2535": "nan", "2540": "nan", "2545": "nan", "2550": "nan", "2555": "nan", "2560": "nan", "2565": "nan", "2570": "nan", "2575": "nan", "2580": "nan", "2585": "nan", "2590": "nan", "2595": "nan", "2600": 3.40484, "2605": "nan", "2610": "nan", "2615": "nan", "2620": "nan", "2625": "nan", "2630": "nan", "2635": "nan", "2640": "nan", "2645": "nan", "2650": "nan", "2655": "nan", "2660": "nan", "2665": "nan", "2670": "nan", "2675": "nan", "2680": "nan", "2685": "nan", "2690": "nan", "2695": "nan", "2700": 3.40448, "2705": "nan", "2710": "nan", "2715": "nan", "2720": "nan", "2725": "nan", "2730": "nan", "2735": "nan", "2740": "nan", "2745": "nan", "2750": "nan", "2755": "nan", "2760": "nan", "2765": "nan", "2770": "nan", "2775": "nan", "2780": "nan", "2785": "nan", "2790": "nan", "2795": "nan", "2800": 3.403, "2805": "nan", "2810": "nan", "2815": "nan", "2820": "nan", "2825": "nan", "2830": "nan", "2835": "nan", "2840": "nan", "2845": "nan", "2850": "nan", "2855": "nan", "2860": "nan", "2865": "nan", "2870": "nan", "2875": "nan", "2880": "nan", "2885": "nan", "2890": "nan", "2895": "nan", "2900": 3.40346, "2905": "nan", "2910": "nan", "2915": "nan", "2920": "nan", "2925": "nan", "2930": "nan", "2935": "nan", "2940": "nan", "2945": "nan", "2950": "nan", "2955": "nan", "2960": "nan", "2965": "nan", "2970": "nan", "2975": "nan", "2980": "nan", "2985": "nan", "2990": "nan", "2995": "nan", "3000": 3.4023, "3005": "nan", "3010": "nan", "3015": "nan", "3020": "nan", "3025": "nan", "3030": "nan", "3035": "nan", "3040": "nan", "3045": "nan", "3050": "nan", "3055": "nan", "3060": "nan", "3065": "nan", "3070": "nan", "3075": "nan", "3080": "nan", "3085": "nan", "3090": "nan", "3095": "nan", "3100": 3.40069, "3105": "nan", "3110": "nan", "3115": "nan", "3120": "nan", "3125": "nan", "3130": "nan", "3135": "nan", "3140": "nan", "3145": "nan", "3150": "nan", "3155": "nan", "3160": "nan", "3165": "nan", "3170": "nan", "3175": "nan", "3180": "nan", "3185": "nan", "3190": "nan", "3195": "nan", "3200": 3.40162, "3205": "nan", "3210": "nan", "3215": "nan", "3220": "nan", "3225": "nan", "3230": "nan", "3235": "nan", "3240": "nan", "3245": "nan", "3250": "nan", "3255": "nan", "3260": "nan", "3265": "nan", "3270": "nan", "3275": "nan", "3280": "nan", "3285": "nan", "3290": "nan", "3295": "nan", "3300": 3.40071, "3305": "nan", "3310": "nan", "3315": "nan", "3320": "nan", "3325": "nan", "3330": "nan", "3335": "nan", "3340": "nan", "3345": "nan", "3350": "nan", "3355": "nan", "3360": "nan", "3365": "nan", "3370": "nan", "3375": "nan", "3380": "nan", "3385": "nan", "3390": "nan", "3395": "nan", "3400": 3.40058, "3405": "nan", "3410": "nan", "3415": "nan", "3420": "nan", "3425": "nan", "3430": "nan", "3435": "nan", "3440": "nan", "3445": "nan", "3450": "nan", "3455": "nan", "3460": "nan", "3465": "nan", "3470": "nan", "3475": "nan", "3480": "nan", "3485": "nan", "3490": "nan", "3495": "nan", "3500": 3.39993, "3505": "nan", "3510": "nan", "3515": "nan", "3520": "nan", "3525": "nan", "3530": "nan", "3535": "nan", "3540": "nan", "3545": "nan", "3550": "nan", "3555": "nan", "3560": "nan", "3565": "nan", "3570": "nan", "3575": "nan", "3580": "nan", "3585": "nan", "3590": "nan", "3595": "nan", "3600": 3.62689, "3605": "nan", "3610": "nan", "3615": "nan", "3620": "nan", "3625": "nan", "3630": "nan", "3635": "nan", "3640": "nan", "3645": "nan", "3650": "nan", "3655": "nan", "3660": "nan", "3665": "nan", "3670": "nan", "3675": "nan", "3680": "nan", "3685": "nan", "3690": "nan", "3695": "nan", "3700": 3.40474, "3705": "nan", "3710": "nan", "3715": "nan", "3720": "nan", "3725": "nan", "3730": "nan", "3735": "nan", "3740": "nan", "3745": "nan", "3750": "nan", "3755": "nan", "3760": "nan", "3765": "nan", "3770": "nan", "3775": "nan", "3780": "nan", "3785": "nan", "3790": "nan", "3795": "nan", "3800": 3.40515, "3805": "nan", "3810": "nan", "3815": "nan", "3820": "nan", "3825": "nan", "3830": "nan", "3835": "nan", "3840": "nan", "3845": "nan", "3850": "nan", "3855": "nan", "3860": "nan", "3865": "nan", "3870": "nan", "3875": "nan", "3880": "nan", "3885": "nan", "3890": "nan", "3895": "nan", "3900": 3.40436, "3905": "nan", "3910": "nan", "3915": "nan", "3920": "nan", "3925": "nan", "3930": "nan", "3935": "nan", "3940": "nan", "3945": "nan", "3950": "nan", "3955": "nan", "3960": "nan", "3965": "nan", "3970": "nan", "3975": "nan", "3980": "nan", "3985": "nan", "3990": "nan", "3995": "nan", "4000": 3.40304, "4005": "nan", "4010": "nan", "4015": "nan", "4020": "nan", "4025": "nan", "4030": "nan", "4035": "nan", "4040": "nan", "4045": "nan", "4050": "nan", "4055": "nan", "4060": "nan", "4065": "nan", "4070": "nan", "4075": "nan", "4080": "nan", "4085": "nan", "4090": "nan", "4095": "nan", "4100": 3.40076, "4105": "nan", "4110": "nan", "4115": "nan", "4120": "nan", "4125": "nan", "4130": "nan", "4135": "nan", "4140": "nan", "4145": "nan", "4150": "nan", "4155": "nan", "4160": "nan", "4165": "nan", "4170": "nan", "4175": "nan", "4180": "nan", "4185": "nan", "4190": "nan", "4195": "nan", "4200": 3.40222, "4205": "nan", "4210": "nan", "4215": "nan", "4220": "nan", "4225": "nan", "4230": "nan", "4235": "nan", "4240": "nan", "4245": "nan", "4250": "nan", "4255": "nan", "4260": "nan", "4265": "nan", "4270": "nan", "4275": "nan", "4280": "nan", "4285": "nan", "4290": "nan", "4295": "nan", "4300": 3.40241, "4305": "nan", "4310": "nan", "4315": "nan", "4320": "nan", "4325": "nan", "4330": "nan", "4335": "nan", "4340": "nan", "4345": "nan", "4350": "nan", "4355": "nan", "4360": "nan", "4365": "nan", "4370": "nan", "4375": "nan", "4380": "nan", "4385": "nan", "4390": "nan", "4395": "nan", "4400": 3.40228, "4405": "nan", "4410": "nan", "4415": "nan", "4420": "nan", "4425": "nan", "4430": "nan", "4435": "nan", "4440": "nan", "4445": "nan", "4450": "nan", "4455": "nan", "4460": "nan", "4465": "nan", "4470": "nan", "4475": "nan", "4480": "nan", "4485": "nan", "4490": "nan", "4495": "nan", "4500": 3.40282, "4505": "nan", "4510": "nan", "4515": "nan", "4520": "nan", "4525": "nan", "4530": "nan", "4535": "nan", "4540": "nan", "4545": "nan", "4550": "nan", "4555": "nan", "4560": "nan", "4565": "nan", "4570": "nan", "4575": "nan", "4580": "nan", "4585": "nan", "4590": "nan", "4595": "nan", "4600": 3.40214, "4605": "nan", "4610": "nan", "4615": "nan", "4620": "nan", "4625": "nan", "4630": "nan", "4635": "nan", "4640": "nan", "4645": "nan", "4650": "nan", "4655": "nan", "4660": "nan", "4665": "nan", "4670": "nan", "4675": "nan", "4680": "nan", "4685": "nan", "4690": "nan", "4695": "nan", "4700": 3.40155, "4705": "nan", "4710": "nan", "4715": "nan", "4720": "nan", "4725": "nan", "4730": "nan", "4735": "nan", "4740": "nan", "4745": "nan", "4750": "nan", "4755": "nan", "4760": "nan", "4765": "nan", "4770": "nan", "4775": "nan", "4780": "nan", "4785": "nan", "4790": "nan", "4795": "nan", "4800": 3.4016, "4805": "nan", "4810": "nan", "4815": "nan", "4820": "nan", "4825": "nan", "4830": "nan", "4835": "nan", "4840": "nan", "4845": "nan", "4850": "nan", "4855": "nan", "4860": "nan", "4865": "nan", "4870": "nan", "4875": "nan", "4880": "nan", "4885": "nan", "4890": "nan", "4895": "nan", "4900": 3.40208, "4905": "nan", "4910": "nan", "4915": "nan", "4920": "nan", "4925": "nan", "4930": "nan", "4935": "nan", "4940": "nan", "4945": "nan", "4950": "nan", "4955": "nan", "4960": "nan", "4965": "nan", "4970": "nan", "4975": "nan", "4980": "nan", "4985": "nan", "4990": "nan", "4995": "nan", "5000": 3.40265, "5005": "nan", "5010": "nan", "5015": "nan", "5020": "nan", "5025": "nan", "5030": "nan", "5035": "nan", "5040": "nan", "5045": "nan", "5050": "nan", "5055": "nan", "5060": "nan", "5065": "nan", "5070": "nan", "5075": "nan", "5080": "nan", "5085": "nan", "5090": "nan", "5095": "nan", "5100": 3.3986, "5105": "nan", "5110": "nan", "5115": "nan", "5120": "nan", "5125": "nan", "5130": "nan", "5135": "nan", "5140": "nan", "5145": "nan", "5150": "nan", "5155": "nan", "5160": "nan", "5165": "nan", "5170": "nan", "5175": "nan", "5180": "nan", "5185": "nan", "5190": "nan", "5195": "nan", "5200": 3.39887, "5205": "nan", "5210": "nan", "5215": "nan", "5220": "nan", "5225": "nan", "5230": "nan", "5235": "nan", "5240": "nan", "5245": "nan", "5250": "nan", "5255": "nan", "5260": "nan", "5265": "nan", "5270": "nan", "5275": "nan", "5280": "nan", "5285": "nan", "5290": "nan", "5295": "nan", "5300": 3.3991, "5305": "nan", "5310": "nan", "5315": "nan", "5320": "nan", "5325": "nan", "5330": "nan", "5335": "nan", "5340": "nan", "5345": "nan", "5350": "nan", "5355": "nan", "5360": "nan", "5365": "nan", "5370": "nan", "5375": "nan", "5380": "nan", "5385": "nan", "5390": "nan", "5395": "nan", "5400": 3.40087, "5405": "nan", "5410": "nan", "5415": "nan", "5420": "nan", "5425": "nan", "5430": "nan", "5435": "nan", "5440": "nan", "5445": "nan", "5450": "nan", "5455": "nan", "5460": "nan", "5465": "nan", "5470": "nan", "5475": "nan", "5480": "nan", "5485": "nan", "5490": "nan", "5495": "nan", "5500": 3.40055, "5505": "nan", "5510": "nan", "5515": "nan", "5520": "nan", "5525": "nan", "5530": "nan", "5535": "nan", "5540": "nan", "5545": "nan", "5550": "nan", "5555": "nan", "5560": "nan", "5565": "nan", "5570": "nan", "5575": "nan", "5580": "nan", "5585": "nan", "5590": "nan", "5595": "nan", "5600": 3.40101, "5605": "nan", "5610": "nan", "5615": "nan", "5620": "nan", "5625": "nan", "5630": "nan", "5635": "nan", "5640": "nan", "5645": "nan", "5650": "nan", "5655": "nan", "5660": "nan", "5665": "nan", "5670": "nan", "5675": "nan", "5680": "nan", "5685": "nan", "5690": "nan", "5695": "nan", "5700": 3.4007, "5705": "nan", "5710": "nan", "5715": "nan", "5720": "nan", "5725": "nan", "5730": "nan", "5735": "nan", "5740": "nan", "5745": "nan", "5750": "nan", "5755": "nan", "5760": "nan", "5765": "nan", "5770": "nan", "5775": "nan", "5780": "nan", "5785": "nan", "5790": "nan", "5795": "nan", "5800": 3.40177, "5805": "nan", "5810": "nan", "5815": "nan", "5820": "nan", "5825": "nan", "5830": "nan", "5835": "nan", "5840": "nan", "5845": "nan", "5850": "nan", "5855": "nan", "5860": "nan", "5865": "nan", "5870": "nan", "5875": "nan", "5880": "nan", "5885": "nan", "5890": "nan", "5895": "nan", "5900": 3.40093, "5905": "nan", "5910": "nan", "5915": "nan", "5920": "nan", "5925": "nan", "5930": "nan", "5935": "nan", "5940": "nan", "5945": "nan", "5950": "nan", "5955": "nan", "5960": "nan", "5965": "nan", "5970": "nan", "5975": "nan", "5980": "nan", "5985": "nan", "5990": "nan", "5995": "nan", "6000": 3.40207, "6005": "nan", "6010": "nan", "6015": "nan", "6020": "nan", "6025": "nan", "6030": "nan", "6035": "nan", "6040": "nan", "6045": "nan", "6050": "nan", "6055": "nan", "6060": "nan", "6065": "nan", "6070": "nan", "6075": "nan", "6080": "nan", "6085": "nan", "6090": "nan", "6095": "nan", "6100": 3.40047, "6105": "nan", "6110": "nan", "6115": "nan", "6120": "nan", "6125": "nan", "6130": "nan", "6135": "nan", "6140": "nan", "6145": "nan", "6150": "nan", "6155": "nan", "6160": "nan", "6165": "nan", "6170": "nan", "6175": "nan", "6180": "nan", "6185": "nan", "6190": "nan", "6195": "nan", "6200": 3.40254, "6205": "nan", "6210": "nan", "6215": "nan", "6220": "nan", "6225": "nan", "6230": "nan", "6235": "nan", "6240": "nan", "6245": "nan", "6250": "nan", "6255": "nan", "6260": "nan", "6265": "nan", "6270": "nan", "6275": "nan", "6280": "nan", "6285": "nan", "6290": "nan", "6295": "nan", "6300": 3.4024, "6305": "nan", "6310": "nan", "6315": "nan", "6320": "nan", "6325": "nan", "6330": "nan", "6335": "nan", "6340": "nan", "6345": "nan", "6350": "nan", "6355": "nan", "6360": "nan", "6365": "nan", "6370": "nan", "6375": "nan", "6380": "nan", "6385": "nan", "6390": "nan", "6395": "nan", "6400": 3.40281, "6405": "nan", "6410": "nan", "6415": "nan", "6420": "nan", "6425": "nan", "6430": "nan", "6435": "nan", "6440": "nan", "6445": "nan", "6450": "nan", "6455": "nan", "6460": "nan", "6465": "nan", "6470": "nan", "6475": "nan", "6480": "nan", "6485": "nan", "6490": "nan", "6495": "nan", "6500": 3.40268, "6505": "nan", "6510": "nan", "6515": "nan", "6520": "nan", "6525": "nan", "6530": "nan", "6535": "nan", "6540": "nan", "6545": "nan", "6550": "nan", "6555": "nan", "6560": "nan", "6565": "nan", "6570": "nan", "6575": "nan", "6580": "nan", "6585": "nan", "6590": "nan", "6595": "nan", "6600": 3.40166, "6605": "nan", "6610": "nan", "6615": "nan", "6620": "nan", "6625": "nan", "6630": "nan", "6635": "nan", "6640": "nan", "6645": "nan", "6650": "nan", "6655": "nan", "6660": "nan", "6665": "nan", "6670": "nan", "6675": "nan", "6680": "nan", "6685": "nan", "6690": "nan", "6695": "nan", "6700": 3.40129, "6705": "nan", "6710": "nan", "6715": "nan", "6720": "nan", "6725": "nan", "6730": "nan", "6735": "nan", "6740": "nan", "6745": "nan", "6750": "nan", "6755": "nan", "6760": "nan", "6765": "nan", "6770": "nan", "6775": "nan", "6780": "nan", "6785": "nan", "6790": "nan", "6795": "nan", "6800": 3.40047, "6805": "nan", "6810": "nan", "6815": "nan", "6820": "nan", "6825": "nan", "6830": "nan", "6835": "nan", "6840": "nan", "6845": "nan", "6850": "nan", "6855": "nan", "6860": "nan", "6865": "nan", "6870": "nan", "6875": "nan", "6880": "nan", "6885": "nan", "6890": "nan", "6895": "nan", "6900": 3.40079, "6905": "nan", "6910": "nan", "6915": "nan", "6920": "nan", "6925": "nan", "6930": "nan", "6935": "nan", "6940": "nan", "6945": "nan", "6950": "nan", "6955": "nan", "6960": "nan", "6965": "nan", "6970": "nan", "6975": "nan", "6980": "nan", "6985": "nan", "6990": "nan", "6995": "nan", "7000": 3.40034, "7005": "nan", "7010": "nan", "7015": "nan", "7020": "nan", "7025": "nan", "7030": "nan", "7035": "nan", "7040": "nan", "7045": "nan", "7050": "nan", "7055": "nan", "7060": "nan", "7065": "nan", "7070": "nan", "7075": "nan", "7080": "nan", "7085": "nan", "7090": "nan", "7095": "nan", "7100": 3.39893, "7105": "nan", "7110": "nan", "7115": "nan", "7120": "nan", "7125": "nan", "7130": "nan", "7135": "nan", "7140": "nan", "7145": "nan", "7150": "nan", "7155": "nan", "7160": "nan", "7165": "nan", "7170": "nan", "7175": "nan", "7180": "nan", "7185": "nan", "7190": "nan", "7195": "nan", "7200": 3.40035, "7205": "nan", "7210": "nan", "7215": "nan", "7220": "nan", "7225": "nan", "7230": "nan", "7235": "nan", "7240": "nan", "7245": "nan", "7250": "nan", "7255": "nan", "7260": "nan", "7265": "nan", "7270": "nan", "7275": "nan", "7280": "nan", "7285": "nan", "7290": "nan", "7295": "nan", "7300": 3.39965, "7305": "nan", "7310": "nan", "7315": "nan", "7320": "nan", "7325": "nan", "7330": "nan", "7335": "nan", "7340": "nan", "7345": "nan", "7350": "nan", "7355": "nan", "7360": "nan", "7365": "nan", "7370": "nan", "7375": "nan", "7380": "nan", "7385": "nan", "7390": "nan", "7395": "nan", "7400": 3.40073, "7405": "nan", "7410": "nan", "7415": "nan", "7420": "nan", "7425": "nan", "7430": "nan", "7435": "nan", "7440": "nan", "7445": "nan", "7450": "nan", "7455": "nan", "7460": "nan", "7465": "nan", "7470": "nan", "7475": "nan", "7480": "nan", "7485": "nan", "7490": "nan", "7495": "nan", "7500": 3.40077, "7505": "nan", "7510": "nan", "7515": "nan", "7520": "nan", "7525": "nan", "7530": "nan", "7535": "nan", "7540": "nan", "7545": "nan", "7550": "nan", "7555": "nan", "7560": "nan", "7565": "nan", "7570": "nan", "7575": "nan", "7580": "nan", "7585": "nan", "7590": "nan", "7595": "nan", "7600": 3.39989, "7605": "nan", "7610": "nan", "7615": "nan", "7620": "nan", "7625": "nan", "7630": "nan", "7635": "nan", "7640": "nan", "7645": "nan", "7650": "nan", "7655": "nan", "7660": "nan", "7665": "nan", "7670": "nan", "7675": "nan", "7680": "nan", "7685": "nan", "7690": "nan", "7695": "nan", "7700": 3.3988, "7705": "nan", "7710": "nan", "7715": "nan", "7720": "nan", "7725": "nan", "7730": "nan", "7735": "nan", "7740": "nan", "7745": "nan", "7750": "nan", "7755": "nan", "7760": "nan", "7765": "nan", "7770": "nan", "7775": "nan", "7780": "nan", "7785": "nan", "7790": "nan", "7795": "nan", "7800": 3.3995, "7805": "nan", "7810": "nan", "7815": "nan", "7820": "nan", "7825": "nan", "7830": "nan", "7835": "nan", "7840": "nan", "7845": "nan", "7850": "nan", "7855": "nan", "7860": "nan", "7865": "nan", "7870": "nan", "7875": "nan", "7880": "nan", "7885": "nan", "7890": "nan", "7895": "nan", "7900": 3.39928, "7905": "nan", "7910": "nan", "7915": "nan", "7920": "nan", "7925": "nan", "7930": "nan", "7935": "nan", "7940": "nan", "7945": "nan", "7950": "nan", "7955": "nan", "7960": "nan", "7965": "nan", "7970": "nan", "7975": "nan", "7980": "nan", "7985": "nan", "7990": "nan", "7995": "nan", "8000": 3.39834, "8005": "nan", "8010": "nan", "8015": "nan", "8020": "nan", "8025": "nan", "8030": "nan", "8035": "nan", "8040": "nan", "8045": "nan", "8050": "nan", "8055": "nan", "8060": "nan", "8065": "nan", "8070": "nan", "8075": "nan", "8080": "nan", "8085": "nan", "8090": "nan", "8095": "nan", "8100": 3.39716, "8105": "nan", "8110": "nan", "8115": "nan", "8120": "nan", "8125": "nan", "8130": "nan", "8135": "nan", "8140": "nan", "8145": "nan", "8150": "nan", "8155": "nan", "8160": "nan", "8165": "nan", "8170": "nan", "8175": "nan", "8180": "nan", "8185": "nan", "8190": "nan", "8195": "nan", "8200": 3.39809, "8205": "nan", "8210": "nan", "8215": "nan", "8220": "nan", "8225": "nan", "8230": "nan", "8235": "nan", "8240": "nan", "8245": "nan", "8250": "nan", "8255": "nan", "8260": "nan", "8265": "nan", "8270": "nan", "8275": "nan", "8280": "nan", "8285": "nan", "8290": "nan", "8295": "nan", "8300": 3.39856, "8305": "nan", "8310": "nan", "8315": "nan", "8320": "nan", "8325": "nan", "8330": "nan", "8335": "nan", "8340": "nan", "8345": "nan", "8350": "nan", "8355": "nan", "8360": "nan", "8365": "nan", "8370": "nan", "8375": "nan", "8380": "nan", "8385": "nan", "8390": "nan", "8395": "nan", "8400": 3.39822, "8405": "nan", "8410": "nan", "8415": "nan", "8420": "nan", "8425": "nan", "8430": "nan", "8435": "nan", "8440": "nan", "8445": "nan", "8450": "nan", "8455": "nan", "8460": "nan", "8465": "nan", "8470": "nan", "8475": "nan", "8480": "nan", "8485": "nan", "8490": "nan", "8495": "nan", "8500": 3.39927, "8505": "nan", "8510": "nan", "8515": "nan", "8520": "nan", "8525": "nan", "8530": "nan", "8535": "nan", "8540": "nan", "8545": "nan", "8550": "nan", "8555": "nan", "8560": "nan", "8565": "nan", "8570": "nan", "8575": "nan", "8580": "nan", "8585": "nan", "8590": "nan", "8595": "nan", "8600": 3.40049, "8605": "nan", "8610": "nan", "8615": "nan", "8620": "nan", "8625": "nan", "8630": "nan", "8635": "nan", "8640": "nan", "8645": "nan", "8650": "nan", "8655": "nan", "8660": "nan", "8665": "nan", "8670": "nan", "8675": "nan", "8680": "nan", "8685": "nan", "8690": "nan", "8695": "nan", "8700": 3.39943, "8705": "nan", "8710": "nan", "8715": "nan", "8720": "nan", "8725": "nan", "8730": "nan", "8735": "nan", "8740": "nan", "8745": "nan", "8750": "nan", "8755": "nan", "8760": "nan", "8765": "nan", "8770": "nan", "8775": "nan", "8780": "nan", "8785": "nan", "8790": "nan", "8795": "nan", "8800": 3.3999, "8805": "nan", "8810": "nan", "8815": "nan", "8820": "nan", "8825": "nan", "8830": "nan", "8835": "nan", "8840": "nan", "8845": "nan", "8850": "nan", "8855": "nan", "8860": "nan", "8865": "nan", "8870": "nan", "8875": "nan", "8880": "nan", "8885": "nan", "8890": "nan", "8895": "nan", "8900": 3.39978, "8905": "nan", "8910": "nan", "8915": "nan", "8920": "nan", "8925": "nan", "8930": "nan", "8935": "nan", "8940": "nan", "8945": "nan", "8950": "nan", "8955": "nan", "8960": "nan", "8965": "nan", "8970": "nan", "8975": "nan", "8980": "nan", "8985": "nan", "8990": "nan", "8995": "nan", "9000": 3.40051, "9005": "nan", "9010": "nan", "9015": "nan", "9020": "nan", "9025": "nan", "9030": "nan", "9035": "nan", "9040": "nan", "9045": "nan", "9050": "nan", "9055": "nan", "9060": "nan", "9065": "nan", "9070": "nan", "9075": "nan", "9080": "nan", "9085": "nan", "9090": "nan", "9095": "nan", "9100": 3.39875, "9105": "nan", "9110": "nan", "9115": "nan", "9120": "nan", "9125": "nan", "9130": "nan", "9135": "nan", "9140": "nan", "9145": "nan", "9150": "nan", "9155": "nan", "9160": "nan", "9165": "nan", "9170": "nan", "9175": "nan", "9180": "nan", "9185": "nan", "9190": "nan", "9195": "nan", "9200": 3.39972, "9205": "nan", "9210": "nan", "9215": "nan", "9220": "nan", "9225": "nan", "9230": "nan", "9235": "nan", "9240": "nan", "9245": "nan", "9250": "nan", "9255": "nan", "9260": "nan", "9265": "nan", "9270": "nan", "9275": "nan", "9280": "nan", "9285": "nan", "9290": "nan", "9295": "nan", "9300": 3.40071, "9305": "nan", "9310": "nan", "9315": "nan", "9320": "nan", "9325": "nan", "9330": "nan", "9335": "nan", "9340": "nan", "9345": "nan", "9350": "nan", "9355": "nan", "9360": "nan", "9365": "nan", "9370": "nan", "9375": "nan", "9380": "nan", "9385": "nan", "9390": "nan", "9395": "nan", "9400": 3.40005, "9405": "nan", "9410": "nan", "9415": "nan", "9420": "nan", "9425": "nan", "9430": "nan", "9435": "nan", "9440": "nan", "9445": "nan", "9450": "nan", "9455": "nan", "9460": "nan", "9465": "nan", "9470": "nan", "9475": "nan", "9480": "nan", "9485": "nan", "9490": "nan", "9495": "nan", "9500": 3.39896, "9505": "nan", "9510": "nan", "9515": "nan", "9520": "nan", "9525": "nan", "9530": "nan", "9535": "nan", "9540": "nan", "9545": "nan", "9550": "nan", "9555": "nan", "9560": "nan", "9565": "nan", "9570": "nan", "9575": "nan", "9580": "nan", "9585": "nan", "9590": "nan", "9595": "nan", "9600": 3.39953, "9605": "nan", "9610": "nan", "9615": "nan", "9620": "nan", "9625": "nan", "9630": "nan", "9635": "nan", "9640": "nan", "9645": "nan", "9650": "nan", "9655": "nan", "9660": "nan", "9665": "nan", "9670": "nan", "9675": "nan", "9680": "nan", "9685": "nan", "9690": "nan", "9695": "nan", "9700": 3.39866, "9705": "nan", "9710": "nan", "9715": "nan", "9720": "nan", "9725": "nan", "9730": "nan", "9735": "nan", "9740": "nan", "9745": "nan", "9750": "nan", "9755": "nan", "9760": "nan", "9765": "nan", "9770": "nan", "9775": "nan", "9780": "nan", "9785": "nan", "9790": "nan", "9795": "nan", "9800": 3.40009, "9805": "nan", "9810": "nan", "9815": "nan", "9820": "nan", "9825": "nan", "9830": "nan", "9835": "nan", "9840": "nan", "9845": "nan", "9850": "nan", "9855": "nan", "9860": "nan", "9865": "nan", "9870": "nan", "9875": "nan", "9880": "nan", "9885": "nan", "9890": "nan", "9895": "nan", "9900": 3.39979, "9905": "nan", "9910": "nan", "9915": "nan", "9920": "nan", "9925": "nan", "9930": "nan", "9935": "nan", "9940": "nan", "9945": "nan", "9950": "nan", "9955": "nan", "9960": "nan", "9965": "nan", "9970": "nan", "9975": "nan", "9980": "nan", "9985": "nan", "9990": "nan", "9995": "nan", "10000": 3.3996, "10005": "nan", "10010": "nan", "10015": "nan", "10020": "nan", "10025": "nan", "10030": "nan", "10035": "nan", "10040": "nan", "10045": "nan", "10050": "nan", "10055": "nan", "10060": "nan", "10065": "nan", "10070": "nan", "10075": "nan", "10080": "nan", "10085": "nan", "10090": "nan", "10095": "nan", "10100": 3.39815, "10105": "nan", "10110": "nan", "10115": "nan", "10120": "nan", "10125": "nan", "10130": "nan", "10135": "nan", "10140": "nan", "10145": "nan", "10150": "nan", "10155": "nan", "10160": "nan", "10165": "nan", "10170": "nan", "10175": "nan", "10180": "nan", "10185": "nan", "10190": "nan", "10195": "nan", "10200": 3.3996, "10205": "nan", "10210": "nan", "10215": "nan", "10220": "nan", "10225": "nan", "10230": "nan", "10235": "nan", "10240": "nan", "10245": "nan", "10250": "nan", "10255": "nan", "10260": "nan", "10265": "nan", "10270": "nan", "10275": "nan", "10280": "nan", "10285": "nan", "10290": "nan", "10295": "nan", "10300": 3.40067, "10305": "nan", "10310": "nan", "10315": "nan", "10320": "nan", "10325": "nan", "10330": "nan", "10335": "nan", "10340": "nan", "10345": "nan", "10350": "nan", "10355": "nan", "10360": "nan", "10365": "nan", "10370": "nan", "10375": "nan", "10380": "nan", "10385": "nan", "10390": "nan", "10395": "nan", "10400": 3.39985, "10405": "nan", "10410": "nan", "10415": "nan", "10420": "nan", "10425": "nan", "10430": "nan", "10435": "nan", "10440": "nan", "10445": "nan", "10450": "nan", "10455": "nan", "10460": "nan", "10465": "nan", "10470": "nan", "10475": "nan", "10480": "nan", "10485": "nan", "10490": "nan", "10495": "nan", "10500": 3.39923, "10505": "nan", "10510": "nan", "10515": "nan", "10520": "nan", "10525": "nan", "10530": "nan", "10535": "nan", "10540": "nan", "10545": "nan", "10550": "nan", "10555": "nan", "10560": "nan", "10565": "nan", "10570": "nan", "10575": "nan", "10580": "nan", "10585": "nan", "10590": "nan", "10595": "nan", "10600": 3.40087, "10605": "nan", "10610": "nan", "10615": "nan", "10620": "nan", "10625": "nan", "10630": "nan", "10635": "nan", "10640": "nan", "10645": "nan", "10650": "nan", "10655": "nan", "10660": "nan", "10665": "nan", "10670": "nan", "10675": "nan", "10680": "nan", "10685": "nan", "10690": "nan", "10695": "nan", "10700": 3.40098, "10705": "nan", "10710": "nan", "10715": "nan", "10720": "nan", "10725": "nan", "10730": "nan", "10735": "nan", "10740": "nan", "10745": "nan", "10750": "nan", "10755": "nan", "10760": "nan", "10765": "nan", "10770": "nan", "10775": "nan", "10780": "nan", "10785": "nan", "10790": "nan", "10795": "nan", "10800": 3.39973, "10805": "nan", "10810": "nan", "10815": "nan", "10820": "nan", "10825": "nan", "10830": "nan", "10835": "nan", "10840": "nan", "10845": "nan", "10850": "nan", "10855": "nan", "10860": "nan", "10865": "nan", "10870": "nan", "10875": "nan", "10880": "nan", "10885": "nan", "10890": "nan", "10895": "nan", "10900": 3.66029, "10905": "nan", "10910": "nan", "10915": "nan", "10920": "nan", "10925": "nan", "10930": "nan", "10935": "nan", "10940": "nan", "10945": "nan", "10950": "nan", "10955": "nan", "10960": "nan", "10965": "nan", "10970": "nan", "10975": "nan", "10980": "nan", "10985": "nan", "10990": "nan", "10995": "nan", "11000": 3.38756, "11005": "nan", "11010": "nan", "11015": "nan", "11020": "nan", "11025": "nan", "11030": "nan", "11035": "nan", "11040": "nan", "11045": "nan", "11050": "nan", "11055": "nan", "11060": "nan", "11065": "nan", "11070": "nan", "11075": "nan", "11080": "nan", "11085": "nan", "11090": "nan", "11095": "nan", "11100": 3.38656, "11105": "nan", "11110": "nan", "11115": "nan", "11120": "nan", "11125": "nan", "11130": "nan", "11135": "nan", "11140": "nan", "11145": "nan", "11150": "nan", "11155": "nan", "11160": "nan", "11165": "nan", "11170": "nan", "11175": "nan", "11180": "nan", "11185": "nan", "11190": "nan", "11195": "nan", "11200": 3.38765, "11205": "nan", "11210": "nan", "11215": "nan", "11220": "nan", "11225": "nan", "11230": "nan", "11235": "nan", "11240": "nan", "11245": "nan", "11250": "nan", "11255": "nan", "11260": "nan", "11265": "nan", "11270": "nan", "11275": "nan", "11280": "nan", "11285": "nan", "11290": "nan", "11295": "nan", "11300": 3.38835, "11305": "nan", "11310": "nan", "11315": "nan", "11320": "nan", "11325": "nan", "11330": "nan", "11335": "nan", "11340": "nan", "11345": "nan", "11350": "nan", "11355": "nan", "11360": "nan", "11365": "nan", "11370": "nan", "11375": "nan", "11380": "nan", "11385": "nan", "11390": "nan", "11395": "nan", "11400": 3.38844, "11405": "nan", "11410": "nan", "11415": "nan", "11420": "nan", "11425": "nan", "11430": "nan", "11435": "nan", "11440": "nan", "11445": "nan", "11450": "nan", "11455": "nan", "11460": "nan", "11465": "nan", "11470": "nan", "11475": "nan", "11480": "nan", "11485": "nan", "11490": "nan", "11495": "nan", "11500": 3.38793, "11505": "nan", "11510": "nan", "11515": "nan", "11520": "nan", "11525": "nan", "11530": "nan", "11535": "nan", "11540": "nan", "11545": "nan", "11550": "nan", "11555": "nan", "11560": "nan", "11565": "nan", "11570": "nan", "11575": "nan", "11580": "nan", "11585": "nan", "11590": "nan", "11595": "nan", "11600": 3.38725, "11605": "nan", "11610": "nan", "11615": "nan", "11620": "nan", "11625": "nan", "11630": "nan", "11635": "nan", "11640": "nan", "11645": "nan", "11650": "nan", "11655": "nan", "11660": "nan", "11665": "nan", "11670": "nan", "11675": "nan", "11680": "nan", "11685": "nan", "11690": "nan", "11695": "nan", "11700": 3.38738, "11705": "nan", "11710": "nan", "11715": "nan", "11720": "nan", "11725": "nan", "11730": "nan", "11735": "nan", "11740": "nan", "11745": "nan", "11750": "nan", "11755": "nan", "11760": "nan", "11765": "nan", "11770": "nan", "11775": "nan", "11780": "nan", "11785": "nan", "11790": "nan", "11795": "nan", "11800": 3.38748, "11805": "nan", "11810": "nan", "11815": "nan", "11820": "nan", "11825": "nan", "11830": "nan", "11835": "nan", "11840": "nan", "11845": "nan", "11850": "nan", "11855": "nan", "11860": "nan", "11865": "nan", "11870": "nan", "11875": "nan", "11880": "nan", "11885": "nan", "11890": "nan", "11895": "nan", "11900": 3.38839, "11905": "nan", "11910": "nan", "11915": "nan", "11920": "nan", "11925": "nan", "11930": "nan", "11935": "nan", "11940": "nan", "11945": "nan", "11950": "nan", "11955": "nan", "11960": "nan", "11965": "nan", "11970": "nan", "11975": "nan", "11980": "nan", "11985": "nan", "11990": "nan", "11995": "nan", "12000": 3.38814, "12005": "nan", "12010": "nan", "12015": "nan", "12020": "nan", "12025": "nan", "12030": "nan", "12035": "nan", "12040": "nan", "12045": "nan", "12050": "nan", "12055": "nan", "12060": "nan", "12065": "nan", "12070": "nan", "12075": "nan", "12080": "nan", "12085": "nan", "12090": "nan", "12095": "nan", "12100": 3.38677, "12105": "nan", "12110": "nan", "12115": "nan", "12120": "nan", "12125": "nan", "12130": "nan", "12135": "nan", "12140": "nan", "12145": "nan", "12150": "nan", "12155": "nan", "12160": "nan", "12165": "nan", "12170": "nan", "12175": "nan", "12180": "nan", "12185": "nan", "12190": "nan", "12195": "nan", "12200": 3.38679, "12205": "nan", "12210": "nan", "12215": "nan", "12220": "nan", "12225": "nan", "12230": "nan", "12235": "nan", "12240": "nan", "12245": "nan", "12250": "nan", "12255": "nan", "12260": "nan", "12265": "nan", "12270": "nan", "12275": "nan", "12280": "nan", "12285": "nan", "12290": "nan", "12295": "nan", "12300": 3.38609, "12305": "nan", "12310": "nan", "12315": "nan", "12320": "nan", "12325": "nan", "12330": "nan", "12335": "nan", "12340": "nan", "12345": "nan", "12350": "nan", "12355": "nan", "12360": "nan", "12365": "nan", "12370": "nan", "12375": "nan", "12380": "nan", "12385": "nan", "12390": "nan", "12395": "nan", "12400": 3.38665, "12405": "nan", "12410": "nan", "12415": "nan", "12420": "nan", "12425": "nan", "12430": "nan", "12435": "nan", "12440": "nan", "12445": "nan", "12450": "nan", "12455": "nan", "12460": "nan", "12465": "nan", "12470": "nan", "12475": "nan", "12480": "nan", "12485": "nan", "12490": "nan", "12495": "nan", "12500": 3.38727, "12505": "nan", "12510": "nan", "12515": "nan", "12520": "nan", "12525": "nan", "12530": "nan", "12535": "nan", "12540": "nan", "12545": "nan", "12550": "nan", "12555": "nan", "12560": "nan", "12565": "nan", "12570": "nan", "12575": "nan", "12580": "nan", "12585": "nan", "12590": "nan", "12595": "nan", "12600": 3.38752, "12605": "nan", "12610": "nan", "12615": "nan", "12620": "nan", "12625": "nan", "12630": "nan", "12635": "nan", "12640": "nan", "12645": "nan", "12650": "nan", "12655": "nan", "12660": "nan", "12665": "nan", "12670": "nan", "12675": "nan", "12680": "nan", "12685": "nan", "12690": "nan", "12695": "nan", "12700": 3.38807, "12705": "nan", "12710": "nan", "12715": "nan", "12720": "nan", "12725": "nan", "12730": "nan", "12735": "nan", "12740": "nan", "12745": "nan", "12750": "nan", "12755": "nan", "12760": "nan", "12765": "nan", "12770": "nan", "12775": "nan", "12780": "nan", "12785": "nan", "12790": "nan", "12795": "nan", "12800": "nan", "12805": "nan", "12810": "nan", "12815": "nan", "12820": "nan", "12825": "nan", "12830": "nan", "12835": "nan", "12840": "nan", "12845": "nan", "12850": "nan", "12855": "nan", "12860": "nan", "12865": "nan", "12870": "nan", "12875": "nan", "12880": "nan", "12885": "nan", "12890": "nan", "12895": "nan", "12900": "nan", "12905": "nan", "12910": "nan", "12915": "nan", "12920": "nan", "12925": "nan", "12930": "nan", "12935": "nan", "12940": "nan", "12945": "nan", "12950": "nan", "12955": "nan", "12960": "nan", "12965": "nan", "12970": "nan", "12975": "nan", "12980": "nan", "12985": "nan", "12990": "nan", "12995": "nan", "13000": "nan"}}}
\ No newline at end of file
+{
+ "lm loss": {
+ "start_step": 1,
+ "end_step": 10200,
+ "step_interval": 5,
+ "values": {
+ "1": 12.95636,
+ "5": 12.91493,
+ "10": 12.06068,
+ "15": 11.39907,
+ "20": 10.42782,
+ "25": 9.9771,
+ "30": 9.62861,
+ "35": 9.36625,
+ "40": 9.17252,
+ "45": 9.00247,
+ "50": 8.84227,
+ "55": 8.635,
+ "60": 8.61848,
+ "65": 8.51386,
+ "70": 8.39546,
+ "75": 8.26338,
+ "80": 8.09298,
+ "85": 8.09858,
+ "90": 7.94846,
+ "95": 7.8971,
+ "100": 7.80875,
+ "105": 7.65613,
+ "110": 7.56901,
+ "115": 7.48001,
+ "120": 7.41832,
+ "125": 7.47105,
+ "130": 7.28349,
+ "135": 7.29674,
+ "140": 7.25373,
+ "145": 7.06382,
+ "150": 7.15942,
+ "155": 7.00232,
+ "160": 6.92577,
+ "165": 6.89706,
+ "170": 6.81939,
+ "175": 6.86062,
+ "180": 6.81545,
+ "185": 6.72257,
+ "190": 6.65587,
+ "195": 6.58935,
+ "200": 6.6135,
+ "205": 6.63845,
+ "210": 6.53563,
+ "215": 6.51843,
+ "220": 6.51177,
+ "225": 6.50414,
+ "230": 6.50432,
+ "235": 6.46917,
+ "240": 6.38517,
+ "245": 6.3812,
+ "250": 6.33511,
+ "255": 6.449,
+ "260": 6.33771,
+ "265": 6.27428,
+ "270": 6.23222,
+ "275": 6.27245,
+ "280": 6.20586,
+ "285": 6.23286,
+ "290": 6.16479,
+ "295": 6.145,
+ "300": 6.10719,
+ "305": 6.0417,
+ "310": 6.07263,
+ "315": 6.07514,
+ "320": 5.98003,
+ "325": 5.92946,
+ "330": 5.98563,
+ "335": 6.03283,
+ "340": 5.95895,
+ "345": 5.93647,
+ "350": 5.92017,
+ "355": 5.8495,
+ "360": 5.85881,
+ "365": 5.8328,
+ "370": 5.79334,
+ "375": 5.83779,
+ "380": 5.83614,
+ "385": 5.78613,
+ "390": 5.7777,
+ "395": 5.67732,
+ "400": 5.65984,
+ "405": 5.66333,
+ "410": 5.66665,
+ "415": 5.72633,
+ "420": 5.66944,
+ "425": 5.68244,
+ "430": 5.63871,
+ "435": 5.56635,
+ "440": 5.61469,
+ "445": 5.54454,
+ "450": 5.59407,
+ "455": 5.50781,
+ "460": 5.50087,
+ "465": 5.58081,
+ "470": 5.56589,
+ "475": 5.50951,
+ "480": 5.47548,
+ "485": 5.50235,
+ "490": 5.46533,
+ "495": 5.45996,
+ "500": 5.40736,
+ "505": 5.36926,
+ "510": 5.4224,
+ "515": 5.41691,
+ "520": 5.43066,
+ "525": 5.30525,
+ "530": 5.34552,
+ "535": 5.31057,
+ "540": 5.3297,
+ "545": 5.39306,
+ "550": 5.36947,
+ "555": 5.19608,
+ "560": 5.32548,
+ "565": 5.28038,
+ "570": 5.27177,
+ "575": 5.29276,
+ "580": 5.21254,
+ "585": 5.21476,
+ "590": 5.21614,
+ "595": 5.21547,
+ "600": 5.23604,
+ "605": 5.22126,
+ "610": 5.19884,
+ "615": 5.15789,
+ "620": 5.19311,
+ "625": 5.18747,
+ "630": 5.12499,
+ "635": 5.10527,
+ "640": 5.08589,
+ "645": 5.09422,
+ "650": 5.13357,
+ "655": 5.10849,
+ "660": 5.04614,
+ "665": 5.06634,
+ "670": 5.03349,
+ "675": 4.99757,
+ "680": 4.98096,
+ "685": 4.9669,
+ "690": 4.99708,
+ "695": 4.94269,
+ "700": 4.94045,
+ "705": 4.89612,
+ "710": 4.94295,
+ "715": 4.87556,
+ "720": 4.83941,
+ "725": 4.79045,
+ "730": 4.83137,
+ "735": 4.78843,
+ "740": 4.815,
+ "745": 4.70979,
+ "750": 4.70037,
+ "755": 4.7934,
+ "760": 4.75129,
+ "765": 4.70711,
+ "770": 4.65835,
+ "775": 4.64785,
+ "780": 4.68267,
+ "785": 4.72822,
+ "790": 4.61917,
+ "795": 4.6209,
+ "800": 4.58307,
+ "805": 4.60223,
+ "810": 4.62062,
+ "815": 4.58748,
+ "820": 4.59547,
+ "825": 4.57325,
+ "830": 4.54908,
+ "835": 4.52106,
+ "840": 4.46661,
+ "845": 4.46341,
+ "850": 4.41993,
+ "855": 4.46928,
+ "860": 4.41115,
+ "865": 4.49884,
+ "870": 4.44657,
+ "875": 4.35033,
+ "880": 4.38974,
+ "885": 4.36025,
+ "890": 4.40871,
+ "895": 4.39389,
+ "900": 4.35994,
+ "905": 4.31426,
+ "910": 4.32151,
+ "915": 4.31741,
+ "920": 4.3601,
+ "925": 4.35556,
+ "930": 4.28626,
+ "935": 4.28016,
+ "940": 4.31823,
+ "945": 4.28675,
+ "950": 4.33146,
+ "955": 4.23865,
+ "960": 4.17329,
+ "965": 4.24662,
+ "970": 4.26306,
+ "975": 4.21677,
+ "980": 4.20264,
+ "985": 4.16318,
+ "990": 4.12123,
+ "995": 4.17995,
+ "1000": 4.2382,
+ "1005": 4.17489,
+ "1010": 4.15353,
+ "1015": 4.10781,
+ "1020": 4.14472,
+ "1025": 4.20371,
+ "1030": 4.09909,
+ "1035": 4.09186,
+ "1040": 4.09316,
+ "1045": 4.09014,
+ "1050": 4.11964,
+ "1055": 4.09834,
+ "1060": 4.09551,
+ "1065": 4.05398,
+ "1070": 4.04751,
+ "1075": 4.06679,
+ "1080": 4.05176,
+ "1085": 4.05513,
+ "1090": 4.0203,
+ "1095": 4.08006,
+ "1100": 4.05952,
+ "1105": 4.06174,
+ "1110": 4.04279,
+ "1115": 3.99444,
+ "1120": 4.00593,
+ "1125": 3.9937,
+ "1130": 4.0578,
+ "1135": 4.00553,
+ "1140": 3.98485,
+ "1145": 3.93503,
+ "1150": 4.02991,
+ "1155": 3.98774,
+ "1160": 3.94876,
+ "1165": 3.85343,
+ "1170": 3.91359,
+ "1175": 3.93274,
+ "1180": 3.93706,
+ "1185": 3.97637,
+ "1190": 3.91901,
+ "1195": 3.93409,
+ "1200": 3.87706,
+ "1205": 3.87883,
+ "1210": 3.98038,
+ "1215": 3.82721,
+ "1220": 3.86817,
+ "1225": 3.80347,
+ "1230": 3.89882,
+ "1235": 3.88541,
+ "1240": 3.85377,
+ "1245": 3.79592,
+ "1250": 3.81771,
+ "1255": 3.85764,
+ "1260": 3.88133,
+ "1265": 3.77832,
+ "1270": 3.86905,
+ "1275": 3.82128,
+ "1280": 3.81356,
+ "1285": 3.82855,
+ "1290": 3.84927,
+ "1295": 3.84003,
+ "1300": 3.8012,
+ "1305": 3.81011,
+ "1310": 3.8163,
+ "1315": 3.80504,
+ "1320": 3.81781,
+ "1325": 3.71094,
+ "1330": 3.78861,
+ "1335": 3.7482,
+ "1340": 3.74059,
+ "1345": 3.76409,
+ "1350": 3.72869,
+ "1355": 3.78241,
+ "1360": 3.72585,
+ "1365": 3.72505,
+ "1370": 3.74271,
+ "1375": 3.75098,
+ "1380": 3.7467,
+ "1385": 3.74016,
+ "1390": 3.66738,
+ "1395": 3.73768,
+ "1400": 3.73285,
+ "1405": 3.66947,
+ "1410": 3.65899,
+ "1415": 3.65272,
+ "1420": 3.70798,
+ "1425": 3.70326,
+ "1430": 3.67713,
+ "1435": 3.65521,
+ "1440": 3.62359,
+ "1445": 3.67462,
+ "1450": 3.66895,
+ "1455": 3.63408,
+ "1460": 3.65242,
+ "1465": 3.67233,
+ "1470": 3.61501,
+ "1475": 3.68638,
+ "1480": 3.65857,
+ "1485": 3.67278,
+ "1490": 3.63029,
+ "1495": 3.60394,
+ "1500": 3.65552,
+ "1505": 3.68609,
+ "1510": 3.55109,
+ "1515": 3.59939,
+ "1520": 3.62723,
+ "1525": 3.59192,
+ "1530": 3.59983,
+ "1535": 3.58568,
+ "1540": 3.61642,
+ "1545": 3.59448,
+ "1550": 3.56613,
+ "1555": 3.57416,
+ "1560": 3.60143,
+ "1565": 3.62419,
+ "1570": 3.59238,
+ "1575": 3.55103,
+ "1580": 3.61463,
+ "1585": 3.57681,
+ "1590": 3.46422,
+ "1595": 3.50863,
+ "1600": 3.49103,
+ "1605": 3.53794,
+ "1610": 3.56037,
+ "1615": 3.49817,
+ "1620": 3.52298,
+ "1625": 3.46961,
+ "1630": 3.49722,
+ "1635": 3.5368,
+ "1640": 3.52869,
+ "1645": 3.53266,
+ "1650": 3.47711,
+ "1655": 3.4673,
+ "1660": 3.51444,
+ "1665": 3.45262,
+ "1670": 3.51621,
+ "1675": 3.49635,
+ "1680": 3.47136,
+ "1685": 3.4794,
+ "1690": 3.47578,
+ "1695": 3.49278,
+ "1700": 3.47195,
+ "1705": 3.39545,
+ "1710": 3.48055,
+ "1715": 3.48126,
+ "1720": 3.42363,
+ "1725": 3.41313,
+ "1730": 3.41695,
+ "1735": 3.48191,
+ "1740": 3.45584,
+ "1745": 3.42798,
+ "1750": 3.39635,
+ "1755": 3.41903,
+ "1760": 3.36551,
+ "1765": 3.41537,
+ "1770": 3.44954,
+ "1775": 3.37407,
+ "1780": 3.43219,
+ "1785": 3.41375,
+ "1790": 3.38142,
+ "1795": 3.39439,
+ "1800": 3.33122,
+ "1805": 3.38289,
+ "1810": 3.33314,
+ "1815": 3.42456,
+ "1820": 3.41124,
+ "1825": 3.38737,
+ "1830": 3.321,
+ "1835": 3.42972,
+ "1840": 3.3964,
+ "1845": 3.43042,
+ "1850": 3.37212,
+ "1855": 3.36489,
+ "1860": 3.33899,
+ "1865": 3.38416,
+ "1870": 3.2974,
+ "1875": 3.43275,
+ "1880": 3.33858,
+ "1885": 3.34035,
+ "1890": 3.33182,
+ "1895": 3.38805,
+ "1900": 3.37148,
+ "1905": 3.29374,
+ "1910": 3.32697,
+ "1915": 3.30981,
+ "1920": 3.35061,
+ "1925": 3.32954,
+ "1930": 3.30997,
+ "1935": 3.29828,
+ "1940": 3.34822,
+ "1945": 3.26885,
+ "1950": 3.40411,
+ "1955": 3.29406,
+ "1960": 3.29982,
+ "1965": 3.26189,
+ "1970": 3.29411,
+ "1975": 3.32491,
+ "1980": 3.3304,
+ "1985": 3.23214,
+ "1990": 3.29106,
+ "1995": 3.27313,
+ "2000": 3.26868,
+ "2005": 3.24857,
+ "2010": 3.24793,
+ "2015": 3.21842,
+ "2020": 3.26421,
+ "2025": 3.27751,
+ "2030": 3.27857,
+ "2035": 3.27744,
+ "2040": 3.24719,
+ "2045": 3.23203,
+ "2050": 3.26826,
+ "2055": 3.31971,
+ "2060": 3.2822,
+ "2065": 3.22581,
+ "2070": 3.27883,
+ "2075": 3.24034,
+ "2080": 3.22343,
+ "2085": 3.29053,
+ "2090": 3.13251,
+ "2095": 3.29421,
+ "2100": 3.22448,
+ "2105": 3.18523,
+ "2110": 3.18724,
+ "2115": 3.21807,
+ "2120": 3.17889,
+ "2125": 3.2113,
+ "2130": 3.20286,
+ "2135": 3.26519,
+ "2140": 3.18758,
+ "2145": 3.19507,
+ "2150": 3.20824,
+ "2155": 3.22091,
+ "2160": 3.19528,
+ "2165": 3.35955,
+ "2170": 3.2585,
+ "2175": 3.17861,
+ "2180": 3.21668,
+ "2185": 3.24726,
+ "2190": 3.23016,
+ "2195": 3.14745,
+ "2200": 3.18543,
+ "2205": 3.16084,
+ "2210": 3.11745,
+ "2215": 3.1858,
+ "2220": 3.18572,
+ "2225": 3.17846,
+ "2230": 3.14655,
+ "2235": 3.17171,
+ "2240": 3.22575,
+ "2245": 3.17054,
+ "2250": 3.19236,
+ "2255": 3.12598,
+ "2260": 3.12323,
+ "2265": 3.2398,
+ "2270": 3.17957,
+ "2275": 3.14483,
+ "2280": 3.19429,
+ "2285": 3.17293,
+ "2290": 3.16102,
+ "2295": 3.19661,
+ "2300": 3.12642,
+ "2305": 3.15285,
+ "2310": 3.11359,
+ "2315": 3.05962,
+ "2320": 3.10881,
+ "2325": 3.15813,
+ "2330": 3.11558,
+ "2335": 3.1051,
+ "2340": 3.16096,
+ "2345": 3.12605,
+ "2350": 3.14231,
+ "2355": 3.14361,
+ "2360": 3.15245,
+ "2365": 3.0827,
+ "2370": 3.14312,
+ "2375": 3.14177,
+ "2380": 3.12494,
+ "2385": 3.06938,
+ "2390": 3.08808,
+ "2395": 3.09036,
+ "2400": 3.08116,
+ "2405": 3.08824,
+ "2410": 3.07707,
+ "2415": 3.08042,
+ "2420": 3.06917,
+ "2425": 3.06886,
+ "2430": 3.07357,
+ "2435": 3.05462,
+ "2440": 3.08959,
+ "2445": 3.05963,
+ "2450": 3.12048,
+ "2455": 3.15148,
+ "2460": 3.07551,
+ "2465": 3.06878,
+ "2470": 3.02726,
+ "2475": 3.06894,
+ "2480": 3.08459,
+ "2485": 3.04843,
+ "2490": 3.05287,
+ "2495": 3.06025,
+ "2500": 3.06094,
+ "2505": 3.12591,
+ "2510": 3.11813,
+ "2515": 3.06371,
+ "2520": 3.06863,
+ "2525": 3.01919,
+ "2530": 3.04918,
+ "2535": 3.0825,
+ "2540": 3.07124,
+ "2545": 3.04337,
+ "2550": 2.99371,
+ "2555": 3.0642,
+ "2560": 3.03432,
+ "2565": 3.10038,
+ "2570": 3.02333,
+ "2575": 3.04987,
+ "2580": 3.08227,
+ "2585": 3.01065,
+ "2590": 3.06011,
+ "2595": 3.00654,
+ "2600": 3.05996,
+ "2605": 3.04069,
+ "2610": 3.04466,
+ "2615": 3.05458,
+ "2620": 2.98698,
+ "2625": 3.00577,
+ "2630": 3.03121,
+ "2635": 3.0586,
+ "2640": 3.0119,
+ "2645": 3.04364,
+ "2650": 3.01519,
+ "2655": 2.9886,
+ "2660": 3.00648,
+ "2665": 3.03526,
+ "2670": 2.97842,
+ "2675": 2.95563,
+ "2680": 2.98899,
+ "2685": 3.00002,
+ "2690": 2.99684,
+ "2695": 2.98246,
+ "2700": 3.02031,
+ "2705": 2.97486,
+ "2710": 2.97993,
+ "2715": 2.95982,
+ "2720": 3.02327,
+ "2725": 2.98962,
+ "2730": 3.03754,
+ "2735": 3.04423,
+ "2740": 3.00496,
+ "2745": 3.03712,
+ "2750": 3.01402,
+ "2755": 2.9617,
+ "2760": 2.99646,
+ "2765": 3.00037,
+ "2770": 2.97044,
+ "2775": 2.98405,
+ "2780": 3.00503,
+ "2785": 2.94687,
+ "2790": 2.95285,
+ "2795": 2.9441,
+ "2800": 2.95258,
+ "2805": 2.93276,
+ "2810": 2.97571,
+ "2815": 2.95532,
+ "2820": 2.98101,
+ "2825": 3.00437,
+ "2830": 2.98552,
+ "2835": 2.91254,
+ "2840": 2.92325,
+ "2845": 2.95348,
+ "2850": 2.96382,
+ "2855": 2.96228,
+ "2860": 2.95491,
+ "2865": 2.91272,
+ "2870": 2.98144,
+ "2875": 2.91969,
+ "2880": 2.95539,
+ "2885": 2.9164,
+ "2890": 2.97968,
+ "2895": 2.9266,
+ "2900": 2.951,
+ "2905": 3.00319,
+ "2910": 2.93046,
+ "2915": 2.97319,
+ "2920": 2.95673,
+ "2925": 2.94019,
+ "2930": 2.94216,
+ "2935": 2.93167,
+ "2940": 2.92828,
+ "2945": 2.91138,
+ "2950": 2.97326,
+ "2955": 2.90742,
+ "2960": 2.96518,
+ "2965": 2.86466,
+ "2970": 2.95452,
+ "2975": 2.98998,
+ "2980": 2.93403,
+ "2985": 3.0293,
+ "2990": 2.94204,
+ "2995": 2.86476,
+ "3000": 2.92558,
+ "3005": 2.88291,
+ "3010": 2.93006,
+ "3015": 2.91195,
+ "3020": 2.91509,
+ "3025": 2.91667,
+ "3030": 2.91981,
+ "3035": 2.95562,
+ "3040": 2.91897,
+ "3045": 2.83026,
+ "3050": 2.89438,
+ "3055": 2.89872,
+ "3060": 2.92425,
+ "3065": 2.9202,
+ "3070": 2.87732,
+ "3075": 2.8659,
+ "3080": 2.91886,
+ "3085": 2.89913,
+ "3090": 2.91337,
+ "3095": 2.91795,
+ "3100": 2.86455,
+ "3105": 2.91991,
+ "3110": 2.89521,
+ "3115": 2.94229,
+ "3120": 2.9502,
+ "3125": 2.85442,
+ "3130": 2.92898,
+ "3135": 2.91391,
+ "3140": 2.87153,
+ "3145": 2.91673,
+ "3150": 2.84677,
+ "3155": 2.84841,
+ "3160": 2.84268,
+ "3165": 2.83843,
+ "3170": 2.88775,
+ "3175": 2.90031,
+ "3180": 2.85186,
+ "3185": 2.88764,
+ "3190": 2.90097,
+ "3195": 2.9244,
+ "3200": 2.91971,
+ "3205": 2.86321,
+ "3210": 2.87135,
+ "3215": 2.91131,
+ "3220": 2.86938,
+ "3225": 2.86623,
+ "3230": 2.81226,
+ "3235": 2.86736,
+ "3240": 2.8667,
+ "3245": 2.90142,
+ "3250": 2.85616,
+ "3255": 2.8457,
+ "3260": 2.85684,
+ "3265": 2.86296,
+ "3270": 2.83877,
+ "3275": 2.86179,
+ "3280": 2.7955,
+ "3285": 2.80704,
+ "3290": 2.8637,
+ "3295": 2.89374,
+ "3300": 2.86884,
+ "3305": 2.85627,
+ "3310": 2.85096,
+ "3315": 2.8031,
+ "3320": 2.81957,
+ "3325": 2.8231,
+ "3330": 2.82253,
+ "3335": 2.84275,
+ "3340": 2.82275,
+ "3345": 2.83801,
+ "3350": 2.88535,
+ "3355": 2.93845,
+ "3360": 2.8104,
+ "3365": 2.88218,
+ "3370": 2.84292,
+ "3375": 2.82821,
+ "3380": 2.83961,
+ "3385": 2.86845,
+ "3390": 2.85562,
+ "3395": 2.79888,
+ "3400": 2.7751,
+ "3405": 2.82035,
+ "3410": 2.83623,
+ "3415": 2.85155,
+ "3420": 2.82276,
+ "3425": 2.80723,
+ "3430": 2.82699,
+ "3435": 2.88405,
+ "3440": 2.81654,
+ "3445": 2.8666,
+ "3450": 2.81173,
+ "3455": 2.78937,
+ "3460": 2.81326,
+ "3465": 2.85496,
+ "3470": 2.83669,
+ "3475": 2.76818,
+ "3480": 2.83952,
+ "3485": 2.81857,
+ "3490": 2.88583,
+ "3495": 2.84954,
+ "3500": 2.8294,
+ "3505": 2.82724,
+ "3510": 2.81197,
+ "3515": 2.83002,
+ "3520": 2.77447,
+ "3525": 2.80315,
+ "3530": 2.84695,
+ "3535": 2.77826,
+ "3540": 2.83561,
+ "3545": 2.80697,
+ "3550": 2.79352,
+ "3555": 2.81372,
+ "3560": 2.81919,
+ "3565": 2.82503,
+ "3570": 2.80064,
+ "3575": 2.80282,
+ "3580": 2.81963,
+ "3585": 2.83113,
+ "3590": 2.82883,
+ "3595": 2.77456,
+ "3600": 2.74549,
+ "3605": 2.79356,
+ "3610": 2.85251,
+ "3615": 2.75065,
+ "3620": 2.80136,
+ "3625": 2.88028,
+ "3630": 2.77415,
+ "3635": 2.78563,
+ "3640": 2.78098,
+ "3645": 2.7678,
+ "3650": 2.80164,
+ "3655": 2.81848,
+ "3660": 2.76378,
+ "3665": 2.78241,
+ "3670": 2.76706,
+ "3675": 2.7699,
+ "3680": 2.80551,
+ "3685": 2.79657,
+ "3690": 2.7994,
+ "3695": 2.80976,
+ "3700": 2.78825,
+ "3705": 2.78652,
+ "3710": 2.75031,
+ "3715": 2.80152,
+ "3720": 2.79403,
+ "3725": 2.78917,
+ "3730": 2.83544,
+ "3735": 2.7951,
+ "3740": 2.75106,
+ "3745": 2.78944,
+ "3750": 2.80263,
+ "3755": 2.79078,
+ "3760": 2.75654,
+ "3765": 2.75784,
+ "3770": 2.76197,
+ "3775": 2.76756,
+ "3780": 2.75776,
+ "3785": 2.77592,
+ "3790": 2.73997,
+ "3795": 2.78753,
+ "3800": 2.79987,
+ "3805": 2.74849,
+ "3810": 2.80307,
+ "3815": 2.76119,
+ "3820": 2.78756,
+ "3825": 2.73454,
+ "3830": 2.7435,
+ "3835": 2.8103,
+ "3840": 2.72999,
+ "3845": 2.7159,
+ "3850": 2.77108,
+ "3855": 2.71808,
+ "3860": 2.80515,
+ "3865": 2.74932,
+ "3870": 2.77188,
+ "3875": 2.75713,
+ "3880": 2.78467,
+ "3885": 2.77955,
+ "3890": 2.74036,
+ "3895": 2.79847,
+ "3900": 2.7638,
+ "3905": 2.71774,
+ "3910": 2.74179,
+ "3915": 2.75467,
+ "3920": 2.79458,
+ "3925": 2.77793,
+ "3930": 2.7083,
+ "3935": 2.74319,
+ "3940": 2.74853,
+ "3945": 2.74033,
+ "3950": 2.71968,
+ "3955": 2.77484,
+ "3960": 2.76006,
+ "3965": 2.73678,
+ "3970": 2.75662,
+ "3975": 2.72787,
+ "3980": 2.7365,
+ "3985": 2.74465,
+ "3990": 2.69651,
+ "3995": 2.78349,
+ "4000": 2.73738,
+ "4005": 2.77091,
+ "4010": 2.71132,
+ "4015": 2.7268,
+ "4020": 2.7481,
+ "4025": 2.73123,
+ "4030": 2.6573,
+ "4035": 2.69717,
+ "4040": 2.75386,
+ "4045": 2.74786,
+ "4050": 2.78816,
+ "4055": 2.72171,
+ "4060": 2.71292,
+ "4065": 2.65324,
+ "4070": 2.80526,
+ "4075": 2.75436,
+ "4080": 2.71672,
+ "4085": 2.74687,
+ "4090": 2.67753,
+ "4095": 2.69318,
+ "4100": 2.71292,
+ "4105": 2.73681,
+ "4110": 2.73132,
+ "4115": 2.70148,
+ "4120": 2.72697,
+ "4125": 2.70056,
+ "4130": 2.69693,
+ "4135": 2.68794,
+ "4140": 2.68045,
+ "4145": 2.78003,
+ "4150": 2.70964,
+ "4155": 2.73827,
+ "4160": 2.75924,
+ "4165": 2.71992,
+ "4170": 2.6732,
+ "4175": 2.71977,
+ "4180": 2.72516,
+ "4185": 2.7266,
+ "4190": 2.74574,
+ "4195": 2.69323,
+ "4200": 2.70389,
+ "4205": 2.739,
+ "4210": 2.67821,
+ "4215": 2.66615,
+ "4220": 2.66041,
+ "4225": 2.70194,
+ "4230": 2.72519,
+ "4235": 2.74505,
+ "4240": 2.70556,
+ "4245": 2.70809,
+ "4250": 2.71276,
+ "4255": 2.64758,
+ "4260": 2.72108,
+ "4265": 2.7407,
+ "4270": 2.71826,
+ "4275": 2.68449,
+ "4280": 2.69872,
+ "4285": 2.72975,
+ "4290": 2.68276,
+ "4295": 2.69079,
+ "4300": 2.70099,
+ "4305": 2.69702,
+ "4310": 2.72824,
+ "4315": 2.71117,
+ "4320": 2.6963,
+ "4325": 2.70147,
+ "4330": 2.70861,
+ "4335": 2.69073,
+ "4340": 2.69714,
+ "4345": 2.72371,
+ "4350": 2.67452,
+ "4355": 2.69376,
+ "4360": 2.70886,
+ "4365": 2.78626,
+ "4370": 2.73026,
+ "4375": 2.74021,
+ "4380": 2.69919,
+ "4385": 2.6977,
+ "4390": 2.70243,
+ "4395": 2.75288,
+ "4400": 2.66495,
+ "4405": 2.66457,
+ "4410": 2.68277,
+ "4415": 2.70545,
+ "4420": 2.70636,
+ "4425": 2.72116,
+ "4430": 2.68965,
+ "4435": 2.68021,
+ "4440": 2.69172,
+ "4445": 2.67902,
+ "4450": 2.65267,
+ "4455": 2.66361,
+ "4460": 2.68715,
+ "4465": 2.70505,
+ "4470": 2.67175,
+ "4475": 2.68445,
+ "4480": 2.65432,
+ "4485": 2.69855,
+ "4490": 2.6496,
+ "4495": 2.70815,
+ "4500": 2.70161,
+ "4505": 2.69411,
+ "4510": 2.64848,
+ "4515": 2.70089,
+ "4520": 2.66867,
+ "4525": 2.67028,
+ "4530": 2.67045,
+ "4535": 2.67125,
+ "4540": 2.70756,
+ "4545": 2.65655,
+ "4550": 2.69725,
+ "4555": 2.68117,
+ "4560": 2.65534,
+ "4565": 2.63892,
+ "4570": 2.63915,
+ "4575": 2.66537,
+ "4580": 2.68501,
+ "4585": 2.68386,
+ "4590": 2.61629,
+ "4595": 2.66205,
+ "4600": 2.67639,
+ "4605": 2.67565,
+ "4610": 2.65541,
+ "4615": 2.66315,
+ "4620": 2.65863,
+ "4625": 2.70449,
+ "4630": 2.67529,
+ "4635": 2.64482,
+ "4640": 2.69151,
+ "4645": 2.64525,
+ "4650": 2.69485,
+ "4655": 2.7021,
+ "4660": 2.67166,
+ "4665": 2.68866,
+ "4670": 2.67161,
+ "4675": 2.68208,
+ "4680": 2.66232,
+ "4685": 2.65409,
+ "4690": 2.70073,
+ "4695": 2.65354,
+ "4700": 2.67167,
+ "4705": 2.64927,
+ "4710": 2.67483,
+ "4715": 2.64734,
+ "4720": 2.71881,
+ "4725": 2.62661,
+ "4730": 2.6509,
+ "4735": 2.6871,
+ "4740": 2.64143,
+ "4745": 2.65229,
+ "4750": 2.64026,
+ "4755": 2.65299,
+ "4760": 2.66273,
+ "4765": 2.64635,
+ "4770": 2.62577,
+ "4775": 2.65478,
+ "4780": 2.65753,
+ "4785": 2.69313,
+ "4790": 2.65185,
+ "4795": 2.67228,
+ "4800": 2.62783,
+ "4805": 2.64059,
+ "4810": 2.6644,
+ "4815": 2.64691,
+ "4820": 2.67273,
+ "4825": 2.65228,
+ "4830": 2.61459,
+ "4835": 2.64948,
+ "4840": 2.65752,
+ "4845": 2.63255,
+ "4850": 2.62325,
+ "4855": 2.59969,
+ "4860": 2.65178,
+ "4865": 2.62536,
+ "4870": 2.6393,
+ "4875": 2.61885,
+ "4880": 2.62557,
+ "4885": 2.62692,
+ "4890": 2.67877,
+ "4895": 2.65892,
+ "4900": 2.61592,
+ "4905": 2.61904,
+ "4910": 2.63808,
+ "4915": 2.61278,
+ "4920": 2.65283,
+ "4925": 2.64926,
+ "4930": 2.57001,
+ "4935": 2.64979,
+ "4940": 2.63168,
+ "4945": 2.6367,
+ "4950": 2.6251,
+ "4955": 2.61685,
+ "4960": 2.6205,
+ "4965": 2.6585,
+ "4970": 2.59857,
+ "4975": 2.65638,
+ "4980": 2.61828,
+ "4985": 2.63171,
+ "4990": 2.65627,
+ "4995": 2.58066,
+ "5000": 2.65821,
+ "5005": 2.6652,
+ "5010": 2.68657,
+ "5015": 2.63702,
+ "5020": 2.6427,
+ "5025": 2.68656,
+ "5030": 2.64369,
+ "5035": 2.6177,
+ "5040": 2.62106,
+ "5045": 2.60302,
+ "5050": 2.62586,
+ "5055": 2.64898,
+ "5060": 2.64403,
+ "5065": 2.6874,
+ "5070": 2.60434,
+ "5075": 2.61149,
+ "5080": 2.61127,
+ "5085": 2.61237,
+ "5090": 2.59602,
+ "5095": 2.65327,
+ "5100": 2.64698,
+ "5105": 2.60906,
+ "5110": 2.66203,
+ "5115": 2.61929,
+ "5120": 2.67064,
+ "5125": 2.62921,
+ "5130": 2.61274,
+ "5135": 2.61163,
+ "5140": 2.57705,
+ "5145": 2.62816,
+ "5150": 2.63428,
+ "5155": 2.61691,
+ "5160": 2.66153,
+ "5165": 2.58419,
+ "5170": 2.59059,
+ "5175": 2.61619,
+ "5180": 2.60661,
+ "5185": 2.62149,
+ "5190": 2.62346,
+ "5195": 2.66821,
+ "5200": 2.60109,
+ "5205": 2.60537,
+ "5210": 2.6045,
+ "5215": 2.64533,
+ "5220": 2.58657,
+ "5225": 2.55085,
+ "5230": 2.63463,
+ "5235": 2.61656,
+ "5240": 2.62091,
+ "5245": 2.62955,
+ "5250": 2.59194,
+ "5255": 2.61576,
+ "5260": 2.55526,
+ "5265": 2.5946,
+ "5270": 2.58902,
+ "5275": 2.61851,
+ "5280": 2.61104,
+ "5285": 2.6026,
+ "5290": 2.63419,
+ "5295": 2.62205,
+ "5300": 2.57655,
+ "5305": 2.59742,
+ "5310": 2.61293,
+ "5315": 2.59071,
+ "5320": 2.61645,
+ "5325": 2.64426,
+ "5330": 2.60211,
+ "5335": 2.58442,
+ "5340": 2.56099,
+ "5345": 2.65685,
+ "5350": 2.61867,
+ "5355": 2.57667,
+ "5360": 2.59317,
+ "5365": 2.62154,
+ "5370": 2.6143,
+ "5375": 2.62936,
+ "5380": 2.57983,
+ "5385": 2.56433,
+ "5390": 2.58453,
+ "5395": 2.61878,
+ "5400": 2.60528,
+ "5405": 2.54318,
+ "5410": 2.61244,
+ "5415": 2.59663,
+ "5420": 2.61317,
+ "5425": 2.62412,
+ "5430": 2.62829,
+ "5435": 2.57578,
+ "5440": 2.5861,
+ "5445": 2.62832,
+ "5450": 2.65323,
+ "5455": 2.61253,
+ "5460": 2.59283,
+ "5465": 2.60497,
+ "5470": 2.5971,
+ "5475": 2.62515,
+ "5480": 2.58707,
+ "5485": 2.58882,
+ "5490": 2.57638,
+ "5495": 2.56911,
+ "5500": 2.56798,
+ "5505": 2.6147,
+ "5510": 2.62508,
+ "5515": 2.58073,
+ "5520": 2.55375,
+ "5525": 2.58367,
+ "5530": 2.66448,
+ "5535": 2.62011,
+ "5540": 2.56912,
+ "5545": 2.59599,
+ "5550": 2.54873,
+ "5555": 2.57203,
+ "5560": 2.56243,
+ "5565": 2.60838,
+ "5570": 2.65345,
+ "5575": 2.62922,
+ "5580": 2.57343,
+ "5585": 2.5961,
+ "5590": 2.56073,
+ "5595": 2.5818,
+ "5600": 2.55146,
+ "5605": 2.59879,
+ "5610": 2.58312,
+ "5615": 2.58429,
+ "5620": 2.58081,
+ "5625": 2.54856,
+ "5630": 2.56991,
+ "5635": 2.63041,
+ "5640": 2.5916,
+ "5645": 2.56846,
+ "5650": 2.57465,
+ "5655": 2.54631,
+ "5660": 2.55573,
+ "5665": 2.58316,
+ "5670": 2.56552,
+ "5675": 2.6051,
+ "5680": 2.52587,
+ "5685": 2.56617,
+ "5690": 2.60027,
+ "5695": 2.5569,
+ "5700": 2.59658,
+ "5705": 2.59486,
+ "5710": 2.57702,
+ "5715": 2.58277,
+ "5720": 2.53627,
+ "5725": 2.60322,
+ "5730": 2.57254,
+ "5735": 2.6087,
+ "5740": 2.59522,
+ "5745": 2.55931,
+ "5750": 2.53991,
+ "5755": 2.55759,
+ "5760": 2.62474,
+ "5765": 2.55861,
+ "5770": 2.54026,
+ "5775": 2.58406,
+ "5780": 2.57586,
+ "5785": 2.5382,
+ "5790": 2.56312,
+ "5795": 2.59953,
+ "5800": 2.54244,
+ "5805": 2.53338,
+ "5810": 2.55644,
+ "5815": 2.52433,
+ "5820": 2.59823,
+ "5825": 2.50593,
+ "5830": 2.49732,
+ "5835": 2.59651,
+ "5840": 2.53896,
+ "5845": 2.55347,
+ "5850": 2.61179,
+ "5855": 2.51002,
+ "5860": 2.55898,
+ "5865": 2.51794,
+ "5870": 2.57307,
+ "5875": 2.60738,
+ "5880": 2.58521,
+ "5885": 2.5662,
+ "5890": 2.58404,
+ "5895": 2.55371,
+ "5900": 2.61276,
+ "5905": 2.55604,
+ "5910": 2.59526,
+ "5915": 2.61001,
+ "5920": 2.58723,
+ "5925": 2.53738,
+ "5930": 2.57762,
+ "5935": 2.55201,
+ "5940": 2.57061,
+ "5945": 2.51742,
+ "5950": 2.55385,
+ "5955": 2.59921,
+ "5960": 2.56774,
+ "5965": 2.61891,
+ "5970": 2.55084,
+ "5975": 2.58223,
+ "5980": 2.55929,
+ "5985": 2.56146,
+ "5990": 2.55428,
+ "5995": 2.55604,
+ "6000": 2.55347,
+ "6005": 2.51918,
+ "6010": 2.55968,
+ "6015": 2.52192,
+ "6020": 2.53377,
+ "6025": 2.5569,
+ "6030": 2.60368,
+ "6035": 2.54135,
+ "6040": 2.54912,
+ "6045": 2.49048,
+ "6050": 2.59428,
+ "6055": 2.51886,
+ "6060": 2.54379,
+ "6065": 2.52482,
+ "6070": 2.52825,
+ "6075": 2.53458,
+ "6080": 2.53462,
+ "6085": 2.59606,
+ "6090": 2.56861,
+ "6095": 2.53271,
+ "6100": 2.54134,
+ "6105": 2.52059,
+ "6110": 2.55382,
+ "6115": 2.58369,
+ "6120": 2.55719,
+ "6125": 2.53864,
+ "6130": 2.47506,
+ "6135": 2.55486,
+ "6140": 2.55502,
+ "6145": 2.55645,
+ "6150": 2.52511,
+ "6155": 2.50788,
+ "6160": 2.53773,
+ "6165": 2.57035,
+ "6170": 2.54359,
+ "6175": 2.59872,
+ "6180": 2.50939,
+ "6185": 2.54966,
+ "6190": 2.49143,
+ "6195": 2.57856,
+ "6200": 2.55195,
+ "6205": 2.53696,
+ "6210": 2.51913,
+ "6215": 2.51433,
+ "6220": 2.56607,
+ "6225": 2.51286,
+ "6230": 2.51108,
+ "6235": 2.56061,
+ "6240": 2.54928,
+ "6245": 2.52483,
+ "6250": 2.52912,
+ "6255": 2.57251,
+ "6260": 2.52083,
+ "6265": 2.57221,
+ "6270": 2.52297,
+ "6275": 2.56269,
+ "6280": 2.52402,
+ "6285": 2.52172,
+ "6290": 2.52273,
+ "6295": 2.50766,
+ "6300": 2.55521,
+ "6305": 2.52333,
+ "6310": 2.51033,
+ "6315": 2.53641,
+ "6320": 2.48793,
+ "6325": 2.59556,
+ "6330": 2.55493,
+ "6335": 2.51056,
+ "6340": 2.51365,
+ "6345": 2.554,
+ "6350": 2.55544,
+ "6355": 2.52304,
+ "6360": 2.52028,
+ "6365": 2.48473,
+ "6370": 2.53351,
+ "6375": 2.49303,
+ "6380": 2.55576,
+ "6385": 2.57478,
+ "6390": 2.50485,
+ "6395": 2.54909,
+ "6400": 2.50599,
+ "6405": 2.52514,
+ "6410": 2.51147,
+ "6415": 2.52125,
+ "6420": 2.53958,
+ "6425": 2.53282,
+ "6430": 2.57602,
+ "6435": 2.54302,
+ "6440": 2.53463,
+ "6445": 2.52617,
+ "6450": 2.53113,
+ "6455": 2.52147,
+ "6460": 2.51423,
+ "6465": 2.55926,
+ "6470": 2.51637,
+ "6475": 2.52603,
+ "6480": 2.4882,
+ "6485": 2.52882,
+ "6490": 2.50677,
+ "6495": 2.49902,
+ "6500": 2.52356,
+ "6505": 2.49389,
+ "6510": 2.5409,
+ "6515": 2.50842,
+ "6520": 2.50856,
+ "6525": 2.49197,
+ "6530": 2.54065,
+ "6535": 2.53032,
+ "6540": 2.52904,
+ "6545": 2.55998,
+ "6550": 2.50044,
+ "6555": 2.55575,
+ "6560": 2.50919,
+ "6565": 2.51983,
+ "6570": 2.58266,
+ "6575": 2.5221,
+ "6580": 2.4976,
+ "6585": 2.50481,
+ "6590": 2.50722,
+ "6595": 2.49751,
+ "6600": 2.49428,
+ "6605": 2.53842,
+ "6610": 2.47637,
+ "6615": 2.56638,
+ "6620": 2.53233,
+ "6625": 2.51062,
+ "6630": 2.51116,
+ "6635": 2.47299,
+ "6640": 2.53891,
+ "6645": 2.5957,
+ "6650": 2.50945,
+ "6655": 2.49555,
+ "6660": 2.57228,
+ "6665": 2.5186,
+ "6670": 2.56576,
+ "6675": 2.46509,
+ "6680": 2.54752,
+ "6685": 2.53432,
+ "6690": 2.51231,
+ "6695": 2.48519,
+ "6700": 2.52191,
+ "6705": 2.51661,
+ "6710": 2.49005,
+ "6715": 2.51625,
+ "6720": 2.50795,
+ "6725": 2.51834,
+ "6730": 2.51954,
+ "6735": 2.48309,
+ "6740": 2.51335,
+ "6745": 2.49433,
+ "6750": 2.55697,
+ "6755": 2.47385,
+ "6760": 2.54193,
+ "6765": 2.48826,
+ "6770": 2.51722,
+ "6775": 2.50587,
+ "6780": 2.53774,
+ "6785": 2.47123,
+ "6790": 2.54526,
+ "6795": 2.49809,
+ "6800": 2.52509,
+ "6805": 2.51004,
+ "6810": 2.5022,
+ "6815": 2.52175,
+ "6820": 2.48494,
+ "6825": 2.5025,
+ "6830": 2.53822,
+ "6835": 2.5061,
+ "6840": 2.50895,
+ "6845": 2.5234,
+ "6850": 2.47328,
+ "6855": 2.51141,
+ "6860": 2.50139,
+ "6865": 2.48788,
+ "6870": 2.55255,
+ "6875": 2.47292,
+ "6880": 2.54863,
+ "6885": 2.47591,
+ "6890": 2.54517,
+ "6895": 2.50126,
+ "6900": 2.48503,
+ "6905": 2.49614,
+ "6910": 2.51701,
+ "6915": 2.51638,
+ "6920": 2.53141,
+ "6925": 2.53752,
+ "6930": 2.48681,
+ "6935": 2.51676,
+ "6940": 2.49758,
+ "6945": 2.45779,
+ "6950": 2.48149,
+ "6955": 2.52444,
+ "6960": 2.51846,
+ "6965": 2.49132,
+ "6970": 2.47056,
+ "6975": 2.52071,
+ "6980": 2.45195,
+ "6985": 2.51437,
+ "6990": 2.52894,
+ "6995": 2.46158,
+ "7000": 2.48597,
+ "7005": 2.4686,
+ "7010": 2.47377,
+ "7015": 2.52019,
+ "7020": 2.46598,
+ "7025": 2.45097,
+ "7030": 2.48257,
+ "7035": 2.47716,
+ "7040": 2.50435,
+ "7045": 2.51849,
+ "7050": 2.52366,
+ "7055": 2.43866,
+ "7060": 2.47405,
+ "7065": 2.4831,
+ "7070": 2.48979,
+ "7075": 2.49284,
+ "7080": 2.53297,
+ "7085": 2.48384,
+ "7090": 2.47443,
+ "7095": 2.50012,
+ "7100": 2.5144,
+ "7105": 2.48598,
+ "7110": 2.48519,
+ "7115": 2.50298,
+ "7120": 2.46732,
+ "7125": 2.45979,
+ "7130": 2.48348,
+ "7135": 2.51033,
+ "7140": 2.49838,
+ "7145": 2.49599,
+ "7150": 2.50775,
+ "7155": 2.50185,
+ "7160": 2.472,
+ "7165": 2.45522,
+ "7170": 2.50356,
+ "7175": 2.50054,
+ "7180": 2.50191,
+ "7185": 2.48,
+ "7190": 2.45987,
+ "7195": 2.46375,
+ "7200": 2.50646,
+ "7205": 2.48744,
+ "7210": 2.44239,
+ "7215": 2.47767,
+ "7220": 2.44151,
+ "7225": 2.51106,
+ "7230": 2.50582,
+ "7235": 2.48172,
+ "7240": 2.4782,
+ "7245": 2.49936,
+ "7250": 2.50678,
+ "7255": 2.4912,
+ "7260": 2.45802,
+ "7265": 2.45003,
+ "7270": 2.46693,
+ "7275": 2.49683,
+ "7280": 2.49101,
+ "7285": 2.42033,
+ "7290": 2.47866,
+ "7295": 2.48483,
+ "7300": 2.41501,
+ "7305": 2.44141,
+ "7310": 2.44461,
+ "7315": 2.48723,
+ "7320": 2.48097,
+ "7325": 2.45667,
+ "7330": 2.48916,
+ "7335": 2.47148,
+ "7340": 2.4637,
+ "7345": 2.49267,
+ "7350": 2.51067,
+ "7355": 2.49399,
+ "7360": 2.47747,
+ "7365": 2.46623,
+ "7370": 2.46976,
+ "7375": 2.44817,
+ "7380": 2.49099,
+ "7385": 2.48213,
+ "7390": 2.46996,
+ "7395": 2.46937,
+ "7400": 2.47759,
+ "7405": 2.43735,
+ "7410": 2.479,
+ "7415": 2.46864,
+ "7420": 2.49062,
+ "7425": 2.45385,
+ "7430": 2.52079,
+ "7435": 2.48865,
+ "7440": 2.51835,
+ "7445": 2.50675,
+ "7450": 2.4717,
+ "7455": 2.45394,
+ "7460": 2.46163,
+ "7465": 2.47321,
+ "7470": 2.44826,
+ "7475": 2.45502,
+ "7480": 2.50736,
+ "7485": 2.44851,
+ "7490": 2.47418,
+ "7495": 2.47985,
+ "7500": 2.49378,
+ "7505": 2.43918,
+ "7510": 2.43573,
+ "7515": 2.42055,
+ "7520": 2.49283,
+ "7525": 2.49607,
+ "7530": 2.47395,
+ "7535": 2.45882,
+ "7540": 2.47236,
+ "7545": 2.47257,
+ "7550": 2.48854,
+ "7555": 2.45427,
+ "7560": 2.42558,
+ "7565": 2.50802,
+ "7570": 2.48336,
+ "7575": 2.43657,
+ "7580": 2.45585,
+ "7585": 2.4796,
+ "7590": 2.47883,
+ "7595": 2.46048,
+ "7600": 2.4605,
+ "7605": 2.44515,
+ "7610": 2.44772,
+ "7615": 2.42408,
+ "7620": 2.54293,
+ "7625": 2.47978,
+ "7630": 2.42405,
+ "7635": 2.42523,
+ "7640": 2.45197,
+ "7645": 2.47074,
+ "7650": 2.46103,
+ "7655": 2.48292,
+ "7660": 2.45069,
+ "7665": 2.43132,
+ "7670": 2.43977,
+ "7675": 2.45451,
+ "7680": 2.48428,
+ "7685": 2.43105,
+ "7690": 2.47923,
+ "7695": 2.45298,
+ "7700": 2.49224,
+ "7705": 2.5344,
+ "7710": 2.49781,
+ "7715": 2.44151,
+ "7720": 2.46783,
+ "7725": 2.47891,
+ "7730": 2.45705,
+ "7735": 2.47177,
+ "7740": 2.43645,
+ "7745": 2.44594,
+ "7750": 2.43539,
+ "7755": 2.46419,
+ "7760": 2.44791,
+ "7765": 2.4518,
+ "7770": 2.4666,
+ "7775": 2.44972,
+ "7780": 2.41283,
+ "7785": 2.44234,
+ "7790": 2.48019,
+ "7795": 2.43806,
+ "7800": 2.46172,
+ "7805": 2.47968,
+ "7810": 2.50095,
+ "7815": 2.4866,
+ "7820": 2.44473,
+ "7825": 2.51138,
+ "7830": 2.45119,
+ "7835": 2.46518,
+ "7840": 2.47743,
+ "7845": 2.45952,
+ "7850": 2.41496,
+ "7855": 2.47017,
+ "7860": 2.49724,
+ "7865": 2.42238,
+ "7870": 2.46624,
+ "7875": 2.4439,
+ "7880": 2.45106,
+ "7885": 2.46043,
+ "7890": 2.47034,
+ "7895": 2.44531,
+ "7900": 2.43635,
+ "7905": 2.43389,
+ "7910": 2.42388,
+ "7915": 2.47992,
+ "7920": 2.47327,
+ "7925": 2.41978,
+ "7930": 2.47016,
+ "7935": 2.45012,
+ "7940": 2.41955,
+ "7945": 2.46827,
+ "7950": 2.4431,
+ "7955": 2.41866,
+ "7960": 2.48694,
+ "7965": 2.51733,
+ "7970": 2.52064,
+ "7975": 2.44744,
+ "7980": 2.44021,
+ "7985": 2.46499,
+ "7990": 2.42982,
+ "7995": 2.4678,
+ "8000": 2.43369,
+ "8005": 2.41551,
+ "8010": 2.4566,
+ "8015": 2.46743,
+ "8020": 2.4794,
+ "8025": 2.47237,
+ "8030": 2.44969,
+ "8035": 2.46877,
+ "8040": 2.42022,
+ "8045": 2.45138,
+ "8050": 2.44722,
+ "8055": 2.42479,
+ "8060": 2.44355,
+ "8065": 2.463,
+ "8070": 2.45573,
+ "8075": 2.45766,
+ "8080": 2.44452,
+ "8085": 2.43918,
+ "8090": 2.42947,
+ "8095": 2.42336,
+ "8100": 2.43809,
+ "8105": 2.4929,
+ "8110": 2.43687,
+ "8115": 2.44253,
+ "8120": 2.46616,
+ "8125": 2.46474,
+ "8130": 2.45131,
+ "8135": 2.45162,
+ "8140": 2.43804,
+ "8145": 2.42476,
+ "8150": 2.4201,
+ "8155": 2.48593,
+ "8160": 2.45442,
+ "8165": 2.44161,
+ "8170": 2.43359,
+ "8175": 2.42032,
+ "8180": 2.49387,
+ "8185": 2.42374,
+ "8190": 2.46733,
+ "8195": 2.45639,
+ "8200": 2.44552,
+ "8205": 2.44393,
+ "8210": 2.42974,
+ "8215": 2.43789,
+ "8220": 2.43568,
+ "8225": 2.40813,
+ "8230": 2.44002,
+ "8235": 2.46402,
+ "8240": 2.42523,
+ "8245": 2.44686,
+ "8250": 2.44348,
+ "8255": 2.43916,
+ "8260": 2.43307,
+ "8265": 2.42573,
+ "8270": 2.43174,
+ "8275": 2.44039,
+ "8280": 2.3963,
+ "8285": 2.43685,
+ "8290": 2.47852,
+ "8295": 2.44671,
+ "8300": 2.45587,
+ "8305": 2.40638,
+ "8310": 2.43379,
+ "8315": 2.45525,
+ "8320": 2.39709,
+ "8325": 2.39084,
+ "8330": 2.43277,
+ "8335": 2.44245,
+ "8340": 2.4878,
+ "8345": 2.44494,
+ "8350": 2.44735,
+ "8355": 2.40568,
+ "8360": 2.39807,
+ "8365": 2.45388,
+ "8370": 2.4495,
+ "8375": 2.42225,
+ "8380": 2.41565,
+ "8385": 2.42229,
+ "8390": 2.43699,
+ "8395": 2.43929,
+ "8400": 2.43589,
+ "8405": 2.4875,
+ "8410": 2.43766,
+ "8415": 2.43224,
+ "8420": 2.41348,
+ "8425": 2.43603,
+ "8430": 2.45874,
+ "8435": 2.40392,
+ "8440": 2.44872,
+ "8445": 2.45716,
+ "8450": 2.40546,
+ "8455": 2.45741,
+ "8460": 2.4534,
+ "8465": 2.43498,
+ "8470": 2.40732,
+ "8475": 2.47375,
+ "8480": 2.40045,
+ "8485": 2.41586,
+ "8490": 2.4639,
+ "8495": 2.43561,
+ "8500": 2.44316,
+ "8505": 2.40386,
+ "8510": 2.40086,
+ "8515": 2.42745,
+ "8520": 2.42214,
+ "8525": 2.4898,
+ "8530": 2.37128,
+ "8535": 2.39928,
+ "8540": 2.48121,
+ "8545": 2.37881,
+ "8550": 2.43739,
+ "8555": 2.44828,
+ "8560": 2.46927,
+ "8565": 2.41924,
+ "8570": 2.42864,
+ "8575": 2.44669,
+ "8580": 2.43829,
+ "8585": 2.41804,
+ "8590": 2.40217,
+ "8595": 2.42543,
+ "8600": 2.40923,
+ "8605": 2.48923,
+ "8610": 2.4189,
+ "8615": 2.38627,
+ "8620": 2.44771,
+ "8625": 2.42556,
+ "8630": 2.45885,
+ "8635": 2.45722,
+ "8640": 2.43592,
+ "8645": 2.47348,
+ "8650": 2.42058,
+ "8655": 2.45362,
+ "8660": 2.45518,
+ "8665": 2.38364,
+ "8670": 2.40841,
+ "8675": 2.42831,
+ "8680": 2.44533,
+ "8685": 2.42795,
+ "8690": 2.40821,
+ "8695": 2.44087,
+ "8700": 2.43118,
+ "8705": 2.41801,
+ "8710": 2.42606,
+ "8715": 2.44636,
+ "8720": 2.47348,
+ "8725": 2.40723,
+ "8730": 2.38832,
+ "8735": 2.43292,
+ "8740": 2.42791,
+ "8745": 2.39523,
+ "8750": 2.43296,
+ "8755": 2.4228,
+ "8760": 2.39965,
+ "8765": 2.43238,
+ "8770": 2.40279,
+ "8775": 2.43581,
+ "8780": 2.4169,
+ "8785": 2.47064,
+ "8790": 2.41812,
+ "8795": 2.41771,
+ "8800": 2.4148,
+ "8805": 2.40261,
+ "8810": 2.40855,
+ "8815": 2.47233,
+ "8820": 2.45221,
+ "8825": 2.42327,
+ "8830": 2.38596,
+ "8835": 2.42007,
+ "8840": 2.3912,
+ "8845": 2.42402,
+ "8850": 2.43315,
+ "8855": 2.40161,
+ "8860": 2.42753,
+ "8865": 2.42451,
+ "8870": 2.43294,
+ "8875": 2.43697,
+ "8880": 2.41075,
+ "8885": 2.39174,
+ "8890": 2.44505,
+ "8895": 2.42678,
+ "8900": 2.41124,
+ "8905": 2.40029,
+ "8910": 2.39777,
+ "8915": 2.41598,
+ "8920": 2.43089,
+ "8925": 2.46425,
+ "8930": 2.41457,
+ "8935": 2.40635,
+ "8940": 2.38763,
+ "8945": 2.39267,
+ "8950": 2.41516,
+ "8955": 2.39226,
+ "8960": 2.43161,
+ "8965": 2.41676,
+ "8970": 2.40264,
+ "8975": 2.47364,
+ "8980": 2.43865,
+ "8985": 2.37275,
+ "8990": 2.40714,
+ "8995": 2.41421,
+ "9000": 2.45425,
+ "9005": 2.41041,
+ "9010": 2.37331,
+ "9015": 2.40629,
+ "9020": 2.39684,
+ "9025": 2.36793,
+ "9030": 2.39775,
+ "9035": 2.4222,
+ "9040": 2.42089,
+ "9045": 2.4185,
+ "9050": 2.39505,
+ "9055": 2.41883,
+ "9060": 2.41777,
+ "9065": 2.40319,
+ "9070": 2.44397,
+ "9075": 2.39439,
+ "9080": 2.43305,
+ "9085": 2.41006,
+ "9090": 2.40938,
+ "9095": 2.3941,
+ "9100": 2.39969,
+ "9105": 2.35719,
+ "9110": 2.46849,
+ "9115": 2.4178,
+ "9120": 2.4039,
+ "9125": 2.45748,
+ "9130": 2.39278,
+ "9135": 2.44705,
+ "9140": 2.43327,
+ "9145": 2.42517,
+ "9150": 2.42294,
+ "9155": 2.37266,
+ "9160": 2.41355,
+ "9165": 2.42349,
+ "9170": 2.37092,
+ "9175": 2.41554,
+ "9180": 2.3779,
+ "9185": 2.438,
+ "9190": 2.41083,
+ "9195": 2.39616,
+ "9200": 2.40527,
+ "9205": 2.45505,
+ "9210": 2.36436,
+ "9215": 2.46318,
+ "9220": 2.44705,
+ "9225": 2.38094,
+ "9230": 2.44355,
+ "9235": 2.39394,
+ "9240": 2.39928,
+ "9245": 2.43306,
+ "9250": 2.43135,
+ "9255": 2.42746,
+ "9260": 2.38386,
+ "9265": 2.43528,
+ "9270": 2.43273,
+ "9275": 2.39047,
+ "9280": 2.38694,
+ "9285": 2.419,
+ "9290": 2.40065,
+ "9295": 2.38095,
+ "9300": 2.42048,
+ "9305": 2.40199,
+ "9310": 2.41314,
+ "9315": 2.40673,
+ "9320": 2.44261,
+ "9325": 2.36848,
+ "9330": 2.40099,
+ "9335": 2.35836,
+ "9340": 2.40562,
+ "9345": 2.41339,
+ "9350": 2.43711,
+ "9355": 2.54976,
+ "9360": 2.46791,
+ "9365": 2.40083,
+ "9370": 2.44296,
+ "9375": 2.43843,
+ "9380": 2.35492,
+ "9385": 2.40106,
+ "9390": 2.37923,
+ "9395": 2.38537,
+ "9400": 2.44192,
+ "9405": 2.41134,
+ "9410": 2.39282,
+ "9415": 2.43379,
+ "9420": 2.44336,
+ "9425": 2.42952,
+ "9430": 2.44459,
+ "9435": 2.41314,
+ "9440": 2.47521,
+ "9445": 2.37319,
+ "9450": 2.39425,
+ "9455": 2.40156,
+ "9460": 2.3856,
+ "9465": 2.37862,
+ "9470": 2.38082,
+ "9475": 2.36462,
+ "9480": 2.43197,
+ "9485": 2.38766,
+ "9490": 2.42039,
+ "9495": 2.38324,
+ "9500": 2.36513,
+ "9505": 2.43009,
+ "9510": 2.39857,
+ "9515": 2.42968,
+ "9520": 2.41785,
+ "9525": 2.38847,
+ "9530": 2.45376,
+ "9535": 2.40056,
+ "9540": 2.41611,
+ "9545": 2.37735,
+ "9550": 2.42123,
+ "9555": 2.38863,
+ "9560": 2.42176,
+ "9565": 2.4065,
+ "9570": 2.37249,
+ "9575": 2.41052,
+ "9580": 2.3951,
+ "9585": 2.42188,
+ "9590": 2.42821,
+ "9595": 2.44726,
+ "9600": 2.38934,
+ "9605": 2.38358,
+ "9610": 2.42106,
+ "9615": 2.4142,
+ "9620": 2.41302,
+ "9625": 2.44609,
+ "9630": 2.3957,
+ "9635": 2.40141,
+ "9640": 2.44439,
+ "9645": 2.41011,
+ "9650": 2.39715,
+ "9655": 2.3728,
+ "9660": 2.42407,
+ "9665": 2.38942,
+ "9670": 2.38106,
+ "9675": 2.35623,
+ "9680": 2.39556,
+ "9685": 2.39468,
+ "9690": 2.46311,
+ "9695": 2.3803,
+ "9700": 2.37568,
+ "9705": 2.38264,
+ "9710": 2.36519,
+ "9715": 2.38727,
+ "9720": 2.43505,
+ "9725": 2.44188,
+ "9730": 2.42828,
+ "9735": 2.38742,
+ "9740": 2.38014,
+ "9745": 2.42614,
+ "9750": 2.39713,
+ "9755": 2.40574,
+ "9760": 2.41046,
+ "9765": 2.36693,
+ "9770": 2.46311,
+ "9775": 2.40027,
+ "9780": 2.3623,
+ "9785": 2.40073,
+ "9790": 2.40637,
+ "9795": 2.35988,
+ "9800": 2.39429,
+ "9805": 2.40453,
+ "9810": 2.40856,
+ "9815": 2.37781,
+ "9820": 2.37606,
+ "9825": 2.40326,
+ "9830": 2.42047,
+ "9835": 2.38296,
+ "9840": 2.41561,
+ "9845": 2.36392,
+ "9850": 2.39833,
+ "9855": 2.39588,
+ "9860": 2.39015,
+ "9865": 2.37872,
+ "9870": 2.38479,
+ "9875": 2.37975,
+ "9880": 2.45111,
+ "9885": 2.39299,
+ "9890": 2.3509,
+ "9895": 2.31999,
+ "9900": 2.39398,
+ "9905": 2.42373,
+ "9910": 2.35469,
+ "9915": 2.36188,
+ "9920": 2.41069,
+ "9925": 2.3962,
+ "9930": 2.37914,
+ "9935": 2.34871,
+ "9940": 2.38316,
+ "9945": 2.37785,
+ "9950": 2.40242,
+ "9955": 2.44647,
+ "9960": 2.431,
+ "9965": 2.35688,
+ "9970": 2.40859,
+ "9975": 2.3835,
+ "9980": 2.33008,
+ "9985": 2.40516,
+ "9990": 2.39642,
+ "9995": 2.39479,
+ "10000": 2.36666,
+ "10005": 2.37185,
+ "10010": 2.38179,
+ "10015": 2.44374,
+ "10020": 2.36251,
+ "10025": 2.38767,
+ "10030": 2.38659,
+ "10035": 2.40904,
+ "10040": 2.40154,
+ "10045": 2.38171,
+ "10050": 2.35085,
+ "10055": 2.36819,
+ "10060": 2.41802,
+ "10065": 2.3733,
+ "10070": 2.42068,
+ "10075": 2.37133,
+ "10080": 2.36112,
+ "10085": 2.36913,
+ "10090": 2.34609,
+ "10095": 2.4004,
+ "10100": 2.31173,
+ "10105": 2.37923,
+ "10110": 2.40867,
+ "10115": 2.38512,
+ "10120": 2.35537,
+ "10125": 2.36885,
+ "10130": 2.35772,
+ "10135": 2.38091,
+ "10140": 2.41174,
+ "10145": 2.40531,
+ "10150": 2.37466,
+ "10155": 2.39341,
+ "10160": 2.36024,
+ "10165": 2.38118,
+ "10170": 2.42175,
+ "10175": 2.32278,
+ "10180": 2.39348,
+ "10185": 2.38112,
+ "10190": 2.44208,
+ "10195": 2.40114,
+ "10200": 2.38989
+ }
+ },
+ "num-zeros": {
+ "start_step": 1,
+ "end_step": 10200,
+ "step_interval": 5,
+ "values": {
+ "1": 956237952.0,
+ "5": 967338112.0,
+ "10": 971388416.0,
+ "15": 946492416.0,
+ "20": 957330304.0,
+ "25": 1064217280.0,
+ "30": 1177614208.0,
+ "35": 1231743488.0,
+ "40": 1192773248.0,
+ "45": 1126056192.0,
+ "50": 1114252800.0,
+ "55": 1079868160.0,
+ "60": 1027359552.0,
+ "65": 1009657856.0,
+ "70": 993522560.0,
+ "75": 986463616.0,
+ "80": 1003645696.0,
+ "85": 998620032.0,
+ "90": 980123200.0,
+ "95": 955802304.0,
+ "100": 970032128.0,
+ "105": 979558464.0,
+ "110": 977155840.0,
+ "115": 976995648.0,
+ "120": 960840640.0,
+ "125": 943207360.0,
+ "130": 976265792.0,
+ "135": 965483008.0,
+ "140": 963541184.0,
+ "145": 976785856.0,
+ "150": 921376256.0,
+ "155": 968230080.0,
+ "160": 956410112.0,
+ "165": 959776064.0,
+ "170": 974353984.0,
+ "175": 949006720.0,
+ "180": 946693120.0,
+ "185": 972018368.0,
+ "190": 969083072.0,
+ "195": 985127296.0,
+ "200": 945772672.0,
+ "205": 958353408.0,
+ "210": 979445952.0,
+ "215": 967486976.0,
+ "220": 956421696.0,
+ "225": 962398720.0,
+ "230": 948180224.0,
+ "235": 965227392.0,
+ "240": 966053440.0,
+ "245": 969162176.0,
+ "250": 974440448.0,
+ "255": 925063104.0,
+ "260": 965635712.0,
+ "265": 970669504.0,
+ "270": 959133056.0,
+ "275": 953996096.0,
+ "280": 963428992.0,
+ "285": 945783360.0,
+ "290": 974114048.0,
+ "295": 966696704.0,
+ "300": 967154368.0,
+ "305": 964506688.0,
+ "310": 940354688.0,
+ "315": 967400640.0,
+ "320": 969003968.0,
+ "325": 980556544.0,
+ "330": 972097024.0,
+ "335": 946861312.0,
+ "340": 966592256.0,
+ "345": 973024128.0,
+ "350": 973919360.0,
+ "355": 963257216.0,
+ "360": 948355072.0,
+ "365": 964821888.0,
+ "370": 962956160.0,
+ "375": 958448192.0,
+ "380": 947154304.0,
+ "385": 955990528.0,
+ "390": 945401728.0,
+ "395": 970420864.0,
+ "400": 979777280.0,
+ "405": 968349440.0,
+ "410": 970066624.0,
+ "415": 953156736.0,
+ "420": 943564288.0,
+ "425": 954775680.0,
+ "430": 962658560.0,
+ "435": 977076480.0,
+ "440": 954808960.0,
+ "445": 971887936.0,
+ "450": 963509120.0,
+ "455": 973133504.0,
+ "460": 983711680.0,
+ "465": 945280256.0,
+ "470": 942053120.0,
+ "475": 967006336.0,
+ "480": 966106368.0,
+ "485": 976411264.0,
+ "490": 962540736.0,
+ "495": 945465216.0,
+ "500": 964456640.0,
+ "505": 986011904.0,
+ "510": 965684480.0,
+ "515": 943408256.0,
+ "520": 945018624.0,
+ "525": 971265920.0,
+ "530": 971885824.0,
+ "535": 979139712.0,
+ "540": 969531776.0,
+ "545": 954128384.0,
+ "550": 951264512.0,
+ "555": 987223680.0,
+ "560": 960431488.0,
+ "565": 966615296.0,
+ "570": 975727872.0,
+ "575": 927221504.0,
+ "580": 970695168.0,
+ "585": 961176704.0,
+ "590": 972965824.0,
+ "595": 963685696.0,
+ "600": 937080896.0,
+ "605": 951472448.0,
+ "610": 963365312.0,
+ "615": 970013120.0,
+ "620": 976467456.0,
+ "625": 949582144.0,
+ "630": 954443392.0,
+ "635": 986044928.0,
+ "640": 980976256.0,
+ "645": 955010944.0,
+ "650": 958547072.0,
+ "655": 951653568.0,
+ "660": 961041088.0,
+ "665": 967554752.0,
+ "670": 962515968.0,
+ "675": 968332416.0,
+ "680": 965617152.0,
+ "685": 962867648.0,
+ "690": 961918848.0,
+ "695": 954766656.0,
+ "700": 970338304.0,
+ "705": 945516672.0,
+ "710": 943885568.0,
+ "715": 973354368.0,
+ "720": 968367552.0,
+ "725": 978491392.0,
+ "730": 952193920.0,
+ "735": 948812288.0,
+ "740": 955630848.0,
+ "745": 975866880.0,
+ "750": 981238016.0,
+ "755": 962152064.0,
+ "760": 951964928.0,
+ "765": 967348608.0,
+ "770": 976151168.0,
+ "775": 970545920.0,
+ "780": 977537536.0,
+ "785": 931530240.0,
+ "790": 960442880.0,
+ "795": 964583104.0,
+ "800": 967022336.0,
+ "805": 962319104.0,
+ "810": 940973184.0,
+ "815": 949035712.0,
+ "820": 953181248.0,
+ "825": 954500288.0,
+ "830": 976441984.0,
+ "835": 956073600.0,
+ "840": 948406720.0,
+ "845": 965158016.0,
+ "850": 966029056.0,
+ "855": 960904896.0,
+ "860": 976026752.0,
+ "865": 938160640.0,
+ "870": 966414464.0,
+ "875": 972316416.0,
+ "880": 963124608.0,
+ "885": 967743488.0,
+ "890": 949968896.0,
+ "895": 960018432.0,
+ "900": 974232704.0,
+ "905": 963969664.0,
+ "910": 958437888.0,
+ "915": 956355456.0,
+ "920": 943971968.0,
+ "925": 960835456.0,
+ "930": 978848640.0,
+ "935": 971072640.0,
+ "940": 960906880.0,
+ "945": 945063680.0,
+ "950": 957426432.0,
+ "955": 979036352.0,
+ "960": 983589184.0,
+ "965": 966167808.0,
+ "970": 951230848.0,
+ "975": 961577024.0,
+ "980": 968069504.0,
+ "985": 968991744.0,
+ "990": 984392448.0,
+ "995": 953289600.0,
+ "1000": 934775808.0,
+ "1005": 960145920.0,
+ "1010": 971539584.0,
+ "1015": 985186304.0,
+ "1020": 962782080.0,
+ "1025": 935012032.0,
+ "1030": 974681856.0,
+ "1035": 964992512.0,
+ "1040": 980466560.0,
+ "1045": 960827392.0,
+ "1050": 955202112.0,
+ "1055": 957778496.0,
+ "1060": 967746560.0,
+ "1065": 967116288.0,
+ "1070": 966603008.0,
+ "1075": 950059968.0,
+ "1080": 954511488.0,
+ "1085": 967252544.0,
+ "1090": 977133568.0,
+ "1095": 961236864.0,
+ "1100": 979609024.0,
+ "1105": 953366336.0,
+ "1110": 965954304.0,
+ "1115": 966985344.0,
+ "1120": 970350720.0,
+ "1125": 965709824.0,
+ "1130": 954939008.0,
+ "1135": 965841728.0,
+ "1140": 965176832.0,
+ "1145": 970989504.0,
+ "1150": 955556288.0,
+ "1155": 930579200.0,
+ "1160": 957775040.0,
+ "1165": 978125824.0,
+ "1170": 974303808.0,
+ "1175": 973059712.0,
+ "1180": 973081600.0,
+ "1185": 947342208.0,
+ "1190": 964794880.0,
+ "1195": 953139200.0,
+ "1200": 972845568.0,
+ "1205": 988479104.0,
+ "1210": 931126400.0,
+ "1215": 968647168.0,
+ "1220": 969162304.0,
+ "1225": 975951360.0,
+ "1230": 967335360.0,
+ "1235": 943445120.0,
+ "1240": 955857792.0,
+ "1245": 981503808.0,
+ "1250": 966111232.0,
+ "1255": 973673344.0,
+ "1260": 946494592.0,
+ "1265": 963999488.0,
+ "1270": 960487680.0,
+ "1275": 973616128.0,
+ "1280": 961113856.0,
+ "1285": 957583488.0,
+ "1290": 952529984.0,
+ "1295": 971612800.0,
+ "1300": 968863616.0,
+ "1305": 963741440.0,
+ "1310": 963333248.0,
+ "1315": 943553536.0,
+ "1320": 966307328.0,
+ "1325": 989787264.0,
+ "1330": 969507712.0,
+ "1335": 972301824.0,
+ "1340": 972270720.0,
+ "1345": 960657792.0,
+ "1350": 968638464.0,
+ "1355": 955853120.0,
+ "1360": 971822336.0,
+ "1365": 960387584.0,
+ "1370": 948790784.0,
+ "1375": 973530240.0,
+ "1380": 953468864.0,
+ "1385": 969147008.0,
+ "1390": 975717376.0,
+ "1395": 931675264.0,
+ "1400": 945854976.0,
+ "1405": 976751488.0,
+ "1410": 974512384.0,
+ "1415": 967568640.0,
+ "1420": 966748928.0,
+ "1425": 937378944.0,
+ "1430": 973915136.0,
+ "1435": 978334336.0,
+ "1440": 964178496.0,
+ "1445": 958059456.0,
+ "1450": 946148224.0,
+ "1455": 983922496.0,
+ "1460": 968650368.0,
+ "1465": 948745536.0,
+ "1470": 984242432.0,
+ "1475": 943906048.0,
+ "1480": 963975680.0,
+ "1485": 957348864.0,
+ "1490": 961262592.0,
+ "1495": 980537280.0,
+ "1500": 958331264.0,
+ "1505": 942866944.0,
+ "1510": 984179712.0,
+ "1515": 959093120.0,
+ "1520": 959104640.0,
+ "1525": 952784704.0,
+ "1530": 957740608.0,
+ "1535": 949429632.0,
+ "1540": 971081792.0,
+ "1545": 963131776.0,
+ "1550": 978667264.0,
+ "1555": 952316864.0,
+ "1560": 980088832.0,
+ "1565": 967314944.0,
+ "1570": 973842944.0,
+ "1575": 975493248.0,
+ "1580": 941857664.0,
+ "1585": 970029056.0,
+ "1590": 983819456.0,
+ "1595": 948632192.0,
+ "1600": 967443328.0,
+ "1605": 952451072.0,
+ "1610": 969618944.0,
+ "1615": 983146624.0,
+ "1620": 968018048.0,
+ "1625": 970716096.0,
+ "1630": 962886784.0,
+ "1635": 942310912.0,
+ "1640": 981611776.0,
+ "1645": 973976512.0,
+ "1650": 974183872.0,
+ "1655": 967264640.0,
+ "1660": 940688512.0,
+ "1665": 961703040.0,
+ "1670": 962901120.0,
+ "1675": 971277184.0,
+ "1680": 980877056.0,
+ "1685": 944416128.0,
+ "1690": 964686656.0,
+ "1695": 965641088.0,
+ "1700": 966339584.0,
+ "1705": 985198208.0,
+ "1710": 978354176.0,
+ "1715": 943209216.0,
+ "1720": 977088000.0,
+ "1725": 965873792.0,
+ "1730": 968969408.0,
+ "1735": 965084288.0,
+ "1740": 949714944.0,
+ "1745": 970010688.0,
+ "1750": 959682944.0,
+ "1755": 960089600.0,
+ "1760": 966381056.0,
+ "1765": 951815488.0,
+ "1770": 954663296.0,
+ "1775": 973752448.0,
+ "1780": 970534144.0,
+ "1785": 968825472.0,
+ "1790": 950234368.0,
+ "1795": 945131648.0,
+ "1800": 984667392.0,
+ "1805": 987162112.0,
+ "1810": 977768064.0,
+ "1815": 948004352.0,
+ "1820": 949208000.0,
+ "1825": 978852096.0,
+ "1830": 966362816.0,
+ "1835": 964133376.0,
+ "1840": 972319616.0,
+ "1845": 935413504.0,
+ "1850": 952499200.0,
+ "1855": 980050816.0,
+ "1860": 975869248.0,
+ "1865": 958964544.0,
+ "1870": 958950400.0,
+ "1875": 932595072.0,
+ "1880": 973574272.0,
+ "1885": 978847104.0,
+ "1890": 971360000.0,
+ "1895": 959212032.0,
+ "1900": 947393792.0,
+ "1905": 981830912.0,
+ "1910": 969124096.0,
+ "1915": 970039936.0,
+ "1920": 975597952.0,
+ "1925": 960497280.0,
+ "1930": 977924032.0,
+ "1935": 963251200.0,
+ "1940": 952460224.0,
+ "1945": 981337472.0,
+ "1950": 939171904.0,
+ "1955": 960605568.0,
+ "1960": 970030464.0,
+ "1965": 981177472.0,
+ "1970": 962043776.0,
+ "1975": 952823424.0,
+ "1980": 936848896.0,
+ "1985": 975940736.0,
+ "1990": 965966464.0,
+ "1995": 962611456.0,
+ "2000": 960555008.0,
+ "2005": 954499200.0,
+ "2010": 975581056.0,
+ "2015": 991801792.0,
+ "2020": 975435008.0,
+ "2025": 974302976.0,
+ "2030": 952083712.0,
+ "2035": 967848448.0,
+ "2040": 987458944.0,
+ "2045": 976479360.0,
+ "2050": 984702976.0,
+ "2055": 942838016.0,
+ "2060": 942594048.0,
+ "2065": 966208512.0,
+ "2070": 969622976.0,
+ "2075": 980553792.0,
+ "2080": 977598656.0,
+ "2085": 939638016.0,
+ "2090": 969875584.0,
+ "2095": 961274048.0,
+ "2100": 976719040.0,
+ "2105": 972537344.0,
+ "2110": 959904896.0,
+ "2115": 956878848.0,
+ "2120": 977481088.0,
+ "2125": 962566656.0,
+ "2130": 979619328.0,
+ "2135": 950538496.0,
+ "2140": 946996160.0,
+ "2145": 962277888.0,
+ "2150": 973404544.0,
+ "2155": 972690560.0,
+ "2160": 970314880.0,
+ "2165": 948640000.0,
+ "2170": 961543680.0,
+ "2175": 969376896.0,
+ "2180": 969329408.0,
+ "2185": 947448576.0,
+ "2190": 940479616.0,
+ "2195": 986087424.0,
+ "2200": 961861504.0,
+ "2205": 978924288.0,
+ "2210": 964101632.0,
+ "2215": 963502080.0,
+ "2220": 951311616.0,
+ "2225": 969316992.0,
+ "2230": 976328064.0,
+ "2235": 974026560.0,
+ "2240": 975493120.0,
+ "2245": 960232576.0,
+ "2250": 967639616.0,
+ "2255": 969130752.0,
+ "2260": 975064896.0,
+ "2265": 968259968.0,
+ "2270": 951744256.0,
+ "2275": 962768576.0,
+ "2280": 969637248.0,
+ "2285": 971692416.0,
+ "2290": 962890240.0,
+ "2295": 931408128.0,
+ "2300": 959906304.0,
+ "2305": 970426752.0,
+ "2310": 967444608.0,
+ "2315": 970905344.0,
+ "2320": 975590400.0,
+ "2325": 938586880.0,
+ "2330": 988439424.0,
+ "2335": 977490176.0,
+ "2340": 964596672.0,
+ "2345": 964165888.0,
+ "2350": 947553664.0,
+ "2355": 977028864.0,
+ "2360": 966901632.0,
+ "2365": 977296576.0,
+ "2370": 965071616.0,
+ "2375": 953964800.0,
+ "2380": 962916736.0,
+ "2385": 967195392.0,
+ "2390": 963075968.0,
+ "2395": 974463232.0,
+ "2400": 958411520.0,
+ "2405": 968120704.0,
+ "2410": 951586944.0,
+ "2415": 965904896.0,
+ "2420": 966517056.0,
+ "2425": 959046016.0,
+ "2430": 956686464.0,
+ "2435": 961390208.0,
+ "2440": 959755712.0,
+ "2445": 970888704.0,
+ "2450": 961996800.0,
+ "2455": 922722368.0,
+ "2460": 951953664.0,
+ "2465": 955730688.0,
+ "2470": 972571136.0,
+ "2475": 973811776.0,
+ "2480": 943895680.0,
+ "2485": 944185216.0,
+ "2490": 972411648.0,
+ "2495": 974452480.0,
+ "2500": 973908352.0,
+ "2505": 958490112.0,
+ "2510": 939510528.0,
+ "2515": 979550912.0,
+ "2520": 970472448.0,
+ "2525": 964388608.0,
+ "2530": 955796608.0,
+ "2535": 936597696.0,
+ "2540": 969027008.0,
+ "2545": 970385088.0,
+ "2550": 969461376.0,
+ "2555": 969440256.0,
+ "2560": 964978688.0,
+ "2565": 959765504.0,
+ "2570": 985177344.0,
+ "2575": 957424640.0,
+ "2580": 967423872.0,
+ "2585": 966023168.0,
+ "2590": 956354816.0,
+ "2595": 981828736.0,
+ "2600": 959531456.0,
+ "2605": 963000320.0,
+ "2610": 965972736.0,
+ "2615": 951925504.0,
+ "2620": 971241792.0,
+ "2625": 976456000.0,
+ "2630": 974411008.0,
+ "2635": 948070528.0,
+ "2640": 948137536.0,
+ "2645": 963037504.0,
+ "2650": 953983488.0,
+ "2655": 977112320.0,
+ "2660": 949622400.0,
+ "2665": 953930112.0,
+ "2670": 959063104.0,
+ "2675": 979275328.0,
+ "2680": 961396608.0,
+ "2685": 970700992.0,
+ "2690": 965221376.0,
+ "2695": 943552256.0,
+ "2700": 969424320.0,
+ "2705": 978961152.0,
+ "2710": 971810624.0,
+ "2715": 990813696.0,
+ "2720": 942648576.0,
+ "2725": 967955072.0,
+ "2730": 955466432.0,
+ "2735": 970671680.0,
+ "2740": 977920768.0,
+ "2745": 932277184.0,
+ "2750": 947856896.0,
+ "2755": 956317312.0,
+ "2760": 981696576.0,
+ "2765": 966112512.0,
+ "2770": 948914816.0,
+ "2775": 935831168.0,
+ "2780": 964779392.0,
+ "2785": 969569152.0,
+ "2790": 974273280.0,
+ "2795": 966886464.0,
+ "2800": 944390336.0,
+ "2805": 964353152.0,
+ "2810": 969610112.0,
+ "2815": 975844992.0,
+ "2820": 963087232.0,
+ "2825": 937630464.0,
+ "2830": 956741184.0,
+ "2835": 986323136.0,
+ "2840": 961760640.0,
+ "2845": 967509376.0,
+ "2850": 951717760.0,
+ "2855": 962093184.0,
+ "2860": 954244160.0,
+ "2865": 955883520.0,
+ "2870": 944663744.0,
+ "2875": 974664576.0,
+ "2880": 968203584.0,
+ "2885": 981082880.0,
+ "2890": 953455424.0,
+ "2895": 957180416.0,
+ "2900": 964989952.0,
+ "2905": 931707840.0,
+ "2910": 955731584.0,
+ "2915": 979475584.0,
+ "2920": 970492544.0,
+ "2925": 964975744.0,
+ "2930": 964048320.0,
+ "2935": 940140672.0,
+ "2940": 964912256.0,
+ "2945": 989149120.0,
+ "2950": 965210112.0,
+ "2955": 965103936.0,
+ "2960": 933162304.0,
+ "2965": 968795008.0,
+ "2970": 973035456.0,
+ "2975": 958092224.0,
+ "2980": 964498432.0,
+ "2985": 937269120.0,
+ "2990": 951253824.0,
+ "2995": 978316800.0,
+ "3000": 969276480.0,
+ "3005": 974686272.0,
+ "3010": 950235392.0,
+ "3015": 943840768.0,
+ "3020": 958442752.0,
+ "3025": 975186560.0,
+ "3030": 965017536.0,
+ "3035": 963453888.0,
+ "3040": 952134912.0,
+ "3045": 989794240.0,
+ "3050": 965545472.0,
+ "3055": 982520576.0,
+ "3060": 971226112.0,
+ "3065": 943915776.0,
+ "3070": 978408576.0,
+ "3075": 975205120.0,
+ "3080": 960991872.0,
+ "3085": 962352384.0,
+ "3090": 945953152.0,
+ "3095": 938115200.0,
+ "3100": 972929600.0,
+ "3105": 961989824.0,
+ "3110": 970656768.0,
+ "3115": 963390464.0,
+ "3120": 947116928.0,
+ "3125": 972720768.0,
+ "3130": 952973952.0,
+ "3135": 966041472.0,
+ "3140": 968488256.0,
+ "3145": 937852416.0,
+ "3150": 975010240.0,
+ "3155": 976814592.0,
+ "3160": 969627968.0,
+ "3165": 982194304.0,
+ "3170": 937959744.0,
+ "3175": 953826560.0,
+ "3180": 983809536.0,
+ "3185": 965169152.0,
+ "3190": 968482880.0,
+ "3195": 950932736.0,
+ "3200": 945101824.0,
+ "3205": 959861504.0,
+ "3210": 957486464.0,
+ "3215": 958020480.0,
+ "3220": 968130112.0,
+ "3225": 935615104.0,
+ "3230": 962589184.0,
+ "3235": 975776128.0,
+ "3240": 962621696.0,
+ "3245": 981274240.0,
+ "3250": 943259968.0,
+ "3255": 954599104.0,
+ "3260": 980363392.0,
+ "3265": 963620864.0,
+ "3270": 965163904.0,
+ "3275": 959731840.0,
+ "3280": 967046016.0,
+ "3285": 982476928.0,
+ "3290": 947689088.0,
+ "3295": 966422912.0,
+ "3300": 959166464.0,
+ "3305": 949131840.0,
+ "3310": 979510912.0,
+ "3315": 964284352.0,
+ "3320": 969208320.0,
+ "3325": 956195136.0,
+ "3330": 941167040.0,
+ "3335": 964974336.0,
+ "3340": 956901504.0,
+ "3345": 972501184.0,
+ "3350": 964574464.0,
+ "3355": 943343168.0,
+ "3360": 970036736.0,
+ "3365": 969453248.0,
+ "3370": 954763264.0,
+ "3375": 958677248.0,
+ "3380": 971464320.0,
+ "3385": 947973824.0,
+ "3390": 965781312.0,
+ "3395": 978391424.0,
+ "3400": 978128512.0,
+ "3405": 976725248.0,
+ "3410": 924193024.0,
+ "3415": 955425536.0,
+ "3420": 971820672.0,
+ "3425": 977156416.0,
+ "3430": 973826816.0,
+ "3435": 936072704.0,
+ "3440": 970503232.0,
+ "3445": 957302336.0,
+ "3450": 959840384.0,
+ "3455": 963854720.0,
+ "3460": 967882624.0,
+ "3465": 931319936.0,
+ "3470": 952335488.0,
+ "3475": 973704640.0,
+ "3480": 959738368.0,
+ "3485": 979948800.0,
+ "3490": 944671360.0,
+ "3495": 953905216.0,
+ "3500": 969318272.0,
+ "3505": 964348544.0,
+ "3510": 971222272.0,
+ "3515": 955946880.0,
+ "3520": 958734080.0,
+ "3525": 971914944.0,
+ "3530": 964126528.0,
+ "3535": 983191936.0,
+ "3540": 937491328.0,
+ "3545": 944731776.0,
+ "3550": 984462720.0,
+ "3555": 978058944.0,
+ "3560": 974374784.0,
+ "3565": 968801216.0,
+ "3570": 946694976.0,
+ "3575": 976117248.0,
+ "3580": 977494464.0,
+ "3585": 954581248.0,
+ "3590": 956426496.0,
+ "3595": 951457280.0,
+ "3600": 989009024.0,
+ "3605": 962008832.0,
+ "3610": 965065792.0,
+ "3615": 974648192.0,
+ "3620": 954900480.0,
+ "3625": 939528256.0,
+ "3630": 990161536.0,
+ "3635": 971435904.0,
+ "3640": 976024896.0,
+ "3645": 961489024.0,
+ "3650": 945804160.0,
+ "3655": 965794816.0,
+ "3660": 976196480.0,
+ "3665": 964021760.0,
+ "3670": 977445760.0,
+ "3675": 943476224.0,
+ "3680": 958185856.0,
+ "3685": 964286400.0,
+ "3690": 982094144.0,
+ "3695": 963125888.0,
+ "3700": 950571328.0,
+ "3705": 947345920.0,
+ "3710": 982356096.0,
+ "3715": 972681856.0,
+ "3720": 976138112.0,
+ "3725": 964038656.0,
+ "3730": 948849536.0,
+ "3735": 967079680.0,
+ "3740": 960960832.0,
+ "3745": 969322944.0,
+ "3750": 963936896.0,
+ "3755": 953420800.0,
+ "3760": 976641984.0,
+ "3765": 979829376.0,
+ "3770": 972359744.0,
+ "3775": 972373376.0,
+ "3780": 952573888.0,
+ "3785": 960225408.0,
+ "3790": 985586048.0,
+ "3795": 969183872.0,
+ "3800": 957865728.0,
+ "3805": 972396800.0,
+ "3810": 954505408.0,
+ "3815": 974555136.0,
+ "3820": 963006592.0,
+ "3825": 962059520.0,
+ "3830": 969383040.0,
+ "3835": 934733568.0,
+ "3840": 971243904.0,
+ "3845": 986837248.0,
+ "3850": 968849024.0,
+ "3855": 965235648.0,
+ "3860": 948045568.0,
+ "3865": 975036736.0,
+ "3870": 985086976.0,
+ "3875": 983029376.0,
+ "3880": 963603200.0,
+ "3885": 953000064.0,
+ "3890": 960284160.0,
+ "3895": 960578496.0,
+ "3900": 984913536.0,
+ "3905": 976211584.0,
+ "3910": 987323712.0,
+ "3915": 946029888.0,
+ "3920": 974879360.0,
+ "3925": 961235456.0,
+ "3930": 976765824.0,
+ "3935": 978901120.0,
+ "3940": 950293184.0,
+ "3945": 960261504.0,
+ "3950": 974173440.0,
+ "3955": 972979840.0,
+ "3960": 974042048.0,
+ "3965": 950886400.0,
+ "3970": 980680576.0,
+ "3975": 960738176.0,
+ "3980": 977519104.0,
+ "3985": 962937600.0,
+ "3990": 972755712.0,
+ "3995": 953677056.0,
+ "4000": 974963584.0,
+ "4005": 971645952.0,
+ "4010": 978396480.0,
+ "4015": 971467008.0,
+ "4020": 950334528.0,
+ "4025": 968445184.0,
+ "4030": 997960384.0,
+ "4035": 978560832.0,
+ "4040": 959813632.0,
+ "4045": 939663552.0,
+ "4050": 944706176.0,
+ "4055": 980959680.0,
+ "4060": 977695232.0,
+ "4065": 975718208.0,
+ "4070": 942152576.0,
+ "4075": 945752320.0,
+ "4080": 988741376.0,
+ "4085": 962067968.0,
+ "4090": 983356928.0,
+ "4095": 986945728.0,
+ "4100": 957188160.0,
+ "4105": 954086272.0,
+ "4110": 966486528.0,
+ "4115": 976029568.0,
+ "4120": 983530752.0,
+ "4125": 960041856.0,
+ "4130": 967289984.0,
+ "4135": 971454464.0,
+ "4140": 963159168.0,
+ "4145": 956199808.0,
+ "4150": 960319296.0,
+ "4155": 946193280.0,
+ "4160": 968426880.0,
+ "4165": 970351488.0,
+ "4170": 972008576.0,
+ "4175": 955898944.0,
+ "4180": 940996608.0,
+ "4185": 968293504.0,
+ "4190": 968040000.0,
+ "4195": 989248320.0,
+ "4200": 962678656.0,
+ "4205": 960595072.0,
+ "4210": 971837568.0,
+ "4215": 974168832.0,
+ "4220": 981145280.0,
+ "4225": 975208704.0,
+ "4230": 952678272.0,
+ "4235": 958456192.0,
+ "4240": 966824128.0,
+ "4245": 961718784.0,
+ "4250": 965833216.0,
+ "4255": 958255872.0,
+ "4260": 949557888.0,
+ "4265": 964137536.0,
+ "4270": 978396672.0,
+ "4275": 975410176.0,
+ "4280": 962671680.0,
+ "4285": 951440064.0,
+ "4290": 980029632.0,
+ "4295": 968818560.0,
+ "4300": 958274304.0,
+ "4305": 966815104.0,
+ "4310": 939614208.0,
+ "4315": 949429312.0,
+ "4320": 984548608.0,
+ "4325": 982584960.0,
+ "4330": 974746112.0,
+ "4335": 949471616.0,
+ "4340": 959541248.0,
+ "4345": 956619648.0,
+ "4350": 979947648.0,
+ "4355": 968833536.0,
+ "4360": 966363520.0,
+ "4365": 941244800.0,
+ "4370": 969470976.0,
+ "4375": 972995776.0,
+ "4380": 966147968.0,
+ "4385": 972035840.0,
+ "4390": 954153408.0,
+ "4395": 951868544.0,
+ "4400": 973640320.0,
+ "4405": 972478848.0,
+ "4410": 967920896.0,
+ "4415": 958937088.0,
+ "4420": 960777856.0,
+ "4425": 976428544.0,
+ "4430": 965968384.0,
+ "4435": 975901440.0,
+ "4440": 962332288.0,
+ "4445": 954849664.0,
+ "4450": 978307456.0,
+ "4455": 960230464.0,
+ "4460": 968569216.0,
+ "4465": 968757120.0,
+ "4470": 944102272.0,
+ "4475": 951972800.0,
+ "4480": 978813056.0,
+ "4485": 968302848.0,
+ "4490": 957025536.0,
+ "4495": 938938176.0,
+ "4500": 953241088.0,
+ "4505": 977178752.0,
+ "4510": 978724864.0,
+ "4515": 962468608.0,
+ "4520": 958824704.0,
+ "4525": 958102016.0,
+ "4530": 964416384.0,
+ "4535": 976616576.0,
+ "4540": 976871680.0,
+ "4545": 970144640.0,
+ "4550": 953199232.0,
+ "4555": 959578048.0,
+ "4560": 972472384.0,
+ "4565": 973467776.0,
+ "4570": 978947072.0,
+ "4575": 957798080.0,
+ "4580": 963180800.0,
+ "4585": 957380608.0,
+ "4590": 986567936.0,
+ "4595": 960214592.0,
+ "4600": 952255872.0,
+ "4605": 959274432.0,
+ "4610": 963772160.0,
+ "4615": 958012672.0,
+ "4620": 960256320.0,
+ "4625": 973835648.0,
+ "4630": 944496000.0,
+ "4635": 977013440.0,
+ "4640": 960259968.0,
+ "4645": 981992512.0,
+ "4650": 962375232.0,
+ "4655": 939472128.0,
+ "4660": 964000832.0,
+ "4665": 962565760.0,
+ "4670": 976683904.0,
+ "4675": 963478144.0,
+ "4680": 957451840.0,
+ "4685": 949662656.0,
+ "4690": 956937856.0,
+ "4695": 969950592.0,
+ "4700": 961190656.0,
+ "4705": 970768960.0,
+ "4710": 934469120.0,
+ "4715": 970426496.0,
+ "4720": 966336768.0,
+ "4725": 980291584.0,
+ "4730": 965826176.0,
+ "4735": 937835776.0,
+ "4740": 960112128.0,
+ "4745": 976009856.0,
+ "4750": 967935488.0,
+ "4755": 984935680.0,
+ "4760": 959148800.0,
+ "4765": 955200960.0,
+ "4770": 958584768.0,
+ "4775": 991080192.0,
+ "4780": 976757824.0,
+ "4785": 967570560.0,
+ "4790": 943719616.0,
+ "4795": 955799936.0,
+ "4800": 967697152.0,
+ "4805": 976492800.0,
+ "4810": 965148224.0,
+ "4815": 957982720.0,
+ "4820": 973943872.0,
+ "4825": 961432704.0,
+ "4830": 962614016.0,
+ "4835": 972544512.0,
+ "4840": 948901888.0,
+ "4845": 965715584.0,
+ "4850": 960293184.0,
+ "4855": 964168320.0,
+ "4860": 963038464.0,
+ "4865": 967507072.0,
+ "4870": 957207808.0,
+ "4875": 983562112.0,
+ "4880": 957035712.0,
+ "4885": 977040896.0,
+ "4890": 959740608.0,
+ "4895": 942201728.0,
+ "4900": 973663296.0,
+ "4905": 975201920.0,
+ "4910": 969245376.0,
+ "4915": 970078336.0,
+ "4920": 941145280.0,
+ "4925": 954784768.0,
+ "4930": 977046848.0,
+ "4935": 963774912.0,
+ "4940": 972611648.0,
+ "4945": 959971008.0,
+ "4950": 940758016.0,
+ "4955": 967988160.0,
+ "4960": 976764672.0,
+ "4965": 960972608.0,
+ "4970": 958543744.0,
+ "4975": 933702400.0,
+ "4980": 960813056.0,
+ "4985": 962963392.0,
+ "4990": 963553152.0,
+ "4995": 986284480.0,
+ "5000": 940703360.0,
+ "5005": 968887552.0,
+ "5010": 970308096.0,
+ "5015": 965208704.0,
+ "5020": 966690944.0,
+ "5025": 949385920.0,
+ "5030": 953442560.0,
+ "5035": 967304320.0,
+ "5040": 955678272.0,
+ "5045": 969118336.0,
+ "5050": 953356416.0,
+ "5055": 954799616.0,
+ "5060": 963030400.0,
+ "5065": 952183168.0,
+ "5070": 973597952.0,
+ "5075": 978556800.0,
+ "5080": 942865920.0,
+ "5085": 965826112.0,
+ "5090": 972856384.0,
+ "5095": 964435200.0,
+ "5100": 958367808.0,
+ "5105": 965341952.0,
+ "5110": 950402368.0,
+ "5115": 972383616.0,
+ "5120": 960425728.0,
+ "5125": 969731200.0,
+ "5130": 938820736.0,
+ "5135": 943697216.0,
+ "5140": 969888896.0,
+ "5145": 968603200.0,
+ "5150": 970638336.0,
+ "5155": 972598912.0,
+ "5160": 926527232.0,
+ "5165": 961629184.0,
+ "5170": 966850304.0,
+ "5175": 966061312.0,
+ "5180": 963668224.0,
+ "5185": 930852608.0,
+ "5190": 949780992.0,
+ "5195": 972447104.0,
+ "5200": 973784000.0,
+ "5205": 968213824.0,
+ "5210": 960503936.0,
+ "5215": 928804608.0,
+ "5220": 979174272.0,
+ "5225": 984779648.0,
+ "5230": 975024896.0,
+ "5235": 975043648.0,
+ "5240": 944313472.0,
+ "5245": 970787328.0,
+ "5250": 972415104.0,
+ "5255": 966871872.0,
+ "5260": 976729728.0,
+ "5265": 942223360.0,
+ "5270": 969202304.0,
+ "5275": 970110528.0,
+ "5280": 962826112.0,
+ "5285": 964096896.0,
+ "5290": 932539008.0,
+ "5295": 951751808.0,
+ "5300": 975588608.0,
+ "5305": 951842688.0,
+ "5310": 968040512.0,
+ "5315": 955720064.0,
+ "5320": 950885120.0,
+ "5325": 973053056.0,
+ "5330": 967782656.0,
+ "5335": 967585280.0,
+ "5340": 966443584.0,
+ "5345": 962912384.0,
+ "5350": 978899968.0,
+ "5355": 972204288.0,
+ "5360": 963818368.0,
+ "5365": 965210752.0,
+ "5370": 947777152.0,
+ "5375": 948783488.0,
+ "5380": 967150208.0,
+ "5385": 980441984.0,
+ "5390": 965277568.0,
+ "5395": 955074816.0,
+ "5400": 948262656.0,
+ "5405": 974243840.0,
+ "5410": 967782976.0,
+ "5415": 976050816.0,
+ "5420": 967358464.0,
+ "5425": 937314752.0,
+ "5430": 963808192.0,
+ "5435": 971801344.0,
+ "5440": 969017856.0,
+ "5445": 957391360.0,
+ "5450": 919358848.0,
+ "5455": 952027648.0,
+ "5460": 962326656.0,
+ "5465": 978770752.0,
+ "5470": 980846592.0,
+ "5475": 941512960.0,
+ "5480": 955731392.0,
+ "5485": 964854272.0,
+ "5490": 975913088.0,
+ "5495": 962719104.0,
+ "5500": 971083136.0,
+ "5505": 956855296.0,
+ "5510": 968464000.0,
+ "5515": 945392640.0,
+ "5520": 963158272.0,
+ "5525": 975747904.0,
+ "5530": 936486784.0,
+ "5535": 970620928.0,
+ "5540": 960246272.0,
+ "5545": 972023808.0,
+ "5550": 967775104.0,
+ "5555": 955895808.0,
+ "5560": 954422592.0,
+ "5565": 968691520.0,
+ "5570": 945089216.0,
+ "5575": 960418688.0,
+ "5580": 960526976.0,
+ "5585": 959396736.0,
+ "5590": 977582848.0,
+ "5595": 975146240.0,
+ "5600": 962923904.0,
+ "5605": 964054400.0,
+ "5610": 943058944.0,
+ "5615": 966442240.0,
+ "5620": 963212480.0,
+ "5625": 982264192.0,
+ "5630": 975889600.0,
+ "5635": 957163136.0,
+ "5640": 951369664.0,
+ "5645": 967695616.0,
+ "5650": 978997888.0,
+ "5655": 983425152.0,
+ "5660": 956272384.0,
+ "5665": 953364864.0,
+ "5670": 966013248.0,
+ "5675": 967555456.0,
+ "5680": 978488192.0,
+ "5685": 961909888.0,
+ "5690": 935774208.0,
+ "5695": 963653376.0,
+ "5700": 952427392.0,
+ "5705": 974427328.0,
+ "5710": 971157440.0,
+ "5715": 946073792.0,
+ "5720": 974930304.0,
+ "5725": 967388288.0,
+ "5730": 978427520.0,
+ "5735": 964795648.0,
+ "5740": 943538560.0,
+ "5745": 971106368.0,
+ "5750": 981828864.0,
+ "5755": 956631808.0,
+ "5760": 963585280.0,
+ "5765": 957608640.0,
+ "5770": 955601216.0,
+ "5775": 970675776.0,
+ "5780": 962738752.0,
+ "5785": 970490624.0,
+ "5790": 974666624.0,
+ "5795": 949705088.0,
+ "5800": 965914048.0,
+ "5805": 968700800.0,
+ "5810": 975987200.0,
+ "5815": 969991936.0,
+ "5820": 936229632.0,
+ "5825": 969217152.0,
+ "5830": 977565696.0,
+ "5835": 974930944.0,
+ "5840": 962975936.0,
+ "5845": 968653120.0,
+ "5850": 942886528.0,
+ "5855": 975877120.0,
+ "5860": 979501056.0,
+ "5865": 978198336.0,
+ "5870": 968715008.0,
+ "5875": 942132480.0,
+ "5880": 964251712.0,
+ "5885": 974611072.0,
+ "5890": 972438784.0,
+ "5895": 965571968.0,
+ "5900": 941212992.0,
+ "5905": 961669376.0,
+ "5910": 958372608.0,
+ "5915": 968039040.0,
+ "5920": 977222784.0,
+ "5925": 959432832.0,
+ "5930": 946590336.0,
+ "5935": 952272704.0,
+ "5940": 977613312.0,
+ "5945": 984918272.0,
+ "5950": 980554688.0,
+ "5955": 934946432.0,
+ "5960": 961601984.0,
+ "5965": 965853312.0,
+ "5970": 970501248.0,
+ "5975": 961882112.0,
+ "5980": 958110464.0,
+ "5985": 964414784.0,
+ "5990": 973356544.0,
+ "5995": 955833984.0,
+ "6000": 955485696.0,
+ "6005": 961293504.0,
+ "6010": 952645440.0,
+ "6015": 974415872.0,
+ "6020": 978156224.0,
+ "6025": 972178944.0,
+ "6030": 955398272.0,
+ "6035": 946868864.0,
+ "6040": 962679808.0,
+ "6045": 983656832.0,
+ "6050": 956588800.0,
+ "6055": 963322816.0,
+ "6060": 945595392.0,
+ "6065": 958394496.0,
+ "6070": 978304896.0,
+ "6075": 978074880.0,
+ "6080": 957444096.0,
+ "6085": 947629952.0,
+ "6090": 953646976.0,
+ "6095": 964623552.0,
+ "6100": 979801088.0,
+ "6105": 971049216.0,
+ "6110": 961718080.0,
+ "6115": 943655552.0,
+ "6120": 968487808.0,
+ "6125": 960593088.0,
+ "6130": 983892416.0,
+ "6135": 960969600.0,
+ "6140": 958597248.0,
+ "6145": 971126656.0,
+ "6150": 968345856.0,
+ "6155": 974893056.0,
+ "6160": 977191168.0,
+ "6165": 952755968.0,
+ "6170": 951080576.0,
+ "6175": 963409152.0,
+ "6180": 969340416.0,
+ "6185": 966259200.0,
+ "6190": 963511168.0,
+ "6195": 947045312.0,
+ "6200": 969109504.0,
+ "6205": 967150016.0,
+ "6210": 959082176.0,
+ "6215": 972729600.0,
+ "6220": 936227072.0,
+ "6225": 978544512.0,
+ "6230": 975876224.0,
+ "6235": 971570944.0,
+ "6240": 965809728.0,
+ "6245": 955906304.0,
+ "6250": 956425216.0,
+ "6255": 972970624.0,
+ "6260": 978730816.0,
+ "6265": 974782336.0,
+ "6270": 958695808.0,
+ "6275": 963775040.0,
+ "6280": 973067264.0,
+ "6285": 965963008.0,
+ "6290": 970720576.0,
+ "6295": 987342912.0,
+ "6300": 947369216.0,
+ "6305": 964586496.0,
+ "6310": 978893696.0,
+ "6315": 978454144.0,
+ "6320": 971650560.0,
+ "6325": 922867200.0,
+ "6330": 959047104.0,
+ "6335": 974670464.0,
+ "6340": 984554240.0,
+ "6345": 966774720.0,
+ "6350": 944513408.0,
+ "6355": 957805696.0,
+ "6360": 972588800.0,
+ "6365": 972220608.0,
+ "6370": 958906048.0,
+ "6375": 967154496.0,
+ "6380": 951240640.0,
+ "6385": 973634432.0,
+ "6390": 965183552.0,
+ "6395": 974971584.0,
+ "6400": 984093056.0,
+ "6405": 943782784.0,
+ "6410": 977277312.0,
+ "6415": 971329280.0,
+ "6420": 956394752.0,
+ "6425": 960816512.0,
+ "6430": 957378304.0,
+ "6435": 959965056.0,
+ "6440": 968701376.0,
+ "6445": 973296128.0,
+ "6450": 974207936.0,
+ "6455": 961958528.0,
+ "6460": 941006720.0,
+ "6465": 974272768.0,
+ "6470": 979691584.0,
+ "6475": 960665216.0,
+ "6480": 967482240.0,
+ "6485": 948620800.0,
+ "6490": 970648896.0,
+ "6495": 988281728.0,
+ "6500": 980382464.0,
+ "6505": 971875712.0,
+ "6510": 951611008.0,
+ "6515": 957443328.0,
+ "6520": 978938496.0,
+ "6525": 978842496.0,
+ "6530": 973296256.0,
+ "6535": 967939328.0,
+ "6540": 950123456.0,
+ "6545": 966146752.0,
+ "6550": 979249280.0,
+ "6555": 967058688.0,
+ "6560": 975182336.0,
+ "6565": 949290112.0,
+ "6570": 951981440.0,
+ "6575": 962234560.0,
+ "6580": 975681280.0,
+ "6585": 979275392.0,
+ "6590": 948979200.0,
+ "6595": 961632640.0,
+ "6600": 960998144.0,
+ "6605": 961411968.0,
+ "6610": 985152000.0,
+ "6615": 959273408.0,
+ "6620": 944343296.0,
+ "6625": 970979712.0,
+ "6630": 971369024.0,
+ "6635": 964018688.0,
+ "6640": 959501376.0,
+ "6645": 950697984.0,
+ "6650": 978663936.0,
+ "6655": 965815168.0,
+ "6660": 968359488.0,
+ "6665": 968707584.0,
+ "6670": 932667904.0,
+ "6675": 970711552.0,
+ "6680": 969041152.0,
+ "6685": 958538048.0,
+ "6690": 956117248.0,
+ "6695": 955316864.0,
+ "6700": 962073984.0,
+ "6705": 978985088.0,
+ "6710": 970864576.0,
+ "6715": 966629824.0,
+ "6720": 973916544.0,
+ "6725": 941781952.0,
+ "6730": 979276864.0,
+ "6735": 994439680.0,
+ "6740": 976625664.0,
+ "6745": 974443200.0,
+ "6750": 938891264.0,
+ "6755": 977391616.0,
+ "6760": 969598208.0,
+ "6765": 978356480.0,
+ "6770": 975059072.0,
+ "6775": 943254272.0,
+ "6780": 947074624.0,
+ "6785": 975161856.0,
+ "6790": 960433984.0,
+ "6795": 975886336.0,
+ "6800": 972829312.0,
+ "6805": 946376320.0,
+ "6810": 958179712.0,
+ "6815": 970549696.0,
+ "6820": 977619456.0,
+ "6825": 969243264.0,
+ "6830": 950263168.0,
+ "6835": 981328512.0,
+ "6840": 982974272.0,
+ "6845": 948935808.0,
+ "6850": 965416512.0,
+ "6855": 953916544.0,
+ "6860": 979025792.0,
+ "6865": 983853312.0,
+ "6870": 964855232.0,
+ "6875": 978762496.0,
+ "6880": 950433920.0,
+ "6885": 958312448.0,
+ "6890": 960313472.0,
+ "6895": 965972928.0,
+ "6900": 985248128.0,
+ "6905": 968528640.0,
+ "6910": 949863808.0,
+ "6915": 971062208.0,
+ "6920": 967351744.0,
+ "6925": 964971776.0,
+ "6930": 964557376.0,
+ "6935": 952123328.0,
+ "6940": 963032320.0,
+ "6945": 986507520.0,
+ "6950": 973327232.0,
+ "6955": 964531968.0,
+ "6960": 940003712.0,
+ "6965": 974492160.0,
+ "6970": 978420224.0,
+ "6975": 984993472.0,
+ "6980": 982692096.0,
+ "6985": 959667584.0,
+ "6990": 945238272.0,
+ "6995": 987459264.0,
+ "7000": 963220480.0,
+ "7005": 962812480.0,
+ "7010": 984947712.0,
+ "7015": 945496384.0,
+ "7020": 983131008.0,
+ "7025": 968731392.0,
+ "7030": 953218944.0,
+ "7035": 982774144.0,
+ "7040": 950460160.0,
+ "7045": 955747072.0,
+ "7050": 959962240.0,
+ "7055": 963622912.0,
+ "7060": 976790272.0,
+ "7065": 968059264.0,
+ "7070": 953450048.0,
+ "7075": 956437504.0,
+ "7080": 969003520.0,
+ "7085": 965390272.0,
+ "7090": 969313408.0,
+ "7095": 959464128.0,
+ "7100": 973246080.0,
+ "7105": 972939648.0,
+ "7110": 970071104.0,
+ "7115": 958216192.0,
+ "7120": 949039872.0,
+ "7125": 963264320.0,
+ "7130": 971177472.0,
+ "7135": 964579328.0,
+ "7140": 961276800.0,
+ "7145": 930369536.0,
+ "7150": 946022400.0,
+ "7155": 990835520.0,
+ "7160": 968320896.0,
+ "7165": 956643200.0,
+ "7170": 968043008.0,
+ "7175": 955791488.0,
+ "7180": 957939072.0,
+ "7185": 984720896.0,
+ "7190": 978552640.0,
+ "7195": 973603584.0,
+ "7200": 935673600.0,
+ "7205": 957390912.0,
+ "7210": 967078464.0,
+ "7215": 969339904.0,
+ "7220": 982085632.0,
+ "7225": 928925056.0,
+ "7230": 949390144.0,
+ "7235": 966846336.0,
+ "7240": 966649472.0,
+ "7245": 967233792.0,
+ "7250": 949331392.0,
+ "7255": 957081344.0,
+ "7260": 970027584.0,
+ "7265": 974604544.0,
+ "7270": 959689920.0,
+ "7275": 959067136.0,
+ "7280": 956852096.0,
+ "7285": 977440768.0,
+ "7290": 976980608.0,
+ "7295": 962803904.0,
+ "7300": 975069824.0,
+ "7305": 963961984.0,
+ "7310": 977308032.0,
+ "7315": 966674560.0,
+ "7320": 974209536.0,
+ "7325": 966674496.0,
+ "7330": 959420800.0,
+ "7335": 963904000.0,
+ "7340": 977262336.0,
+ "7345": 967447680.0,
+ "7350": 984572288.0,
+ "7355": 959405568.0,
+ "7360": 948622016.0,
+ "7365": 972354432.0,
+ "7370": 982372864.0,
+ "7375": 962879360.0,
+ "7380": 964038784.0,
+ "7385": 948431488.0,
+ "7390": 963677696.0,
+ "7395": 957965440.0,
+ "7400": 969640576.0,
+ "7405": 987491264.0,
+ "7410": 951945152.0,
+ "7415": 950271936.0,
+ "7420": 966599680.0,
+ "7425": 982952832.0,
+ "7430": 965750400.0,
+ "7435": 973161024.0,
+ "7440": 936884736.0,
+ "7445": 968767616.0,
+ "7450": 980462208.0,
+ "7455": 971280896.0,
+ "7460": 972147904.0,
+ "7465": 939168000.0,
+ "7470": 971950592.0,
+ "7475": 957960000.0,
+ "7480": 969186432.0,
+ "7485": 961158592.0,
+ "7490": 934207552.0,
+ "7495": 957347008.0,
+ "7500": 968909696.0,
+ "7505": 969939584.0,
+ "7510": 971938176.0,
+ "7515": 979643968.0,
+ "7520": 951674240.0,
+ "7525": 970150400.0,
+ "7530": 954250368.0,
+ "7535": 971002624.0,
+ "7540": 979351424.0,
+ "7545": 959002112.0,
+ "7550": 959936640.0,
+ "7555": 960035136.0,
+ "7560": 969951360.0,
+ "7565": 954850304.0,
+ "7570": 942336704.0,
+ "7575": 965396736.0,
+ "7580": 981979904.0,
+ "7585": 978810240.0,
+ "7590": 970279296.0,
+ "7595": 949956608.0,
+ "7600": 945846592.0,
+ "7605": 982612096.0,
+ "7610": 969044480.0,
+ "7615": 988564736.0,
+ "7620": 957000192.0,
+ "7625": 941454976.0,
+ "7630": 971133056.0,
+ "7635": 984264576.0,
+ "7640": 983299712.0,
+ "7645": 968097472.0,
+ "7650": 959086656.0,
+ "7655": 962007040.0,
+ "7660": 969076928.0,
+ "7665": 978171584.0,
+ "7670": 975401856.0,
+ "7675": 975338368.0,
+ "7680": 942762112.0,
+ "7685": 959972096.0,
+ "7690": 975364608.0,
+ "7695": 982221568.0,
+ "7700": 979086976.0,
+ "7705": 940546432.0,
+ "7710": 974737280.0,
+ "7715": 978988352.0,
+ "7720": 967479232.0,
+ "7725": 960220928.0,
+ "7730": 942869632.0,
+ "7735": 968105792.0,
+ "7740": 980045568.0,
+ "7745": 963941632.0,
+ "7750": 963412288.0,
+ "7755": 959505408.0,
+ "7760": 970593920.0,
+ "7765": 970864320.0,
+ "7770": 962472832.0,
+ "7775": 982167424.0,
+ "7780": 964654592.0,
+ "7785": 959739904.0,
+ "7790": 968466944.0,
+ "7795": 968762112.0,
+ "7800": 972053184.0,
+ "7805": 968564992.0,
+ "7810": 945972224.0,
+ "7815": 963664576.0,
+ "7820": 974677184.0,
+ "7825": 963635712.0,
+ "7830": 957630336.0,
+ "7835": 949710016.0,
+ "7840": 957638784.0,
+ "7845": 953894016.0,
+ "7850": 979439872.0,
+ "7855": 987051200.0,
+ "7860": 947830016.0,
+ "7865": 949621120.0,
+ "7870": 965052160.0,
+ "7875": 975974016.0,
+ "7880": 968774208.0,
+ "7885": 969494720.0,
+ "7890": 952188608.0,
+ "7895": 974695424.0,
+ "7900": 963989504.0,
+ "7905": 964420608.0,
+ "7910": 965869184.0,
+ "7915": 943378176.0,
+ "7920": 951312064.0,
+ "7925": 969302016.0,
+ "7930": 964944256.0,
+ "7935": 984349376.0,
+ "7940": 964437184.0,
+ "7945": 951030016.0,
+ "7950": 962343936.0,
+ "7955": 980155648.0,
+ "7960": 963591552.0,
+ "7965": 952763776.0,
+ "7970": 951961984.0,
+ "7975": 969783232.0,
+ "7980": 965072384.0,
+ "7985": 959288192.0,
+ "7990": 968166144.0,
+ "7995": 946780480.0,
+ "8000": 962926784.0,
+ "8005": 981072256.0,
+ "8010": 965470912.0,
+ "8015": 982932416.0,
+ "8020": 960808768.0,
+ "8025": 965372544.0,
+ "8030": 958532992.0,
+ "8035": 975850880.0,
+ "8040": 960656128.0,
+ "8045": 948529280.0,
+ "8050": 959733888.0,
+ "8055": 979126144.0,
+ "8060": 969222912.0,
+ "8065": 958210240.0,
+ "8070": 963956032.0,
+ "8075": 941995392.0,
+ "8080": 966046912.0,
+ "8085": 966803776.0,
+ "8090": 983576064.0,
+ "8095": 988772800.0,
+ "8100": 966714752.0,
+ "8105": 944461696.0,
+ "8110": 968896000.0,
+ "8115": 985733696.0,
+ "8120": 974861568.0,
+ "8125": 964166528.0,
+ "8130": 966678400.0,
+ "8135": 967760256.0,
+ "8140": 963773824.0,
+ "8145": 995469568.0,
+ "8150": 973350080.0,
+ "8155": 938673152.0,
+ "8160": 964474496.0,
+ "8165": 973369472.0,
+ "8170": 968755200.0,
+ "8175": 961722624.0,
+ "8180": 936287232.0,
+ "8185": 963154048.0,
+ "8190": 968499840.0,
+ "8195": 977409792.0,
+ "8200": 956736768.0,
+ "8205": 960825152.0,
+ "8210": 946704320.0,
+ "8215": 982718592.0,
+ "8220": 988075776.0,
+ "8225": 966563072.0,
+ "8230": 962898240.0,
+ "8235": 933923072.0,
+ "8240": 980574848.0,
+ "8245": 976655872.0,
+ "8250": 964328128.0,
+ "8255": 977591424.0,
+ "8260": 957202304.0,
+ "8265": 982956800.0,
+ "8270": 953142528.0,
+ "8275": 974529792.0,
+ "8280": 974647808.0,
+ "8285": 953322048.0,
+ "8290": 939886464.0,
+ "8295": 981518720.0,
+ "8300": 973431872.0,
+ "8305": 978196352.0,
+ "8310": 951561920.0,
+ "8315": 938326912.0,
+ "8320": 977595072.0,
+ "8325": 968155456.0,
+ "8330": 990191552.0,
+ "8335": 976103488.0,
+ "8340": 947710464.0,
+ "8345": 971095872.0,
+ "8350": 970304128.0,
+ "8355": 975432832.0,
+ "8360": 979810560.0,
+ "8365": 932929344.0,
+ "8370": 965587200.0,
+ "8375": 980236416.0,
+ "8380": 965580416.0,
+ "8385": 973086464.0,
+ "8390": 962870912.0,
+ "8395": 951507392.0,
+ "8400": 972631168.0,
+ "8405": 951465728.0,
+ "8410": 960777792.0,
+ "8415": 965439360.0,
+ "8420": 942011776.0,
+ "8425": 968658048.0,
+ "8430": 960766080.0,
+ "8435": 966214144.0,
+ "8440": 970051712.0,
+ "8445": 952774912.0,
+ "8450": 984547584.0,
+ "8455": 990665216.0,
+ "8460": 968987456.0,
+ "8465": 967294720.0,
+ "8470": 963132736.0,
+ "8475": 942846592.0,
+ "8480": 987200512.0,
+ "8485": 979788480.0,
+ "8490": 992367168.0,
+ "8495": 971569856.0,
+ "8500": 951234176.0,
+ "8505": 983252608.0,
+ "8510": 974223552.0,
+ "8515": 968555328.0,
+ "8520": 962099200.0,
+ "8525": 945303040.0,
+ "8530": 984622528.0,
+ "8535": 978460928.0,
+ "8540": 967298304.0,
+ "8545": 968855680.0,
+ "8550": 942065152.0,
+ "8555": 972102656.0,
+ "8560": 958762624.0,
+ "8565": 976119360.0,
+ "8570": 975011264.0,
+ "8575": 971620608.0,
+ "8580": 932116864.0,
+ "8585": 966041920.0,
+ "8590": 979227200.0,
+ "8595": 978896640.0,
+ "8600": 984223104.0,
+ "8605": 957447552.0,
+ "8610": 983899008.0,
+ "8615": 977896448.0,
+ "8620": 962810816.0,
+ "8625": 979565440.0,
+ "8630": 943517952.0,
+ "8635": 961760384.0,
+ "8640": 973499008.0,
+ "8645": 970415616.0,
+ "8650": 969116672.0,
+ "8655": 971144000.0,
+ "8660": 944569088.0,
+ "8665": 987124736.0,
+ "8670": 960697792.0,
+ "8675": 974081792.0,
+ "8680": 962718080.0,
+ "8685": 955789568.0,
+ "8690": 978378368.0,
+ "8695": 968400640.0,
+ "8700": 972974720.0,
+ "8705": 974037760.0,
+ "8710": 947079872.0,
+ "8715": 973847232.0,
+ "8720": 958228864.0,
+ "8725": 979063488.0,
+ "8730": 985832448.0,
+ "8735": 951944576.0,
+ "8740": 940816000.0,
+ "8745": 987874240.0,
+ "8750": 972060672.0,
+ "8755": 971515904.0,
+ "8760": 965372928.0,
+ "8765": 934839296.0,
+ "8770": 986313344.0,
+ "8775": 970021952.0,
+ "8780": 967712704.0,
+ "8785": 962283008.0,
+ "8790": 947736832.0,
+ "8795": 968976000.0,
+ "8800": 970581504.0,
+ "8805": 973301376.0,
+ "8810": 983639104.0,
+ "8815": 951717120.0,
+ "8820": 939956864.0,
+ "8825": 964321920.0,
+ "8830": 981076992.0,
+ "8835": 971459840.0,
+ "8840": 979418368.0,
+ "8845": 951370560.0,
+ "8850": 986478080.0,
+ "8855": 971028992.0,
+ "8860": 961827264.0,
+ "8865": 957370624.0,
+ "8870": 945910528.0,
+ "8875": 968308736.0,
+ "8880": 983829568.0,
+ "8885": 971367680.0,
+ "8890": 969993344.0,
+ "8895": 952713792.0,
+ "8900": 961952256.0,
+ "8905": 976987136.0,
+ "8910": 981570048.0,
+ "8915": 980806208.0,
+ "8920": 967926528.0,
+ "8925": 939821440.0,
+ "8930": 970123264.0,
+ "8935": 963770368.0,
+ "8940": 977559552.0,
+ "8945": 981912960.0,
+ "8950": 945599360.0,
+ "8955": 972731008.0,
+ "8960": 973885056.0,
+ "8965": 973850112.0,
+ "8970": 966361728.0,
+ "8975": 937235264.0,
+ "8980": 952251776.0,
+ "8985": 978014016.0,
+ "8990": 967350656.0,
+ "8995": 980370176.0,
+ "9000": 952395840.0,
+ "9005": 950832768.0,
+ "9010": 975267712.0,
+ "9015": 982975232.0,
+ "9020": 959192448.0,
+ "9025": 978796352.0,
+ "9030": 953629376.0,
+ "9035": 968787200.0,
+ "9040": 978294400.0,
+ "9045": 968522496.0,
+ "9050": 983036480.0,
+ "9055": 948155392.0,
+ "9060": 956478848.0,
+ "9065": 969792000.0,
+ "9070": 967683456.0,
+ "9075": 980881152.0,
+ "9080": 952258304.0,
+ "9085": 971295232.0,
+ "9090": 963557184.0,
+ "9095": 968166016.0,
+ "9100": 974511104.0,
+ "9105": 960145344.0,
+ "9110": 948329728.0,
+ "9115": 956402816.0,
+ "9120": 985494912.0,
+ "9125": 963296384.0,
+ "9130": 958346432.0,
+ "9135": 951620288.0,
+ "9140": 967284224.0,
+ "9145": 976844544.0,
+ "9150": 987023232.0,
+ "9155": 976968704.0,
+ "9160": 957694208.0,
+ "9165": 950415872.0,
+ "9170": 988382016.0,
+ "9175": 971692480.0,
+ "9180": 967374912.0,
+ "9185": 955127040.0,
+ "9190": 956710656.0,
+ "9195": 965721472.0,
+ "9200": 968597760.0,
+ "9205": 967291904.0,
+ "9210": 984681216.0,
+ "9215": 931374400.0,
+ "9220": 949709440.0,
+ "9225": 971070592.0,
+ "9230": 971195264.0,
+ "9235": 971746176.0,
+ "9240": 959830400.0,
+ "9245": 964026752.0,
+ "9250": 961097344.0,
+ "9255": 983241472.0,
+ "9260": 979074048.0,
+ "9265": 953027136.0,
+ "9270": 949216512.0,
+ "9275": 978441152.0,
+ "9280": 977570688.0,
+ "9285": 962618944.0,
+ "9290": 979501440.0,
+ "9295": 958696192.0,
+ "9300": 965544960.0,
+ "9305": 969071744.0,
+ "9310": 973001216.0,
+ "9315": 976390912.0,
+ "9320": 948280320.0,
+ "9325": 979885568.0,
+ "9330": 978062592.0,
+ "9335": 975746688.0,
+ "9340": 960410432.0,
+ "9345": 943464448.0,
+ "9350": 952934016.0,
+ "9355": 963355520.0,
+ "9360": 960499136.0,
+ "9365": 983692608.0,
+ "9370": 982297984.0,
+ "9375": 942116224.0,
+ "9380": 982577024.0,
+ "9385": 985393536.0,
+ "9390": 973418240.0,
+ "9395": 977974528.0,
+ "9400": 938173312.0,
+ "9405": 968168192.0,
+ "9410": 981667648.0,
+ "9415": 991883392.0,
+ "9420": 960392064.0,
+ "9425": 956951872.0,
+ "9430": 939298432.0,
+ "9435": 974398784.0,
+ "9440": 959543808.0,
+ "9445": 973733120.0,
+ "9450": 961915008.0,
+ "9455": 946070656.0,
+ "9460": 978455360.0,
+ "9465": 988236032.0,
+ "9470": 963308672.0,
+ "9475": 983797504.0,
+ "9480": 931000960.0,
+ "9485": 987539456.0,
+ "9490": 963408832.0,
+ "9495": 972525184.0,
+ "9500": 982379136.0,
+ "9505": 970260160.0,
+ "9510": 964553472.0,
+ "9515": 957060224.0,
+ "9520": 948433984.0,
+ "9525": 965556608.0,
+ "9530": 958432128.0,
+ "9535": 950951168.0,
+ "9540": 954255872.0,
+ "9545": 979665920.0,
+ "9550": 956061696.0,
+ "9555": 953442368.0,
+ "9560": 958456448.0,
+ "9565": 970137984.0,
+ "9570": 977209856.0,
+ "9575": 958959744.0,
+ "9580": 963097600.0,
+ "9585": 946270400.0,
+ "9590": 948540928.0,
+ "9595": 966947008.0,
+ "9600": 984798208.0,
+ "9605": 985302016.0,
+ "9610": 943246592.0,
+ "9615": 952814016.0,
+ "9620": 980981696.0,
+ "9625": 978266112.0,
+ "9630": 969978496.0,
+ "9635": 974721536.0,
+ "9640": 940294272.0,
+ "9645": 961910080.0,
+ "9650": 970943744.0,
+ "9655": 987569792.0,
+ "9660": 963184320.0,
+ "9665": 950168192.0,
+ "9670": 965781632.0,
+ "9675": 963111936.0,
+ "9680": 964954816.0,
+ "9685": 986709120.0,
+ "9690": 940607744.0,
+ "9695": 950447616.0,
+ "9700": 975738816.0,
+ "9705": 972589056.0,
+ "9710": 967656704.0,
+ "9715": 971132416.0,
+ "9720": 940416640.0,
+ "9725": 965998080.0,
+ "9730": 973718784.0,
+ "9735": 974172928.0,
+ "9740": 971431104.0,
+ "9745": 950998912.0,
+ "9750": 979592064.0,
+ "9755": 970403840.0,
+ "9760": 968064640.0,
+ "9765": 963786880.0,
+ "9770": 952209408.0,
+ "9775": 956731136.0,
+ "9780": 970107776.0,
+ "9785": 958344448.0,
+ "9790": 961042816.0,
+ "9795": 958213248.0,
+ "9800": 949058560.0,
+ "9805": 962196992.0,
+ "9810": 978331520.0,
+ "9815": 977753600.0,
+ "9820": 982361216.0,
+ "9825": 939473536.0,
+ "9830": 969626048.0,
+ "9835": 972510208.0,
+ "9840": 971687936.0,
+ "9845": 967061760.0,
+ "9850": 946544128.0,
+ "9855": 957252992.0,
+ "9860": 987359744.0,
+ "9865": 970127168.0,
+ "9870": 989524736.0,
+ "9875": 957073408.0,
+ "9880": 930847232.0,
+ "9885": 963629696.0,
+ "9890": 972392448.0,
+ "9895": 983574528.0,
+ "9900": 956631744.0,
+ "9905": 939009728.0,
+ "9910": 978860288.0,
+ "9915": 973504960.0,
+ "9920": 944413952.0,
+ "9925": 962928512.0,
+ "9930": 947381120.0,
+ "9935": 960634880.0,
+ "9940": 966066752.0,
+ "9945": 958494784.0,
+ "9950": 964043072.0,
+ "9955": 943413824.0,
+ "9960": 967004224.0,
+ "9965": 983596800.0,
+ "9970": 966349184.0,
+ "9975": 963217664.0,
+ "9980": 980808320.0,
+ "9985": 941910144.0,
+ "9990": 976343936.0,
+ "9995": 982828672.0,
+ "10000": 972023552.0,
+ "10005": 969480960.0,
+ "10010": 944330624.0,
+ "10015": 983276288.0,
+ "10020": 978455552.0,
+ "10025": 979844096.0,
+ "10030": 971601280.0,
+ "10035": 946594944.0,
+ "10040": 950559168.0,
+ "10045": 978050944.0,
+ "10050": 985832832.0,
+ "10055": 990444288.0,
+ "10060": 958898048.0,
+ "10065": 947517056.0,
+ "10070": 967166656.0,
+ "10075": 979106432.0,
+ "10080": 971621632.0,
+ "10085": 974302720.0,
+ "10090": 944020544.0,
+ "10095": 962781696.0,
+ "10100": 972061120.0,
+ "10105": 975506304.0,
+ "10110": 971932352.0,
+ "10115": 948612096.0,
+ "10120": 962324864.0,
+ "10125": 974035904.0,
+ "10130": 980282240.0,
+ "10135": 972211584.0,
+ "10140": 957941568.0,
+ "10145": 933985280.0,
+ "10150": 973634752.0,
+ "10155": 969765824.0,
+ "10160": 962060800.0,
+ "10165": 974346112.0,
+ "10170": 944527424.0,
+ "10175": 978734144.0,
+ "10180": 983909952.0,
+ "10185": 978637312.0,
+ "10190": 955692928.0,
+ "10195": 936523904.0,
+ "10200": 988193152.0
+ }
+ },
+ "mem-allocated-bytes": {
+ "start_step": 1,
+ "end_step": 10200,
+ "step_interval": 5,
+ "values": {
+ "1": 13270018048.0,
+ "5": 13270018048.0,
+ "10": 13270018048.0,
+ "15": 13270018048.0,
+ "20": 13270018048.0,
+ "25": 13270018048.0,
+ "30": 13270018048.0,
+ "35": 13270018048.0,
+ "40": 13270018048.0,
+ "45": 13270018048.0,
+ "50": 13270018048.0,
+ "55": 13270018048.0,
+ "60": 13270018048.0,
+ "65": 13270018048.0,
+ "70": 13270018048.0,
+ "75": 13270018048.0,
+ "80": 13270018048.0,
+ "85": 13270018048.0,
+ "90": 13270018048.0,
+ "95": 13270018048.0,
+ "100": 13270018048.0,
+ "105": 13270018048.0,
+ "110": 13270018048.0,
+ "115": 13270018048.0,
+ "120": 13270018048.0,
+ "125": 13270018048.0,
+ "130": 13270018048.0,
+ "135": 13270018048.0,
+ "140": 13270018048.0,
+ "145": 13270018048.0,
+ "150": 13270018048.0,
+ "155": 13270018048.0,
+ "160": 13270018048.0,
+ "165": 13270018048.0,
+ "170": 13270018048.0,
+ "175": 13270018048.0,
+ "180": 13270018048.0,
+ "185": 13270018048.0,
+ "190": 13270018048.0,
+ "195": 13270018048.0,
+ "200": 13270018048.0,
+ "205": 13270018048.0,
+ "210": 13270018048.0,
+ "215": 13270018048.0,
+ "220": 13270018048.0,
+ "225": 13270018048.0,
+ "230": 13270018048.0,
+ "235": 13270018048.0,
+ "240": 13270018048.0,
+ "245": 13270018048.0,
+ "250": 13270018048.0,
+ "255": 13270018048.0,
+ "260": 13270018048.0,
+ "265": 13270018048.0,
+ "270": 13270018048.0,
+ "275": 13270018048.0,
+ "280": 13270018048.0,
+ "285": 13270018048.0,
+ "290": 13270018048.0,
+ "295": 13270018048.0,
+ "300": 13270018048.0,
+ "305": 13270018048.0,
+ "310": 13270018048.0,
+ "315": 13270018048.0,
+ "320": 13270018048.0,
+ "325": 13270018048.0,
+ "330": 13270018048.0,
+ "335": 13270018048.0,
+ "340": 13270018048.0,
+ "345": 13270018048.0,
+ "350": 13270018048.0,
+ "355": 13270018048.0,
+ "360": 13270018048.0,
+ "365": 13270018048.0,
+ "370": 13270018048.0,
+ "375": 13270018048.0,
+ "380": 13270018048.0,
+ "385": 13270018048.0,
+ "390": 13270018048.0,
+ "395": 13270018048.0,
+ "400": 13270018048.0,
+ "405": 13270018048.0,
+ "410": 13270018048.0,
+ "415": 13270018048.0,
+ "420": 13270018048.0,
+ "425": 13270018048.0,
+ "430": 13270018048.0,
+ "435": 13270018048.0,
+ "440": 13270018048.0,
+ "445": 13270018048.0,
+ "450": 13270018048.0,
+ "455": 13270018048.0,
+ "460": 13270018048.0,
+ "465": 13270018048.0,
+ "470": 13270018048.0,
+ "475": 13270018048.0,
+ "480": 13270018048.0,
+ "485": 13270018048.0,
+ "490": 13270018048.0,
+ "495": 13270018048.0,
+ "500": 13270018048.0,
+ "505": 13270018048.0,
+ "510": 13270018048.0,
+ "515": 13270018048.0,
+ "520": 13270018048.0,
+ "525": 13270018048.0,
+ "530": 13270018048.0,
+ "535": 13270018048.0,
+ "540": 13270018048.0,
+ "545": 13270018048.0,
+ "550": 13270018048.0,
+ "555": 13270018048.0,
+ "560": 13270018048.0,
+ "565": 13270018048.0,
+ "570": 13270018048.0,
+ "575": 13270018048.0,
+ "580": 13270018048.0,
+ "585": 13270018048.0,
+ "590": 13270018048.0,
+ "595": 13270018048.0,
+ "600": 13270018048.0,
+ "605": 13270018048.0,
+ "610": 13270018048.0,
+ "615": 13270018048.0,
+ "620": 13270018048.0,
+ "625": 13270018048.0,
+ "630": 13270018048.0,
+ "635": 13270018048.0,
+ "640": 13270018048.0,
+ "645": 13270018048.0,
+ "650": 13270018048.0,
+ "655": 13270018048.0,
+ "660": 13270018048.0,
+ "665": 13270018048.0,
+ "670": 13270018048.0,
+ "675": 13270018048.0,
+ "680": 13270018048.0,
+ "685": 13270018048.0,
+ "690": 13270018048.0,
+ "695": 13270018048.0,
+ "700": 13270018048.0,
+ "705": 13270018048.0,
+ "710": 13270018048.0,
+ "715": 13270018048.0,
+ "720": 13270018048.0,
+ "725": 13270018048.0,
+ "730": 13270018048.0,
+ "735": 13270018048.0,
+ "740": 13270018048.0,
+ "745": 13270018048.0,
+ "750": 13270018048.0,
+ "755": 13270018048.0,
+ "760": 13270018048.0,
+ "765": 13270018048.0,
+ "770": 13270018048.0,
+ "775": 13270018048.0,
+ "780": 13270018048.0,
+ "785": 13270018048.0,
+ "790": 13270018048.0,
+ "795": 13270018048.0,
+ "800": 13270018048.0,
+ "805": 13270018048.0,
+ "810": 13270018048.0,
+ "815": 13270018048.0,
+ "820": 13270018048.0,
+ "825": 13270018048.0,
+ "830": 13270018048.0,
+ "835": 13270018048.0,
+ "840": 13270018048.0,
+ "845": 13270018048.0,
+ "850": 13270018048.0,
+ "855": 13270018048.0,
+ "860": 13270018048.0,
+ "865": 13270018048.0,
+ "870": 13270018048.0,
+ "875": 13270018048.0,
+ "880": 13270018048.0,
+ "885": 13270018048.0,
+ "890": 13270018048.0,
+ "895": 13270018048.0,
+ "900": 13270018048.0,
+ "905": 13270018048.0,
+ "910": 13270018048.0,
+ "915": 13270018048.0,
+ "920": 13270018048.0,
+ "925": 13270018048.0,
+ "930": 13270018048.0,
+ "935": 13270018048.0,
+ "940": 13270018048.0,
+ "945": 13270018048.0,
+ "950": 13270018048.0,
+ "955": 13270018048.0,
+ "960": 13270018048.0,
+ "965": 13270018048.0,
+ "970": 13270018048.0,
+ "975": 13270018048.0,
+ "980": 13270018048.0,
+ "985": 13270018048.0,
+ "990": 13270018048.0,
+ "995": 13270018048.0,
+ "1000": 13270018048.0,
+ "1005": 13270018048.0,
+ "1010": 13270018048.0,
+ "1015": 13270018048.0,
+ "1020": 13270018048.0,
+ "1025": 13270018048.0,
+ "1030": 13270018048.0,
+ "1035": 13270018048.0,
+ "1040": 13270018048.0,
+ "1045": 13270018048.0,
+ "1050": 13270018048.0,
+ "1055": 13270018048.0,
+ "1060": 13270018048.0,
+ "1065": 13270018048.0,
+ "1070": 13270018048.0,
+ "1075": 13270018048.0,
+ "1080": 13270018048.0,
+ "1085": 13270018048.0,
+ "1090": 13270018048.0,
+ "1095": 13270018048.0,
+ "1100": 13270018048.0,
+ "1105": 13270018048.0,
+ "1110": 13270018048.0,
+ "1115": 13270018048.0,
+ "1120": 13270018048.0,
+ "1125": 13270018048.0,
+ "1130": 13270018048.0,
+ "1135": 13270018048.0,
+ "1140": 13270018048.0,
+ "1145": 13270018048.0,
+ "1150": 13270018048.0,
+ "1155": 13270018048.0,
+ "1160": 13270018048.0,
+ "1165": 13270018048.0,
+ "1170": 13270018048.0,
+ "1175": 13270018048.0,
+ "1180": 13270018048.0,
+ "1185": 13270018048.0,
+ "1190": 13270018048.0,
+ "1195": 13270018048.0,
+ "1200": 13270018048.0,
+ "1205": 13270018048.0,
+ "1210": 13270018048.0,
+ "1215": 13270018048.0,
+ "1220": 13270018048.0,
+ "1225": 13270018048.0,
+ "1230": 13270018048.0,
+ "1235": 13270018048.0,
+ "1240": 13270018048.0,
+ "1245": 13270018048.0,
+ "1250": 13270018048.0,
+ "1255": 13270018048.0,
+ "1260": 13270018048.0,
+ "1265": 13270018048.0,
+ "1270": 13270018048.0,
+ "1275": 13270018048.0,
+ "1280": 13270018048.0,
+ "1285": 13270018048.0,
+ "1290": 13270018048.0,
+ "1295": 13270018048.0,
+ "1300": 13270018048.0,
+ "1305": 13270018048.0,
+ "1310": 13270018048.0,
+ "1315": 13270018048.0,
+ "1320": 13270018048.0,
+ "1325": 13270018048.0,
+ "1330": 13270018048.0,
+ "1335": 13270018048.0,
+ "1340": 13270018048.0,
+ "1345": 13270018048.0,
+ "1350": 13270018048.0,
+ "1355": 13270018048.0,
+ "1360": 13270018048.0,
+ "1365": 13270018048.0,
+ "1370": 13270018048.0,
+ "1375": 13270018048.0,
+ "1380": 13270018048.0,
+ "1385": 13270018048.0,
+ "1390": 13270018048.0,
+ "1395": 13270018048.0,
+ "1400": 13270018048.0,
+ "1405": 13270018048.0,
+ "1410": 13270018048.0,
+ "1415": 13270018048.0,
+ "1420": 13270018048.0,
+ "1425": 13270018048.0,
+ "1430": 13270018048.0,
+ "1435": 13270018048.0,
+ "1440": 13270018048.0,
+ "1445": 13270018048.0,
+ "1450": 13270018048.0,
+ "1455": 13270018048.0,
+ "1460": 13270018048.0,
+ "1465": 13270018048.0,
+ "1470": 13270018048.0,
+ "1475": 13270018048.0,
+ "1480": 13270018048.0,
+ "1485": 13270018048.0,
+ "1490": 13270018048.0,
+ "1495": 13270018048.0,
+ "1500": 13270018048.0,
+ "1505": 13270018048.0,
+ "1510": 13270018048.0,
+ "1515": 13270018048.0,
+ "1520": 13270018048.0,
+ "1525": 13270018048.0,
+ "1530": 13270018048.0,
+ "1535": 13270018048.0,
+ "1540": 13270018048.0,
+ "1545": 13270018048.0,
+ "1550": 13270018048.0,
+ "1555": 13270018048.0,
+ "1560": 13270018048.0,
+ "1565": 13270018048.0,
+ "1570": 13270018048.0,
+ "1575": 13270018048.0,
+ "1580": 13270018048.0,
+ "1585": 13270018048.0,
+ "1590": 13270018048.0,
+ "1595": 13270018048.0,
+ "1600": 13270018048.0,
+ "1605": 13270018048.0,
+ "1610": 13270018048.0,
+ "1615": 13270018048.0,
+ "1620": 13270018048.0,
+ "1625": 13270018048.0,
+ "1630": 13270018048.0,
+ "1635": 13270018048.0,
+ "1640": 13270018048.0,
+ "1645": 13270018048.0,
+ "1650": 13270018048.0,
+ "1655": 13270018048.0,
+ "1660": 13270018048.0,
+ "1665": 13270018048.0,
+ "1670": 13270018048.0,
+ "1675": 13270018048.0,
+ "1680": 13270018048.0,
+ "1685": 13270018048.0,
+ "1690": 13270018048.0,
+ "1695": 13270018048.0,
+ "1700": 13270018048.0,
+ "1705": 13270018048.0,
+ "1710": 13270018048.0,
+ "1715": 13270018048.0,
+ "1720": 13270018048.0,
+ "1725": 13270018048.0,
+ "1730": 13270018048.0,
+ "1735": 13270018048.0,
+ "1740": 13270018048.0,
+ "1745": 13270018048.0,
+ "1750": 13270018048.0,
+ "1755": 13270018048.0,
+ "1760": 13270018048.0,
+ "1765": 13270018048.0,
+ "1770": 13270018048.0,
+ "1775": 13270018048.0,
+ "1780": 13270018048.0,
+ "1785": 13270018048.0,
+ "1790": 13270018048.0,
+ "1795": 13270018048.0,
+ "1800": 13270018048.0,
+ "1805": 13270018048.0,
+ "1810": 13270018048.0,
+ "1815": 13270018048.0,
+ "1820": 13270018048.0,
+ "1825": 13270018048.0,
+ "1830": 13270018048.0,
+ "1835": 13270018048.0,
+ "1840": 13270018048.0,
+ "1845": 13270018048.0,
+ "1850": 13270018048.0,
+ "1855": 13270018048.0,
+ "1860": 13270018048.0,
+ "1865": 13270018048.0,
+ "1870": 13270018048.0,
+ "1875": 13270018048.0,
+ "1880": 13270018048.0,
+ "1885": 13270018048.0,
+ "1890": 13270018048.0,
+ "1895": 13270018048.0,
+ "1900": 13270018048.0,
+ "1905": 13270018048.0,
+ "1910": 13270018048.0,
+ "1915": 13270018048.0,
+ "1920": 13270018048.0,
+ "1925": 13270018048.0,
+ "1930": 13270018048.0,
+ "1935": 13270018048.0,
+ "1940": 13270018048.0,
+ "1945": 13270018048.0,
+ "1950": 13270018048.0,
+ "1955": 13270018048.0,
+ "1960": 13270018048.0,
+ "1965": 13270018048.0,
+ "1970": 13270018048.0,
+ "1975": 13270018048.0,
+ "1980": 13270018048.0,
+ "1985": 13270018048.0,
+ "1990": 13270018048.0,
+ "1995": 13270018048.0,
+ "2000": 13270018048.0,
+ "2005": 13270018048.0,
+ "2010": 13270018048.0,
+ "2015": 13270018048.0,
+ "2020": 13270018048.0,
+ "2025": 13270018048.0,
+ "2030": 13270018048.0,
+ "2035": 13270018048.0,
+ "2040": 13270018048.0,
+ "2045": 13270018048.0,
+ "2050": 13270018048.0,
+ "2055": 13270018048.0,
+ "2060": 13270018048.0,
+ "2065": 13270018048.0,
+ "2070": 13270018048.0,
+ "2075": 13270018048.0,
+ "2080": 13270018048.0,
+ "2085": 13270018048.0,
+ "2090": 13270018048.0,
+ "2095": 13270018048.0,
+ "2100": 13270018048.0,
+ "2105": 13270018048.0,
+ "2110": 13270018048.0,
+ "2115": 13270018048.0,
+ "2120": 13270018048.0,
+ "2125": 13270018048.0,
+ "2130": 13270018048.0,
+ "2135": 13270018048.0,
+ "2140": 13270018048.0,
+ "2145": 13270018048.0,
+ "2150": 13270018048.0,
+ "2155": 13270018048.0,
+ "2160": 13270018048.0,
+ "2165": 13270018048.0,
+ "2170": 13270018048.0,
+ "2175": 13270018048.0,
+ "2180": 13270018048.0,
+ "2185": 13270018048.0,
+ "2190": 13270018048.0,
+ "2195": 13270018048.0,
+ "2200": 13270018048.0,
+ "2205": 13270018048.0,
+ "2210": 13270018048.0,
+ "2215": 13270018048.0,
+ "2220": 13270018048.0,
+ "2225": 13270018048.0,
+ "2230": 13270018048.0,
+ "2235": 13270018048.0,
+ "2240": 13270018048.0,
+ "2245": 13270018048.0,
+ "2250": 13270018048.0,
+ "2255": 13270018048.0,
+ "2260": 13270018048.0,
+ "2265": 13270018048.0,
+ "2270": 13270018048.0,
+ "2275": 13270018048.0,
+ "2280": 13270018048.0,
+ "2285": 13270018048.0,
+ "2290": 13270018048.0,
+ "2295": 13270018048.0,
+ "2300": 13270018048.0,
+ "2305": 13270018048.0,
+ "2310": 13270018048.0,
+ "2315": 13270018048.0,
+ "2320": 13270018048.0,
+ "2325": 13270018048.0,
+ "2330": 13270018048.0,
+ "2335": 13270018048.0,
+ "2340": 13270018048.0,
+ "2345": 13270018048.0,
+ "2350": 13270018048.0,
+ "2355": 13270018048.0,
+ "2360": 13270018048.0,
+ "2365": 13270018048.0,
+ "2370": 13270018048.0,
+ "2375": 13270018048.0,
+ "2380": 13270018048.0,
+ "2385": 13270018048.0,
+ "2390": 13270018048.0,
+ "2395": 13270018048.0,
+ "2400": 13270018048.0,
+ "2405": 13270018048.0,
+ "2410": 13270018048.0,
+ "2415": 13270018048.0,
+ "2420": 13270018048.0,
+ "2425": 13270018048.0,
+ "2430": 13270018048.0,
+ "2435": 13270018048.0,
+ "2440": 13270018048.0,
+ "2445": 13270018048.0,
+ "2450": 13270018048.0,
+ "2455": 13270018048.0,
+ "2460": 13270018048.0,
+ "2465": 13270018048.0,
+ "2470": 13270018048.0,
+ "2475": 13270018048.0,
+ "2480": 13270018048.0,
+ "2485": 13270018048.0,
+ "2490": 13270018048.0,
+ "2495": 13270018048.0,
+ "2500": 13270018048.0,
+ "2505": 13270018048.0,
+ "2510": 13270018048.0,
+ "2515": 13270018048.0,
+ "2520": 13270018048.0,
+ "2525": 13270018048.0,
+ "2530": 13270018048.0,
+ "2535": 13270018048.0,
+ "2540": 13270018048.0,
+ "2545": 13270018048.0,
+ "2550": 13270018048.0,
+ "2555": 13270018048.0,
+ "2560": 13270018048.0,
+ "2565": 13270018048.0,
+ "2570": 13270018048.0,
+ "2575": 13270018048.0,
+ "2580": 13270018048.0,
+ "2585": 13270018048.0,
+ "2590": 13270018048.0,
+ "2595": 13270018048.0,
+ "2600": 13270018048.0,
+ "2605": 13270018048.0,
+ "2610": 13270018048.0,
+ "2615": 13270018048.0,
+ "2620": 13270018048.0,
+ "2625": 13270018048.0,
+ "2630": 13270018048.0,
+ "2635": 13270018048.0,
+ "2640": 13270018048.0,
+ "2645": 13270018048.0,
+ "2650": 13270018048.0,
+ "2655": 13270018048.0,
+ "2660": 13270018048.0,
+ "2665": 13270018048.0,
+ "2670": 13270018048.0,
+ "2675": 13270018048.0,
+ "2680": 13270018048.0,
+ "2685": 13270018048.0,
+ "2690": 13270018048.0,
+ "2695": 13270018048.0,
+ "2700": 13270018048.0,
+ "2705": 13270018048.0,
+ "2710": 13270018048.0,
+ "2715": 13270018048.0,
+ "2720": 13270018048.0,
+ "2725": 13270018048.0,
+ "2730": 13270018048.0,
+ "2735": 13270018048.0,
+ "2740": 13270018048.0,
+ "2745": 13270018048.0,
+ "2750": 13270018048.0,
+ "2755": 13270018048.0,
+ "2760": 13270018048.0,
+ "2765": 13270018048.0,
+ "2770": 13270018048.0,
+ "2775": 13270018048.0,
+ "2780": 13270018048.0,
+ "2785": 13270018048.0,
+ "2790": 13270018048.0,
+ "2795": 13270018048.0,
+ "2800": 13270018048.0,
+ "2805": 13270018048.0,
+ "2810": 13270018048.0,
+ "2815": 13270018048.0,
+ "2820": 13270018048.0,
+ "2825": 13270018048.0,
+ "2830": 13270018048.0,
+ "2835": 13270018048.0,
+ "2840": 13270018048.0,
+ "2845": 13270018048.0,
+ "2850": 13270018048.0,
+ "2855": 13270018048.0,
+ "2860": 13270018048.0,
+ "2865": 13270018048.0,
+ "2870": 13270018048.0,
+ "2875": 13270018048.0,
+ "2880": 13270018048.0,
+ "2885": 13270018048.0,
+ "2890": 13270018048.0,
+ "2895": 13270018048.0,
+ "2900": 13270018048.0,
+ "2905": 13270018048.0,
+ "2910": 13270018048.0,
+ "2915": 13270018048.0,
+ "2920": 13270018048.0,
+ "2925": 13270018048.0,
+ "2930": 13270018048.0,
+ "2935": 13270018048.0,
+ "2940": 13270018048.0,
+ "2945": 13270018048.0,
+ "2950": 13270018048.0,
+ "2955": 13270018048.0,
+ "2960": 13270018048.0,
+ "2965": 13270018048.0,
+ "2970": 13270018048.0,
+ "2975": 13270018048.0,
+ "2980": 13270018048.0,
+ "2985": 13270018048.0,
+ "2990": 13270018048.0,
+ "2995": 13270018048.0,
+ "3000": 13270018048.0,
+ "3005": 13270018048.0,
+ "3010": 13270018048.0,
+ "3015": 13270018048.0,
+ "3020": 13270018048.0,
+ "3025": 13270018048.0,
+ "3030": 13270018048.0,
+ "3035": 13270018048.0,
+ "3040": 13270018048.0,
+ "3045": 13270018048.0,
+ "3050": 13270018048.0,
+ "3055": 13270018048.0,
+ "3060": 13270018048.0,
+ "3065": 13270018048.0,
+ "3070": 13270018048.0,
+ "3075": 13270018048.0,
+ "3080": 13270018048.0,
+ "3085": 13270018048.0,
+ "3090": 13270018048.0,
+ "3095": 13270018048.0,
+ "3100": 13270018048.0,
+ "3105": 13270018048.0,
+ "3110": 13270018048.0,
+ "3115": 13270018048.0,
+ "3120": 13270018048.0,
+ "3125": 13270018048.0,
+ "3130": 13270018048.0,
+ "3135": 13270018048.0,
+ "3140": 13270018048.0,
+ "3145": 13270018048.0,
+ "3150": 13270018048.0,
+ "3155": 13270018048.0,
+ "3160": 13270018048.0,
+ "3165": 13270018048.0,
+ "3170": 13270018048.0,
+ "3175": 13270018048.0,
+ "3180": 13270018048.0,
+ "3185": 13270018048.0,
+ "3190": 13270018048.0,
+ "3195": 13270018048.0,
+ "3200": 13270018048.0,
+ "3205": 13270018048.0,
+ "3210": 13270018048.0,
+ "3215": 13270018048.0,
+ "3220": 13270018048.0,
+ "3225": 13270018048.0,
+ "3230": 13270018048.0,
+ "3235": 13270018048.0,
+ "3240": 13270018048.0,
+ "3245": 13270018048.0,
+ "3250": 13270018048.0,
+ "3255": 13270018048.0,
+ "3260": 13270018048.0,
+ "3265": 13270018048.0,
+ "3270": 13270018048.0,
+ "3275": 13270018048.0,
+ "3280": 13270018048.0,
+ "3285": 13270018048.0,
+ "3290": 13270018048.0,
+ "3295": 13270018048.0,
+ "3300": 13270018048.0,
+ "3305": 13270018048.0,
+ "3310": 13270018048.0,
+ "3315": 13270018048.0,
+ "3320": 13270018048.0,
+ "3325": 13270018048.0,
+ "3330": 13270018048.0,
+ "3335": 13270018048.0,
+ "3340": 13270018048.0,
+ "3345": 13270018048.0,
+ "3350": 13270018048.0,
+ "3355": 13270018048.0,
+ "3360": 13270018048.0,
+ "3365": 13270018048.0,
+ "3370": 13270018048.0,
+ "3375": 13270018048.0,
+ "3380": 13270018048.0,
+ "3385": 13270018048.0,
+ "3390": 13270018048.0,
+ "3395": 13270018048.0,
+ "3400": 13270018048.0,
+ "3405": 13270018048.0,
+ "3410": 13270018048.0,
+ "3415": 13270018048.0,
+ "3420": 13270018048.0,
+ "3425": 13270018048.0,
+ "3430": 13270018048.0,
+ "3435": 13270018048.0,
+ "3440": 13270018048.0,
+ "3445": 13270018048.0,
+ "3450": 13270018048.0,
+ "3455": 13270018048.0,
+ "3460": 13270018048.0,
+ "3465": 13270018048.0,
+ "3470": 13270018048.0,
+ "3475": 13270018048.0,
+ "3480": 13270018048.0,
+ "3485": 13270018048.0,
+ "3490": 13270018048.0,
+ "3495": 13270018048.0,
+ "3500": 13270018048.0,
+ "3505": 13270018048.0,
+ "3510": 13270018048.0,
+ "3515": 13270018048.0,
+ "3520": 13270018048.0,
+ "3525": 13270018048.0,
+ "3530": 13270018048.0,
+ "3535": 13270018048.0,
+ "3540": 13270018048.0,
+ "3545": 13270018048.0,
+ "3550": 13270018048.0,
+ "3555": 13270018048.0,
+ "3560": 13270018048.0,
+ "3565": 13270018048.0,
+ "3570": 13270018048.0,
+ "3575": 13270018048.0,
+ "3580": 13270018048.0,
+ "3585": 13270018048.0,
+ "3590": 13270018048.0,
+ "3595": 13270018048.0,
+ "3600": 13270018048.0,
+ "3605": 13270018048.0,
+ "3610": 13270018048.0,
+ "3615": 13270018048.0,
+ "3620": 13270018048.0,
+ "3625": 13270018048.0,
+ "3630": 13270018048.0,
+ "3635": 13270018048.0,
+ "3640": 13270018048.0,
+ "3645": 13270018048.0,
+ "3650": 13270018048.0,
+ "3655": 13270018048.0,
+ "3660": 13270018048.0,
+ "3665": 13270018048.0,
+ "3670": 13270018048.0,
+ "3675": 13270018048.0,
+ "3680": 13270018048.0,
+ "3685": 13270018048.0,
+ "3690": 13270018048.0,
+ "3695": 13270018048.0,
+ "3700": 13270018048.0,
+ "3705": 13270018048.0,
+ "3710": 13270018048.0,
+ "3715": 13270018048.0,
+ "3720": 13270018048.0,
+ "3725": 13270018048.0,
+ "3730": 13270018048.0,
+ "3735": 13270018048.0,
+ "3740": 13270018048.0,
+ "3745": 13270018048.0,
+ "3750": 13270018048.0,
+ "3755": 13270018048.0,
+ "3760": 13270018048.0,
+ "3765": 13270018048.0,
+ "3770": 13270018048.0,
+ "3775": 13270018048.0,
+ "3780": 13270018048.0,
+ "3785": 13270018048.0,
+ "3790": 13270018048.0,
+ "3795": 13270018048.0,
+ "3800": 13270018048.0,
+ "3805": 13270018048.0,
+ "3810": 13270018048.0,
+ "3815": 13270018048.0,
+ "3820": 13270018048.0,
+ "3825": 13270018048.0,
+ "3830": 13270018048.0,
+ "3835": 13270018048.0,
+ "3840": 13270018048.0,
+ "3845": 13270018048.0,
+ "3850": 13270018048.0,
+ "3855": 13270018048.0,
+ "3860": 13270018048.0,
+ "3865": 13270018048.0,
+ "3870": 13270018048.0,
+ "3875": 13270018048.0,
+ "3880": 13270018048.0,
+ "3885": 13270018048.0,
+ "3890": 13270018048.0,
+ "3895": 13270018048.0,
+ "3900": 13270018048.0,
+ "3905": 13270018048.0,
+ "3910": 13270018048.0,
+ "3915": 13270018048.0,
+ "3920": 13270018048.0,
+ "3925": 13270018048.0,
+ "3930": 13270018048.0,
+ "3935": 13270018048.0,
+ "3940": 13270018048.0,
+ "3945": 13270018048.0,
+ "3950": 13270018048.0,
+ "3955": 13270018048.0,
+ "3960": 13270018048.0,
+ "3965": 13270018048.0,
+ "3970": 13270018048.0,
+ "3975": 13270018048.0,
+ "3980": 13270018048.0,
+ "3985": 13270018048.0,
+ "3990": 13270018048.0,
+ "3995": 13270018048.0,
+ "4000": 13270018048.0,
+ "4005": 13270018048.0,
+ "4010": 13270018048.0,
+ "4015": 13270018048.0,
+ "4020": 13270018048.0,
+ "4025": 13270018048.0,
+ "4030": 13270018048.0,
+ "4035": 13270018048.0,
+ "4040": 13270018048.0,
+ "4045": 13270018048.0,
+ "4050": 13270018048.0,
+ "4055": 13270018048.0,
+ "4060": 13270018048.0,
+ "4065": 13270018048.0,
+ "4070": 13270018048.0,
+ "4075": 13270018048.0,
+ "4080": 13270018048.0,
+ "4085": 13270018048.0,
+ "4090": 13270018048.0,
+ "4095": 13270018048.0,
+ "4100": 13270018048.0,
+ "4105": 13270018048.0,
+ "4110": 13270018048.0,
+ "4115": 13270018048.0,
+ "4120": 13270018048.0,
+ "4125": 13270018048.0,
+ "4130": 13270018048.0,
+ "4135": 13270018048.0,
+ "4140": 13270018048.0,
+ "4145": 13270018048.0,
+ "4150": 13270018048.0,
+ "4155": 13270018048.0,
+ "4160": 13270018048.0,
+ "4165": 13270018048.0,
+ "4170": 13270018048.0,
+ "4175": 13270018048.0,
+ "4180": 13270018048.0,
+ "4185": 13270018048.0,
+ "4190": 13270018048.0,
+ "4195": 13270018048.0,
+ "4200": 13270018048.0,
+ "4205": 13270018048.0,
+ "4210": 13270018048.0,
+ "4215": 13270018048.0,
+ "4220": 13270018048.0,
+ "4225": 13270018048.0,
+ "4230": 13270018048.0,
+ "4235": 13270018048.0,
+ "4240": 13270018048.0,
+ "4245": 13270018048.0,
+ "4250": 13270018048.0,
+ "4255": 13270018048.0,
+ "4260": 13270018048.0,
+ "4265": 13270018048.0,
+ "4270": 13270018048.0,
+ "4275": 13270018048.0,
+ "4280": 13270018048.0,
+ "4285": 13270018048.0,
+ "4290": 13270018048.0,
+ "4295": 13270018048.0,
+ "4300": 13270018048.0,
+ "4305": 13270018048.0,
+ "4310": 13270018048.0,
+ "4315": 13270018048.0,
+ "4320": 13270018048.0,
+ "4325": 13270018048.0,
+ "4330": 13270018048.0,
+ "4335": 13270018048.0,
+ "4340": 13270018048.0,
+ "4345": 13270018048.0,
+ "4350": 13270018048.0,
+ "4355": 13270018048.0,
+ "4360": 13270018048.0,
+ "4365": 13270018048.0,
+ "4370": 13270018048.0,
+ "4375": 13270018048.0,
+ "4380": 13270018048.0,
+ "4385": 13270018048.0,
+ "4390": 13270018048.0,
+ "4395": 13270018048.0,
+ "4400": 13270018048.0,
+ "4405": 13270018048.0,
+ "4410": 13270018048.0,
+ "4415": 13270018048.0,
+ "4420": 13270018048.0,
+ "4425": 13270018048.0,
+ "4430": 13270018048.0,
+ "4435": 13270018048.0,
+ "4440": 13270018048.0,
+ "4445": 13270018048.0,
+ "4450": 13270018048.0,
+ "4455": 13270018048.0,
+ "4460": 13270018048.0,
+ "4465": 13270018048.0,
+ "4470": 13270018048.0,
+ "4475": 13270018048.0,
+ "4480": 13270018048.0,
+ "4485": 13270018048.0,
+ "4490": 13270018048.0,
+ "4495": 13270018048.0,
+ "4500": 13270018048.0,
+ "4505": 13270018048.0,
+ "4510": 13270018048.0,
+ "4515": 13270018048.0,
+ "4520": 13270018048.0,
+ "4525": 13270018048.0,
+ "4530": 13270018048.0,
+ "4535": 13270018048.0,
+ "4540": 13270018048.0,
+ "4545": 13270018048.0,
+ "4550": 13270018048.0,
+ "4555": 13270018048.0,
+ "4560": 13270018048.0,
+ "4565": 13270018048.0,
+ "4570": 13270018048.0,
+ "4575": 13270018048.0,
+ "4580": 13270018048.0,
+ "4585": 13270018048.0,
+ "4590": 13270018048.0,
+ "4595": 13270018048.0,
+ "4600": 13270018048.0,
+ "4605": 13270018048.0,
+ "4610": 13270018048.0,
+ "4615": 13270018048.0,
+ "4620": 13270018048.0,
+ "4625": 13270018048.0,
+ "4630": 13270018048.0,
+ "4635": 13270018048.0,
+ "4640": 13270018048.0,
+ "4645": 13270018048.0,
+ "4650": 13270018048.0,
+ "4655": 13270018048.0,
+ "4660": 13270018048.0,
+ "4665": 13270018048.0,
+ "4670": 13270018048.0,
+ "4675": 13270018048.0,
+ "4680": 13270018048.0,
+ "4685": 13270018048.0,
+ "4690": 13270018048.0,
+ "4695": 13270018048.0,
+ "4700": 13270018048.0,
+ "4705": 13270018048.0,
+ "4710": 13270018048.0,
+ "4715": 13270018048.0,
+ "4720": 13270018048.0,
+ "4725": 13270018048.0,
+ "4730": 13270018048.0,
+ "4735": 13270018048.0,
+ "4740": 13270018048.0,
+ "4745": 13270018048.0,
+ "4750": 13270018048.0,
+ "4755": 13270018048.0,
+ "4760": 13270018048.0,
+ "4765": 13270018048.0,
+ "4770": 13270018048.0,
+ "4775": 13270018048.0,
+ "4780": 13270018048.0,
+ "4785": 13270018048.0,
+ "4790": 13270018048.0,
+ "4795": 13270018048.0,
+ "4800": 13270018048.0,
+ "4805": 13270018048.0,
+ "4810": 13270018048.0,
+ "4815": 13270018048.0,
+ "4820": 13270018048.0,
+ "4825": 13270018048.0,
+ "4830": 13270018048.0,
+ "4835": 13270018048.0,
+ "4840": 13270018048.0,
+ "4845": 13270018048.0,
+ "4850": 13270018048.0,
+ "4855": 13270018048.0,
+ "4860": 13270018048.0,
+ "4865": 13270018048.0,
+ "4870": 13270018048.0,
+ "4875": 13270018048.0,
+ "4880": 13270018048.0,
+ "4885": 13270018048.0,
+ "4890": 13270018048.0,
+ "4895": 13270018048.0,
+ "4900": 13270018048.0,
+ "4905": 13270018048.0,
+ "4910": 13270018048.0,
+ "4915": 13270018048.0,
+ "4920": 13270018048.0,
+ "4925": 13270018048.0,
+ "4930": 13270018048.0,
+ "4935": 13270018048.0,
+ "4940": 13270018048.0,
+ "4945": 13270018048.0,
+ "4950": 13270018048.0,
+ "4955": 13270018048.0,
+ "4960": 13270018048.0,
+ "4965": 13270018048.0,
+ "4970": 13270018048.0,
+ "4975": 13270018048.0,
+ "4980": 13270018048.0,
+ "4985": 13270018048.0,
+ "4990": 13270018048.0,
+ "4995": 13270018048.0,
+ "5000": 13270018048.0,
+ "5005": 13270018048.0,
+ "5010": 13270018048.0,
+ "5015": 13270018048.0,
+ "5020": 13270018048.0,
+ "5025": 13270018048.0,
+ "5030": 13270018048.0,
+ "5035": 13270018048.0,
+ "5040": 13270018048.0,
+ "5045": 13270018048.0,
+ "5050": 13270018048.0,
+ "5055": 13270018048.0,
+ "5060": 13270018048.0,
+ "5065": 13270018048.0,
+ "5070": 13270018048.0,
+ "5075": 13270018048.0,
+ "5080": 13270018048.0,
+ "5085": 13270018048.0,
+ "5090": 13270018048.0,
+ "5095": 13270018048.0,
+ "5100": 13270018048.0,
+ "5105": 13270018048.0,
+ "5110": 13270018048.0,
+ "5115": 13270018048.0,
+ "5120": 13270018048.0,
+ "5125": 13270018048.0,
+ "5130": 13270018048.0,
+ "5135": 13270018048.0,
+ "5140": 13270018048.0,
+ "5145": 13270018048.0,
+ "5150": 13270018048.0,
+ "5155": 13270018048.0,
+ "5160": 13270018048.0,
+ "5165": 13270018048.0,
+ "5170": 13270018048.0,
+ "5175": 13270018048.0,
+ "5180": 13270018048.0,
+ "5185": 13270018048.0,
+ "5190": 13270018048.0,
+ "5195": 13270018048.0,
+ "5200": 13270018048.0,
+ "5205": 13270018048.0,
+ "5210": 13270018048.0,
+ "5215": 13270018048.0,
+ "5220": 13270018048.0,
+ "5225": 13270018048.0,
+ "5230": 13270018048.0,
+ "5235": 13270018048.0,
+ "5240": 13270018048.0,
+ "5245": 13270018048.0,
+ "5250": 13270018048.0,
+ "5255": 13270018048.0,
+ "5260": 13270018048.0,
+ "5265": 13270018048.0,
+ "5270": 13270018048.0,
+ "5275": 13270018048.0,
+ "5280": 13270018048.0,
+ "5285": 13270018048.0,
+ "5290": 13270018048.0,
+ "5295": 13270018048.0,
+ "5300": 13270018048.0,
+ "5305": 13270018048.0,
+ "5310": 13270018048.0,
+ "5315": 13270018048.0,
+ "5320": 13270018048.0,
+ "5325": 13270018048.0,
+ "5330": 13270018048.0,
+ "5335": 13270018048.0,
+ "5340": 13270018048.0,
+ "5345": 13270018048.0,
+ "5350": 13270018048.0,
+ "5355": 13270018048.0,
+ "5360": 13270018048.0,
+ "5365": 13270018048.0,
+ "5370": 13270018048.0,
+ "5375": 13270018048.0,
+ "5380": 13270018048.0,
+ "5385": 13270018048.0,
+ "5390": 13270018048.0,
+ "5395": 13270018048.0,
+ "5400": 13270018048.0,
+ "5405": 13270018048.0,
+ "5410": 13270018048.0,
+ "5415": 13270018048.0,
+ "5420": 13270018048.0,
+ "5425": 13270018048.0,
+ "5430": 13270018048.0,
+ "5435": 13270018048.0,
+ "5440": 13270018048.0,
+ "5445": 13270018048.0,
+ "5450": 13270018048.0,
+ "5455": 13270018048.0,
+ "5460": 13270018048.0,
+ "5465": 13270018048.0,
+ "5470": 13270018048.0,
+ "5475": 13270018048.0,
+ "5480": 13270018048.0,
+ "5485": 13270018048.0,
+ "5490": 13270018048.0,
+ "5495": 13270018048.0,
+ "5500": 13270018048.0,
+ "5505": 13270018048.0,
+ "5510": 13270018048.0,
+ "5515": 13270018048.0,
+ "5520": 13270018048.0,
+ "5525": 13270018048.0,
+ "5530": 13270018048.0,
+ "5535": 13270018048.0,
+ "5540": 13270018048.0,
+ "5545": 13270018048.0,
+ "5550": 13270018048.0,
+ "5555": 13270018048.0,
+ "5560": 13270018048.0,
+ "5565": 13270018048.0,
+ "5570": 13270018048.0,
+ "5575": 13270018048.0,
+ "5580": 13270018048.0,
+ "5585": 13270018048.0,
+ "5590": 13270018048.0,
+ "5595": 13270018048.0,
+ "5600": 13270018048.0,
+ "5605": 13270018048.0,
+ "5610": 13270018048.0,
+ "5615": 13270018048.0,
+ "5620": 13270018048.0,
+ "5625": 13270018048.0,
+ "5630": 13270018048.0,
+ "5635": 13270018048.0,
+ "5640": 13270018048.0,
+ "5645": 13270018048.0,
+ "5650": 13270018048.0,
+ "5655": 13270018048.0,
+ "5660": 13270018048.0,
+ "5665": 13270018048.0,
+ "5670": 13270018048.0,
+ "5675": 13270018048.0,
+ "5680": 13270018048.0,
+ "5685": 13270018048.0,
+ "5690": 13270018048.0,
+ "5695": 13270018048.0,
+ "5700": 13270018048.0,
+ "5705": 13270018048.0,
+ "5710": 13270018048.0,
+ "5715": 13270018048.0,
+ "5720": 13270018048.0,
+ "5725": 13270018048.0,
+ "5730": 13270018048.0,
+ "5735": 13270018048.0,
+ "5740": 13270018048.0,
+ "5745": 13270018048.0,
+ "5750": 13270018048.0,
+ "5755": 13270018048.0,
+ "5760": 13270018048.0,
+ "5765": 13270018048.0,
+ "5770": 13270018048.0,
+ "5775": 13270018048.0,
+ "5780": 13270018048.0,
+ "5785": 13270018048.0,
+ "5790": 13270018048.0,
+ "5795": 13270018048.0,
+ "5800": 13270018048.0,
+ "5805": 13270018048.0,
+ "5810": 13270018048.0,
+ "5815": 13270018048.0,
+ "5820": 13270018048.0,
+ "5825": 13270018048.0,
+ "5830": 13270018048.0,
+ "5835": 13270018048.0,
+ "5840": 13270018048.0,
+ "5845": 13270018048.0,
+ "5850": 13270018048.0,
+ "5855": 13270018048.0,
+ "5860": 13270018048.0,
+ "5865": 13270018048.0,
+ "5870": 13270018048.0,
+ "5875": 13270018048.0,
+ "5880": 13270018048.0,
+ "5885": 13270018048.0,
+ "5890": 13270018048.0,
+ "5895": 13270018048.0,
+ "5900": 13270018048.0,
+ "5905": 13270018048.0,
+ "5910": 13270018048.0,
+ "5915": 13270018048.0,
+ "5920": 13270018048.0,
+ "5925": 13270018048.0,
+ "5930": 13270018048.0,
+ "5935": 13270018048.0,
+ "5940": 13270018048.0,
+ "5945": 13270018048.0,
+ "5950": 13270018048.0,
+ "5955": 13270018048.0,
+ "5960": 13270018048.0,
+ "5965": 13270018048.0,
+ "5970": 13270018048.0,
+ "5975": 13270018048.0,
+ "5980": 13270018048.0,
+ "5985": 13270018048.0,
+ "5990": 13270018048.0,
+ "5995": 13270018048.0,
+ "6000": 13270018048.0,
+ "6005": 13270018048.0,
+ "6010": 13270018048.0,
+ "6015": 13270018048.0,
+ "6020": 13270018048.0,
+ "6025": 13270018048.0,
+ "6030": 13270018048.0,
+ "6035": 13270018048.0,
+ "6040": 13270018048.0,
+ "6045": 13270018048.0,
+ "6050": 13270018048.0,
+ "6055": 13270018048.0,
+ "6060": 13270018048.0,
+ "6065": 13270018048.0,
+ "6070": 13270018048.0,
+ "6075": 13270018048.0,
+ "6080": 13270018048.0,
+ "6085": 13270018048.0,
+ "6090": 13270018048.0,
+ "6095": 13270018048.0,
+ "6100": 13270018048.0,
+ "6105": 13270018048.0,
+ "6110": 13270018048.0,
+ "6115": 13270018048.0,
+ "6120": 13270018048.0,
+ "6125": 13270018048.0,
+ "6130": 13270018048.0,
+ "6135": 13270018048.0,
+ "6140": 13270018048.0,
+ "6145": 13270018048.0,
+ "6150": 13270018048.0,
+ "6155": 13270018048.0,
+ "6160": 13270018048.0,
+ "6165": 13270018048.0,
+ "6170": 13270018048.0,
+ "6175": 13270018048.0,
+ "6180": 13270018048.0,
+ "6185": 13270018048.0,
+ "6190": 13270018048.0,
+ "6195": 13270018048.0,
+ "6200": 13270018048.0,
+ "6205": 13270018048.0,
+ "6210": 13270018048.0,
+ "6215": 13270018048.0,
+ "6220": 13270018048.0,
+ "6225": 13270018048.0,
+ "6230": 13270018048.0,
+ "6235": 13270018048.0,
+ "6240": 13270018048.0,
+ "6245": 13270018048.0,
+ "6250": 13270018048.0,
+ "6255": 13270018048.0,
+ "6260": 13270018048.0,
+ "6265": 13270018048.0,
+ "6270": 13270018048.0,
+ "6275": 13270018048.0,
+ "6280": 13270018048.0,
+ "6285": 13270018048.0,
+ "6290": 13270018048.0,
+ "6295": 13270018048.0,
+ "6300": 13270018048.0,
+ "6305": 13270018048.0,
+ "6310": 13270018048.0,
+ "6315": 13270018048.0,
+ "6320": 13270018048.0,
+ "6325": 13270018048.0,
+ "6330": 13270018048.0,
+ "6335": 13270018048.0,
+ "6340": 13270018048.0,
+ "6345": 13270018048.0,
+ "6350": 13270018048.0,
+ "6355": 13270018048.0,
+ "6360": 13270018048.0,
+ "6365": 13270018048.0,
+ "6370": 13270018048.0,
+ "6375": 13270018048.0,
+ "6380": 13270018048.0,
+ "6385": 13270018048.0,
+ "6390": 13270018048.0,
+ "6395": 13270018048.0,
+ "6400": 13270018048.0,
+ "6405": 13270018048.0,
+ "6410": 13270018048.0,
+ "6415": 13270018048.0,
+ "6420": 13270018048.0,
+ "6425": 13270018048.0,
+ "6430": 13270018048.0,
+ "6435": 13270018048.0,
+ "6440": 13270018048.0,
+ "6445": 13270018048.0,
+ "6450": 13270018048.0,
+ "6455": 13270018048.0,
+ "6460": 13270018048.0,
+ "6465": 13270018048.0,
+ "6470": 13270018048.0,
+ "6475": 13270018048.0,
+ "6480": 13270018048.0,
+ "6485": 13270018048.0,
+ "6490": 13270018048.0,
+ "6495": 13270018048.0,
+ "6500": 13270018048.0,
+ "6505": 13270018048.0,
+ "6510": 13270018048.0,
+ "6515": 13270018048.0,
+ "6520": 13270018048.0,
+ "6525": 13270018048.0,
+ "6530": 13270018048.0,
+ "6535": 13270018048.0,
+ "6540": 13270018048.0,
+ "6545": 13270018048.0,
+ "6550": 13270018048.0,
+ "6555": 13270018048.0,
+ "6560": 13270018048.0,
+ "6565": 13270018048.0,
+ "6570": 13270018048.0,
+ "6575": 13270018048.0,
+ "6580": 13270018048.0,
+ "6585": 13270018048.0,
+ "6590": 13270018048.0,
+ "6595": 13270018048.0,
+ "6600": 13270018048.0,
+ "6605": 13270018048.0,
+ "6610": 13270018048.0,
+ "6615": 13270018048.0,
+ "6620": 13270018048.0,
+ "6625": 13270018048.0,
+ "6630": 13270018048.0,
+ "6635": 13270018048.0,
+ "6640": 13270018048.0,
+ "6645": 13270018048.0,
+ "6650": 13270018048.0,
+ "6655": 13270018048.0,
+ "6660": 13270018048.0,
+ "6665": 13270018048.0,
+ "6670": 13270018048.0,
+ "6675": 13270018048.0,
+ "6680": 13270018048.0,
+ "6685": 13270018048.0,
+ "6690": 13270018048.0,
+ "6695": 13270018048.0,
+ "6700": 13270018048.0,
+ "6705": 13270018048.0,
+ "6710": 13270018048.0,
+ "6715": 13270018048.0,
+ "6720": 13270018048.0,
+ "6725": 13270018048.0,
+ "6730": 13270018048.0,
+ "6735": 13270018048.0,
+ "6740": 13270018048.0,
+ "6745": 13270018048.0,
+ "6750": 13270018048.0,
+ "6755": 13270018048.0,
+ "6760": 13270018048.0,
+ "6765": 13270018048.0,
+ "6770": 13270018048.0,
+ "6775": 13270018048.0,
+ "6780": 13270018048.0,
+ "6785": 13270018048.0,
+ "6790": 13270018048.0,
+ "6795": 13270018048.0,
+ "6800": 13270018048.0,
+ "6805": 13270018048.0,
+ "6810": 13270018048.0,
+ "6815": 13270018048.0,
+ "6820": 13270018048.0,
+ "6825": 13270018048.0,
+ "6830": 13270018048.0,
+ "6835": 13270018048.0,
+ "6840": 13270018048.0,
+ "6845": 13270018048.0,
+ "6850": 13270018048.0,
+ "6855": 13270018048.0,
+ "6860": 13270018048.0,
+ "6865": 13270018048.0,
+ "6870": 13270018048.0,
+ "6875": 13270018048.0,
+ "6880": 13270018048.0,
+ "6885": 13270018048.0,
+ "6890": 13270018048.0,
+ "6895": 13270018048.0,
+ "6900": 13270018048.0,
+ "6905": 13270018048.0,
+ "6910": 13270018048.0,
+ "6915": 13270018048.0,
+ "6920": 13270018048.0,
+ "6925": 13270018048.0,
+ "6930": 13270018048.0,
+ "6935": 13270018048.0,
+ "6940": 13270018048.0,
+ "6945": 13270018048.0,
+ "6950": 13270018048.0,
+ "6955": 13270018048.0,
+ "6960": 13270018048.0,
+ "6965": 13270018048.0,
+ "6970": 13270018048.0,
+ "6975": 13270018048.0,
+ "6980": 13270018048.0,
+ "6985": 13270018048.0,
+ "6990": 13270018048.0,
+ "6995": 13270018048.0,
+ "7000": 13270018048.0,
+ "7005": 13270018048.0,
+ "7010": 13270018048.0,
+ "7015": 13270018048.0,
+ "7020": 13270018048.0,
+ "7025": 13270018048.0,
+ "7030": 13270018048.0,
+ "7035": 13270018048.0,
+ "7040": 13270018048.0,
+ "7045": 13270018048.0,
+ "7050": 13270018048.0,
+ "7055": 13270018048.0,
+ "7060": 13270018048.0,
+ "7065": 13270018048.0,
+ "7070": 13270018048.0,
+ "7075": 13270018048.0,
+ "7080": 13270018048.0,
+ "7085": 13270018048.0,
+ "7090": 13270018048.0,
+ "7095": 13270018048.0,
+ "7100": 13270018048.0,
+ "7105": 13270018048.0,
+ "7110": 13270018048.0,
+ "7115": 13270018048.0,
+ "7120": 13270018048.0,
+ "7125": 13270018048.0,
+ "7130": 13270018048.0,
+ "7135": 13270018048.0,
+ "7140": 13270018048.0,
+ "7145": 13270018048.0,
+ "7150": 13270018048.0,
+ "7155": 13270018048.0,
+ "7160": 13270018048.0,
+ "7165": 13270018048.0,
+ "7170": 13270018048.0,
+ "7175": 13270018048.0,
+ "7180": 13270018048.0,
+ "7185": 13270018048.0,
+ "7190": 13270018048.0,
+ "7195": 13270018048.0,
+ "7200": 13270018048.0,
+ "7205": 13270018048.0,
+ "7210": 13270018048.0,
+ "7215": 13270018048.0,
+ "7220": 13270018048.0,
+ "7225": 13270018048.0,
+ "7230": 13270018048.0,
+ "7235": 13270018048.0,
+ "7240": 13270018048.0,
+ "7245": 13270018048.0,
+ "7250": 13270018048.0,
+ "7255": 13270018048.0,
+ "7260": 13270018048.0,
+ "7265": 13270018048.0,
+ "7270": 13270018048.0,
+ "7275": 13270018048.0,
+ "7280": 13270018048.0,
+ "7285": 13270018048.0,
+ "7290": 13270018048.0,
+ "7295": 13270018048.0,
+ "7300": 13270018048.0,
+ "7305": 13270018048.0,
+ "7310": 13270018048.0,
+ "7315": 13270018048.0,
+ "7320": 13270018048.0,
+ "7325": 13270018048.0,
+ "7330": 13270018048.0,
+ "7335": 13270018048.0,
+ "7340": 13270018048.0,
+ "7345": 13270018048.0,
+ "7350": 13270018048.0,
+ "7355": 13270018048.0,
+ "7360": 13270018048.0,
+ "7365": 13270018048.0,
+ "7370": 13270018048.0,
+ "7375": 13270018048.0,
+ "7380": 13270018048.0,
+ "7385": 13270018048.0,
+ "7390": 13270018048.0,
+ "7395": 13270018048.0,
+ "7400": 13270018048.0,
+ "7405": 13270018048.0,
+ "7410": 13270018048.0,
+ "7415": 13270018048.0,
+ "7420": 13270018048.0,
+ "7425": 13270018048.0,
+ "7430": 13270018048.0,
+ "7435": 13270018048.0,
+ "7440": 13270018048.0,
+ "7445": 13270018048.0,
+ "7450": 13270018048.0,
+ "7455": 13270018048.0,
+ "7460": 13270018048.0,
+ "7465": 13270018048.0,
+ "7470": 13270018048.0,
+ "7475": 13270018048.0,
+ "7480": 13270018048.0,
+ "7485": 13270018048.0,
+ "7490": 13270018048.0,
+ "7495": 13270018048.0,
+ "7500": 13270018048.0,
+ "7505": 13270018048.0,
+ "7510": 13270018048.0,
+ "7515": 13270018048.0,
+ "7520": 13270018048.0,
+ "7525": 13270018048.0,
+ "7530": 13270018048.0,
+ "7535": 13270018048.0,
+ "7540": 13270018048.0,
+ "7545": 13270018048.0,
+ "7550": 13270018048.0,
+ "7555": 13270018048.0,
+ "7560": 13270018048.0,
+ "7565": 13270018048.0,
+ "7570": 13270018048.0,
+ "7575": 13270018048.0,
+ "7580": 13270018048.0,
+ "7585": 13270018048.0,
+ "7590": 13270018048.0,
+ "7595": 13270018048.0,
+ "7600": 13270018048.0,
+ "7605": 13270018048.0,
+ "7610": 13270018048.0,
+ "7615": 13270018048.0,
+ "7620": 13270018048.0,
+ "7625": 13270018048.0,
+ "7630": 13270018048.0,
+ "7635": 13270018048.0,
+ "7640": 13270018048.0,
+ "7645": 13270018048.0,
+ "7650": 13270018048.0,
+ "7655": 13270018048.0,
+ "7660": 13270018048.0,
+ "7665": 13270018048.0,
+ "7670": 13270018048.0,
+ "7675": 13270018048.0,
+ "7680": 13270018048.0,
+ "7685": 13270018048.0,
+ "7690": 13270018048.0,
+ "7695": 13270018048.0,
+ "7700": 13270018048.0,
+ "7705": 13270018048.0,
+ "7710": 13270018048.0,
+ "7715": 13270018048.0,
+ "7720": 13270018048.0,
+ "7725": 13270018048.0,
+ "7730": 13270018048.0,
+ "7735": 13270018048.0,
+ "7740": 13270018048.0,
+ "7745": 13270018048.0,
+ "7750": 13270018048.0,
+ "7755": 13270018048.0,
+ "7760": 13270018048.0,
+ "7765": 13270018048.0,
+ "7770": 13270018048.0,
+ "7775": 13270018048.0,
+ "7780": 13270018048.0,
+ "7785": 13270018048.0,
+ "7790": 13270018048.0,
+ "7795": 13270018048.0,
+ "7800": 13270018048.0,
+ "7805": 13270018048.0,
+ "7810": 13270018048.0,
+ "7815": 13270018048.0,
+ "7820": 13270018048.0,
+ "7825": 13270018048.0,
+ "7830": 13270018048.0,
+ "7835": 13270018048.0,
+ "7840": 13270018048.0,
+ "7845": 13270018048.0,
+ "7850": 13270018048.0,
+ "7855": 13270018048.0,
+ "7860": 13270018048.0,
+ "7865": 13270018048.0,
+ "7870": 13270018048.0,
+ "7875": 13270018048.0,
+ "7880": 13270018048.0,
+ "7885": 13270018048.0,
+ "7890": 13270018048.0,
+ "7895": 13270018048.0,
+ "7900": 13270018048.0,
+ "7905": 13270018048.0,
+ "7910": 13270018048.0,
+ "7915": 13270018048.0,
+ "7920": 13270018048.0,
+ "7925": 13270018048.0,
+ "7930": 13270018048.0,
+ "7935": 13270018048.0,
+ "7940": 13270018048.0,
+ "7945": 13270018048.0,
+ "7950": 13270018048.0,
+ "7955": 13270018048.0,
+ "7960": 13270018048.0,
+ "7965": 13270018048.0,
+ "7970": 13270018048.0,
+ "7975": 13270018048.0,
+ "7980": 13270018048.0,
+ "7985": 13270018048.0,
+ "7990": 13270018048.0,
+ "7995": 13270018048.0,
+ "8000": 13270018048.0,
+ "8005": 13270018048.0,
+ "8010": 13270018048.0,
+ "8015": 13270018048.0,
+ "8020": 13270018048.0,
+ "8025": 13270018048.0,
+ "8030": 13270018048.0,
+ "8035": 13270018048.0,
+ "8040": 13270018048.0,
+ "8045": 13270018048.0,
+ "8050": 13270018048.0,
+ "8055": 13270018048.0,
+ "8060": 13270018048.0,
+ "8065": 13270018048.0,
+ "8070": 13270018048.0,
+ "8075": 13270018048.0,
+ "8080": 13270018048.0,
+ "8085": 13270018048.0,
+ "8090": 13270018048.0,
+ "8095": 13270018048.0,
+ "8100": 13270018048.0,
+ "8105": 13270018048.0,
+ "8110": 13270018048.0,
+ "8115": 13270018048.0,
+ "8120": 13270018048.0,
+ "8125": 13270018048.0,
+ "8130": 13270018048.0,
+ "8135": 13270018048.0,
+ "8140": 13270018048.0,
+ "8145": 13270018048.0,
+ "8150": 13270018048.0,
+ "8155": 13270018048.0,
+ "8160": 13270018048.0,
+ "8165": 13270018048.0,
+ "8170": 13270018048.0,
+ "8175": 13270018048.0,
+ "8180": 13270018048.0,
+ "8185": 13270018048.0,
+ "8190": 13270018048.0,
+ "8195": 13270018048.0,
+ "8200": 13270018048.0,
+ "8205": 13270018048.0,
+ "8210": 13270018048.0,
+ "8215": 13270018048.0,
+ "8220": 13270018048.0,
+ "8225": 13270018048.0,
+ "8230": 13270018048.0,
+ "8235": 13270018048.0,
+ "8240": 13270018048.0,
+ "8245": 13270018048.0,
+ "8250": 13270018048.0,
+ "8255": 13270018048.0,
+ "8260": 13270018048.0,
+ "8265": 13270018048.0,
+ "8270": 13270018048.0,
+ "8275": 13270018048.0,
+ "8280": 13270018048.0,
+ "8285": 13270018048.0,
+ "8290": 13270018048.0,
+ "8295": 13270018048.0,
+ "8300": 13270018048.0,
+ "8305": 13270018048.0,
+ "8310": 13270018048.0,
+ "8315": 13270018048.0,
+ "8320": 13270018048.0,
+ "8325": 13270018048.0,
+ "8330": 13270018048.0,
+ "8335": 13270018048.0,
+ "8340": 13270018048.0,
+ "8345": 13270018048.0,
+ "8350": 13270018048.0,
+ "8355": 13270018048.0,
+ "8360": 13270018048.0,
+ "8365": 13270018048.0,
+ "8370": 13270018048.0,
+ "8375": 13270018048.0,
+ "8380": 13270018048.0,
+ "8385": 13270018048.0,
+ "8390": 13270018048.0,
+ "8395": 13270018048.0,
+ "8400": 13270018048.0,
+ "8405": 13270018048.0,
+ "8410": 13270018048.0,
+ "8415": 13270018048.0,
+ "8420": 13270018048.0,
+ "8425": 13270018048.0,
+ "8430": 13270018048.0,
+ "8435": 13270018048.0,
+ "8440": 13270018048.0,
+ "8445": 13270018048.0,
+ "8450": 13270018048.0,
+ "8455": 13270018048.0,
+ "8460": 13270018048.0,
+ "8465": 13270018048.0,
+ "8470": 13270018048.0,
+ "8475": 13270018048.0,
+ "8480": 13270018048.0,
+ "8485": 13270018048.0,
+ "8490": 13270018048.0,
+ "8495": 13270018048.0,
+ "8500": 13270018048.0,
+ "8505": 13270018048.0,
+ "8510": 13270018048.0,
+ "8515": 13270018048.0,
+ "8520": 13270018048.0,
+ "8525": 13270018048.0,
+ "8530": 13270018048.0,
+ "8535": 13270018048.0,
+ "8540": 13270018048.0,
+ "8545": 13270018048.0,
+ "8550": 13270018048.0,
+ "8555": 13270018048.0,
+ "8560": 13270018048.0,
+ "8565": 13270018048.0,
+ "8570": 13270018048.0,
+ "8575": 13270018048.0,
+ "8580": 13270018048.0,
+ "8585": 13270018048.0,
+ "8590": 13270018048.0,
+ "8595": 13270018048.0,
+ "8600": 13270018048.0,
+ "8605": 13270018048.0,
+ "8610": 13270018048.0,
+ "8615": 13270018048.0,
+ "8620": 13270018048.0,
+ "8625": 13270018048.0,
+ "8630": 13270018048.0,
+ "8635": 13270018048.0,
+ "8640": 13270018048.0,
+ "8645": 13270018048.0,
+ "8650": 13270018048.0,
+ "8655": 13270018048.0,
+ "8660": 13270018048.0,
+ "8665": 13270018048.0,
+ "8670": 13270018048.0,
+ "8675": 13270018048.0,
+ "8680": 13270018048.0,
+ "8685": 13270018048.0,
+ "8690": 13270018048.0,
+ "8695": 13270018048.0,
+ "8700": 13270018048.0,
+ "8705": 13270018048.0,
+ "8710": 13270018048.0,
+ "8715": 13270018048.0,
+ "8720": 13270018048.0,
+ "8725": 13270018048.0,
+ "8730": 13270018048.0,
+ "8735": 13270018048.0,
+ "8740": 13270018048.0,
+ "8745": 13270018048.0,
+ "8750": 13270018048.0,
+ "8755": 13270018048.0,
+ "8760": 13270018048.0,
+ "8765": 13270018048.0,
+ "8770": 13270018048.0,
+ "8775": 13270018048.0,
+ "8780": 13270018048.0,
+ "8785": 13270018048.0,
+ "8790": 13270018048.0,
+ "8795": 13270018048.0,
+ "8800": 13270018048.0,
+ "8805": 13270018048.0,
+ "8810": 13268436992.0,
+ "8815": 13268436992.0,
+ "8820": 13268436992.0,
+ "8825": 13268436992.0,
+ "8830": 13268436992.0,
+ "8835": 13268436992.0,
+ "8840": 13268436992.0,
+ "8845": 13268436992.0,
+ "8850": 13268436992.0,
+ "8855": 13268436992.0,
+ "8860": 13268436992.0,
+ "8865": 13268436992.0,
+ "8870": 13268436992.0,
+ "8875": 13268436992.0,
+ "8880": 13268436992.0,
+ "8885": 13268436992.0,
+ "8890": 13268436992.0,
+ "8895": 13268436992.0,
+ "8900": 13268436992.0,
+ "8905": 13268436992.0,
+ "8910": 13268436992.0,
+ "8915": 13268436992.0,
+ "8920": 13268436992.0,
+ "8925": 13268436992.0,
+ "8930": 13268436992.0,
+ "8935": 13268436992.0,
+ "8940": 13268436992.0,
+ "8945": 13268436992.0,
+ "8950": 13268436992.0,
+ "8955": 13268436992.0,
+ "8960": 13268436992.0,
+ "8965": 13268436992.0,
+ "8970": 13268436992.0,
+ "8975": 13268436992.0,
+ "8980": 13268436992.0,
+ "8985": 13268436992.0,
+ "8990": 13268436992.0,
+ "8995": 13268436992.0,
+ "9000": 13268436992.0,
+ "9005": 13268436992.0,
+ "9010": 13268436992.0,
+ "9015": 13268436992.0,
+ "9020": 13268436992.0,
+ "9025": 13268436992.0,
+ "9030": 13268436992.0,
+ "9035": 13268436992.0,
+ "9040": 13268436992.0,
+ "9045": 13268436992.0,
+ "9050": 13268436992.0,
+ "9055": 13268436992.0,
+ "9060": 13268436992.0,
+ "9065": 13268436992.0,
+ "9070": 13268436992.0,
+ "9075": 13268436992.0,
+ "9080": 13268436992.0,
+ "9085": 13268436992.0,
+ "9090": 13268436992.0,
+ "9095": 13268436992.0,
+ "9100": 13268436992.0,
+ "9105": 13268436992.0,
+ "9110": 13268436992.0,
+ "9115": 13268436992.0,
+ "9120": 13268436992.0,
+ "9125": 13268436992.0,
+ "9130": 13268436992.0,
+ "9135": 13268436992.0,
+ "9140": 13268436992.0,
+ "9145": 13268436992.0,
+ "9150": 13268436992.0,
+ "9155": 13268436992.0,
+ "9160": 13268436992.0,
+ "9165": 13268436992.0,
+ "9170": 13268436992.0,
+ "9175": 13268436992.0,
+ "9180": 13268436992.0,
+ "9185": 13268436992.0,
+ "9190": 13268436992.0,
+ "9195": 13268436992.0,
+ "9200": 13268436992.0,
+ "9205": 13268436992.0,
+ "9210": 13268436992.0,
+ "9215": 13268436992.0,
+ "9220": 13268436992.0,
+ "9225": 13268436992.0,
+ "9230": 13268436992.0,
+ "9235": 13268436992.0,
+ "9240": 13268436992.0,
+ "9245": 13268436992.0,
+ "9250": 13268436992.0,
+ "9255": 13268436992.0,
+ "9260": 13268436992.0,
+ "9265": 13268436992.0,
+ "9270": 13268436992.0,
+ "9275": 13268436992.0,
+ "9280": 13268436992.0,
+ "9285": 13268436992.0,
+ "9290": 13268436992.0,
+ "9295": 13268436992.0,
+ "9300": 13268436992.0,
+ "9305": 13268436992.0,
+ "9310": 13268436992.0,
+ "9315": 13268436992.0,
+ "9320": 13268436992.0,
+ "9325": 13268436992.0,
+ "9330": 13268436992.0,
+ "9335": 13268436992.0,
+ "9340": 13268436992.0,
+ "9345": 13268436992.0,
+ "9350": 13268436992.0,
+ "9355": 13268436992.0,
+ "9360": 13268436992.0,
+ "9365": 13268436992.0,
+ "9370": 13268436992.0,
+ "9375": 13268436992.0,
+ "9380": 13268436992.0,
+ "9385": 13268436992.0,
+ "9390": 13268436992.0,
+ "9395": 13268436992.0,
+ "9400": 13268436992.0,
+ "9405": 13268436992.0,
+ "9410": 13268436992.0,
+ "9415": 13268436992.0,
+ "9420": 13268436992.0,
+ "9425": 13268436992.0,
+ "9430": 13268436992.0,
+ "9435": 13268436992.0,
+ "9440": 13268436992.0,
+ "9445": 13268436992.0,
+ "9450": 13268436992.0,
+ "9455": 13268436992.0,
+ "9460": 13268436992.0,
+ "9465": 13268436992.0,
+ "9470": 13268436992.0,
+ "9475": 13268436992.0,
+ "9480": 13268436992.0,
+ "9485": 13268436992.0,
+ "9490": 13268436992.0,
+ "9495": 13268436992.0,
+ "9500": 13268436992.0,
+ "9505": 13268436992.0,
+ "9510": 13268436992.0,
+ "9515": 13268436992.0,
+ "9520": 13268436992.0,
+ "9525": 13268436992.0,
+ "9530": 13268436992.0,
+ "9535": 13268436992.0,
+ "9540": 13268436992.0,
+ "9545": 13268436992.0,
+ "9550": 13268436992.0,
+ "9555": 13268436992.0,
+ "9560": 13268436992.0,
+ "9565": 13268436992.0,
+ "9570": 13268436992.0,
+ "9575": 13268436992.0,
+ "9580": 13268436992.0,
+ "9585": 13268436992.0,
+ "9590": 13268436992.0,
+ "9595": 13268436992.0,
+ "9600": 13268436992.0,
+ "9605": 13268436992.0,
+ "9610": 13268436992.0,
+ "9615": 13268436992.0,
+ "9620": 13268436992.0,
+ "9625": 13268436992.0,
+ "9630": 13268436992.0,
+ "9635": 13268436992.0,
+ "9640": 13268436992.0,
+ "9645": 13268436992.0,
+ "9650": 13268436992.0,
+ "9655": 13268436992.0,
+ "9660": 13268436992.0,
+ "9665": 13268436992.0,
+ "9670": 13268436992.0,
+ "9675": 13268436992.0,
+ "9680": 13268436992.0,
+ "9685": 13268436992.0,
+ "9690": 13268436992.0,
+ "9695": 13268436992.0,
+ "9700": 13268436992.0,
+ "9705": 13268436992.0,
+ "9710": 13268436992.0,
+ "9715": 13268436992.0,
+ "9720": 13268436992.0,
+ "9725": 13268436992.0,
+ "9730": 13268436992.0,
+ "9735": 13268436992.0,
+ "9740": 13268436992.0,
+ "9745": 13268436992.0,
+ "9750": 13268436992.0,
+ "9755": 13268436992.0,
+ "9760": 13268436992.0,
+ "9765": 13268436992.0,
+ "9770": 13268436992.0,
+ "9775": 13268436992.0,
+ "9780": 13268436992.0,
+ "9785": 13268436992.0,
+ "9790": 13268436992.0,
+ "9795": 13268436992.0,
+ "9800": 13268436992.0,
+ "9805": 13268436992.0,
+ "9810": 13268436992.0,
+ "9815": 13268436992.0,
+ "9820": 13268436992.0,
+ "9825": 13268436992.0,
+ "9830": 13268436992.0,
+ "9835": 13268436992.0,
+ "9840": 13268436992.0,
+ "9845": 13268436992.0,
+ "9850": 13268436992.0,
+ "9855": 13268436992.0,
+ "9860": 13268436992.0,
+ "9865": 13268436992.0,
+ "9870": 13268436992.0,
+ "9875": 13268436992.0,
+ "9880": 13268436992.0,
+ "9885": 13268436992.0,
+ "9890": 13268436992.0,
+ "9895": 13268436992.0,
+ "9900": 13268436992.0,
+ "9905": 13268436992.0,
+ "9910": 13268436992.0,
+ "9915": 13268436992.0,
+ "9920": 13268436992.0,
+ "9925": 13268436992.0,
+ "9930": 13268436992.0,
+ "9935": 13268436992.0,
+ "9940": 13268436992.0,
+ "9945": 13268436992.0,
+ "9950": 13268436992.0,
+ "9955": 13268436992.0,
+ "9960": 13268436992.0,
+ "9965": 13268436992.0,
+ "9970": 13268436992.0,
+ "9975": 13268436992.0,
+ "9980": 13268436992.0,
+ "9985": 13268436992.0,
+ "9990": 13268436992.0,
+ "9995": 13268436992.0,
+ "10000": 13268436992.0,
+ "10005": 13268436992.0,
+ "10010": 13268436992.0,
+ "10015": 13268436992.0,
+ "10020": 13268436992.0,
+ "10025": 13268436992.0,
+ "10030": 13268436992.0,
+ "10035": 13268436992.0,
+ "10040": 13268436992.0,
+ "10045": 13268436992.0,
+ "10050": 13268436992.0,
+ "10055": 13268436992.0,
+ "10060": 13268436992.0,
+ "10065": 13268436992.0,
+ "10070": 13268436992.0,
+ "10075": 13268436992.0,
+ "10080": 13268436992.0,
+ "10085": 13268436992.0,
+ "10090": 13268436992.0,
+ "10095": 13268436992.0,
+ "10100": 13268436992.0,
+ "10105": 13268436992.0,
+ "10110": 13268436992.0,
+ "10115": 13268436992.0,
+ "10120": 13268436992.0,
+ "10125": 13268436992.0,
+ "10130": 13268436992.0,
+ "10135": 13268436992.0,
+ "10140": 13268436992.0,
+ "10145": 13268436992.0,
+ "10150": 13268436992.0,
+ "10155": 13268436992.0,
+ "10160": 13268436992.0,
+ "10165": 13268436992.0,
+ "10170": 13268436992.0,
+ "10175": 13268436992.0,
+ "10180": 13268436992.0,
+ "10185": 13268436992.0,
+ "10190": 13268436992.0,
+ "10195": 13268436992.0,
+ "10200": 13268436992.0
+ }
+ },
+ "mem-max-allocated-bytes": {
+ "start_step": 1,
+ "end_step": 10200,
+ "step_interval": 5,
+ "values": {
+ "1": 27658637312.0,
+ "5": 28158672896.0,
+ "10": 28158672896.0,
+ "15": 28158672896.0,
+ "20": 28158672896.0,
+ "25": 28158672896.0,
+ "30": 28158672896.0,
+ "35": 28158672896.0,
+ "40": 28158672896.0,
+ "45": 28158672896.0,
+ "50": 28158672896.0,
+ "55": 28158672896.0,
+ "60": 28158672896.0,
+ "65": 28158672896.0,
+ "70": 28158672896.0,
+ "75": 28158672896.0,
+ "80": 28158672896.0,
+ "85": 28158672896.0,
+ "90": 28158672896.0,
+ "95": 28158672896.0,
+ "100": 28158672896.0,
+ "105": 28158672896.0,
+ "110": 28158672896.0,
+ "115": 28158672896.0,
+ "120": 28158672896.0,
+ "125": 28158672896.0,
+ "130": 28158672896.0,
+ "135": 28158672896.0,
+ "140": 28158672896.0,
+ "145": 28158672896.0,
+ "150": 28158672896.0,
+ "155": 28158672896.0,
+ "160": 28158672896.0,
+ "165": 28158672896.0,
+ "170": 28158672896.0,
+ "175": 28158672896.0,
+ "180": 28158672896.0,
+ "185": 28158672896.0,
+ "190": 28158672896.0,
+ "195": 28158672896.0,
+ "200": 28158672896.0,
+ "205": 28158672896.0,
+ "210": 28158672896.0,
+ "215": 28158672896.0,
+ "220": 28158672896.0,
+ "225": 28158672896.0,
+ "230": 28158672896.0,
+ "235": 28158672896.0,
+ "240": 28158672896.0,
+ "245": 28158672896.0,
+ "250": 28158672896.0,
+ "255": 28158672896.0,
+ "260": 28158672896.0,
+ "265": 28158672896.0,
+ "270": 28158672896.0,
+ "275": 28158672896.0,
+ "280": 28158672896.0,
+ "285": 28158672896.0,
+ "290": 28158672896.0,
+ "295": 28158672896.0,
+ "300": 28158672896.0,
+ "305": 28158672896.0,
+ "310": 28158672896.0,
+ "315": 28158672896.0,
+ "320": 28158672896.0,
+ "325": 28158672896.0,
+ "330": 28158672896.0,
+ "335": 28158672896.0,
+ "340": 28158672896.0,
+ "345": 28158672896.0,
+ "350": 28158672896.0,
+ "355": 28158672896.0,
+ "360": 28158672896.0,
+ "365": 28158672896.0,
+ "370": 28158672896.0,
+ "375": 28158672896.0,
+ "380": 28158672896.0,
+ "385": 28158672896.0,
+ "390": 28158672896.0,
+ "395": 28158672896.0,
+ "400": 28158672896.0,
+ "405": 28158672896.0,
+ "410": 28158672896.0,
+ "415": 28158672896.0,
+ "420": 28158672896.0,
+ "425": 28158672896.0,
+ "430": 28158672896.0,
+ "435": 28158672896.0,
+ "440": 28158672896.0,
+ "445": 28158672896.0,
+ "450": 28158672896.0,
+ "455": 28158672896.0,
+ "460": 28158672896.0,
+ "465": 28158672896.0,
+ "470": 28158672896.0,
+ "475": 28158672896.0,
+ "480": 28158672896.0,
+ "485": 28158672896.0,
+ "490": 28158672896.0,
+ "495": 28158672896.0,
+ "500": 28158672896.0,
+ "505": 28158672896.0,
+ "510": 28158672896.0,
+ "515": 28158672896.0,
+ "520": 28158672896.0,
+ "525": 28158672896.0,
+ "530": 28158672896.0,
+ "535": 28158672896.0,
+ "540": 28158672896.0,
+ "545": 28158672896.0,
+ "550": 28158672896.0,
+ "555": 28158672896.0,
+ "560": 28158672896.0,
+ "565": 28158672896.0,
+ "570": 28158672896.0,
+ "575": 28158672896.0,
+ "580": 28158672896.0,
+ "585": 28158672896.0,
+ "590": 28158672896.0,
+ "595": 28158672896.0,
+ "600": 28158672896.0,
+ "605": 28158672896.0,
+ "610": 28158672896.0,
+ "615": 28158672896.0,
+ "620": 28158672896.0,
+ "625": 28158672896.0,
+ "630": 28158672896.0,
+ "635": 28158672896.0,
+ "640": 28158672896.0,
+ "645": 28158672896.0,
+ "650": 28158672896.0,
+ "655": 28158672896.0,
+ "660": 28158672896.0,
+ "665": 28158672896.0,
+ "670": 28158672896.0,
+ "675": 28158672896.0,
+ "680": 28158672896.0,
+ "685": 28158672896.0,
+ "690": 28158672896.0,
+ "695": 28158672896.0,
+ "700": 28158672896.0,
+ "705": 28158672896.0,
+ "710": 28158672896.0,
+ "715": 28158672896.0,
+ "720": 28158672896.0,
+ "725": 28158672896.0,
+ "730": 28158672896.0,
+ "735": 28158672896.0,
+ "740": 28158672896.0,
+ "745": 28158672896.0,
+ "750": 28158672896.0,
+ "755": 28158672896.0,
+ "760": 28158672896.0,
+ "765": 28158672896.0,
+ "770": 28158672896.0,
+ "775": 28158672896.0,
+ "780": 28158672896.0,
+ "785": 28158672896.0,
+ "790": 28158672896.0,
+ "795": 28158672896.0,
+ "800": 28158672896.0,
+ "805": 28158672896.0,
+ "810": 28158672896.0,
+ "815": 28158672896.0,
+ "820": 28158672896.0,
+ "825": 28158672896.0,
+ "830": 28158672896.0,
+ "835": 28158672896.0,
+ "840": 28158672896.0,
+ "845": 28158672896.0,
+ "850": 28158672896.0,
+ "855": 28158672896.0,
+ "860": 28158672896.0,
+ "865": 28158672896.0,
+ "870": 28158672896.0,
+ "875": 28158672896.0,
+ "880": 28158672896.0,
+ "885": 28158672896.0,
+ "890": 28158672896.0,
+ "895": 28158672896.0,
+ "900": 28158672896.0,
+ "905": 28158672896.0,
+ "910": 28158672896.0,
+ "915": 28158672896.0,
+ "920": 28158672896.0,
+ "925": 28158672896.0,
+ "930": 28158672896.0,
+ "935": 28158672896.0,
+ "940": 28158672896.0,
+ "945": 28158672896.0,
+ "950": 28158672896.0,
+ "955": 28158672896.0,
+ "960": 28158672896.0,
+ "965": 28158672896.0,
+ "970": 28158672896.0,
+ "975": 28158672896.0,
+ "980": 28158672896.0,
+ "985": 28158672896.0,
+ "990": 28158672896.0,
+ "995": 28158672896.0,
+ "1000": 28158672896.0,
+ "1005": 28158672896.0,
+ "1010": 28158672896.0,
+ "1015": 28158672896.0,
+ "1020": 28158672896.0,
+ "1025": 28158672896.0,
+ "1030": 28158672896.0,
+ "1035": 28158672896.0,
+ "1040": 28158672896.0,
+ "1045": 28158672896.0,
+ "1050": 28158672896.0,
+ "1055": 28158672896.0,
+ "1060": 28158672896.0,
+ "1065": 28158672896.0,
+ "1070": 28158672896.0,
+ "1075": 28158672896.0,
+ "1080": 28158672896.0,
+ "1085": 28158672896.0,
+ "1090": 28158672896.0,
+ "1095": 28158672896.0,
+ "1100": 28158672896.0,
+ "1105": 28158672896.0,
+ "1110": 28158672896.0,
+ "1115": 28158672896.0,
+ "1120": 28158672896.0,
+ "1125": 28158672896.0,
+ "1130": 28158672896.0,
+ "1135": 28158672896.0,
+ "1140": 28158672896.0,
+ "1145": 28158672896.0,
+ "1150": 28158672896.0,
+ "1155": 28158672896.0,
+ "1160": 28158672896.0,
+ "1165": 28158672896.0,
+ "1170": 28158672896.0,
+ "1175": 28158672896.0,
+ "1180": 28158672896.0,
+ "1185": 28158672896.0,
+ "1190": 28158672896.0,
+ "1195": 28158672896.0,
+ "1200": 28158672896.0,
+ "1205": 28158672896.0,
+ "1210": 28158672896.0,
+ "1215": 28158672896.0,
+ "1220": 28158672896.0,
+ "1225": 28158672896.0,
+ "1230": 28158672896.0,
+ "1235": 28158672896.0,
+ "1240": 28158672896.0,
+ "1245": 28158672896.0,
+ "1250": 28158672896.0,
+ "1255": 28158672896.0,
+ "1260": 28158672896.0,
+ "1265": 28158672896.0,
+ "1270": 28158672896.0,
+ "1275": 28158672896.0,
+ "1280": 28158672896.0,
+ "1285": 28158672896.0,
+ "1290": 28158672896.0,
+ "1295": 28158672896.0,
+ "1300": 28158672896.0,
+ "1305": 28158672896.0,
+ "1310": 28158672896.0,
+ "1315": 28158672896.0,
+ "1320": 28158672896.0,
+ "1325": 28158672896.0,
+ "1330": 28158672896.0,
+ "1335": 28158672896.0,
+ "1340": 28158672896.0,
+ "1345": 28158672896.0,
+ "1350": 28158672896.0,
+ "1355": 28158672896.0,
+ "1360": 28158672896.0,
+ "1365": 28158672896.0,
+ "1370": 28158672896.0,
+ "1375": 28158672896.0,
+ "1380": 28158672896.0,
+ "1385": 28158672896.0,
+ "1390": 28158672896.0,
+ "1395": 28158672896.0,
+ "1400": 28158672896.0,
+ "1405": 28158672896.0,
+ "1410": 28158672896.0,
+ "1415": 28158672896.0,
+ "1420": 28158672896.0,
+ "1425": 28158672896.0,
+ "1430": 28158672896.0,
+ "1435": 28158672896.0,
+ "1440": 28158672896.0,
+ "1445": 28158672896.0,
+ "1450": 28158672896.0,
+ "1455": 28158672896.0,
+ "1460": 28158672896.0,
+ "1465": 28158672896.0,
+ "1470": 28158672896.0,
+ "1475": 28158672896.0,
+ "1480": 28158672896.0,
+ "1485": 28158672896.0,
+ "1490": 28158672896.0,
+ "1495": 28158672896.0,
+ "1500": 28158672896.0,
+ "1505": 28158672896.0,
+ "1510": 28158672896.0,
+ "1515": 28158672896.0,
+ "1520": 28158672896.0,
+ "1525": 28158672896.0,
+ "1530": 28158672896.0,
+ "1535": 28158672896.0,
+ "1540": 28158672896.0,
+ "1545": 28158672896.0,
+ "1550": 28158672896.0,
+ "1555": 28158672896.0,
+ "1560": 28158672896.0,
+ "1565": 28158672896.0,
+ "1570": 28158672896.0,
+ "1575": 28158672896.0,
+ "1580": 28158672896.0,
+ "1585": 28158672896.0,
+ "1590": 28158672896.0,
+ "1595": 28158672896.0,
+ "1600": 28158672896.0,
+ "1605": 28158672896.0,
+ "1610": 28158672896.0,
+ "1615": 28158672896.0,
+ "1620": 28158672896.0,
+ "1625": 28158672896.0,
+ "1630": 28158672896.0,
+ "1635": 28158672896.0,
+ "1640": 28158672896.0,
+ "1645": 28158672896.0,
+ "1650": 28158672896.0,
+ "1655": 28158672896.0,
+ "1660": 28158672896.0,
+ "1665": 28158672896.0,
+ "1670": 28158672896.0,
+ "1675": 28158672896.0,
+ "1680": 28158672896.0,
+ "1685": 28158672896.0,
+ "1690": 28158672896.0,
+ "1695": 28158672896.0,
+ "1700": 28158672896.0,
+ "1705": 28158672896.0,
+ "1710": 28158672896.0,
+ "1715": 28158672896.0,
+ "1720": 28158672896.0,
+ "1725": 28158672896.0,
+ "1730": 28158672896.0,
+ "1735": 28158672896.0,
+ "1740": 28158672896.0,
+ "1745": 28158672896.0,
+ "1750": 28158672896.0,
+ "1755": 28158672896.0,
+ "1760": 28158672896.0,
+ "1765": 28158672896.0,
+ "1770": 28158672896.0,
+ "1775": 28158672896.0,
+ "1780": 28158672896.0,
+ "1785": 28158672896.0,
+ "1790": 28158672896.0,
+ "1795": 28158672896.0,
+ "1800": 28158672896.0,
+ "1805": 28158672896.0,
+ "1810": 28158672896.0,
+ "1815": 28158672896.0,
+ "1820": 28158672896.0,
+ "1825": 28158672896.0,
+ "1830": 28158672896.0,
+ "1835": 28158672896.0,
+ "1840": 28158672896.0,
+ "1845": 28158672896.0,
+ "1850": 28158672896.0,
+ "1855": 28158672896.0,
+ "1860": 28158672896.0,
+ "1865": 28158672896.0,
+ "1870": 28158672896.0,
+ "1875": 28158672896.0,
+ "1880": 28158672896.0,
+ "1885": 28158672896.0,
+ "1890": 28158672896.0,
+ "1895": 28158672896.0,
+ "1900": 28158672896.0,
+ "1905": 28158672896.0,
+ "1910": 28158672896.0,
+ "1915": 28158672896.0,
+ "1920": 28158672896.0,
+ "1925": 28158672896.0,
+ "1930": 28158672896.0,
+ "1935": 28158672896.0,
+ "1940": 28158672896.0,
+ "1945": 28158672896.0,
+ "1950": 28158672896.0,
+ "1955": 28158672896.0,
+ "1960": 28158672896.0,
+ "1965": 28158672896.0,
+ "1970": 28158672896.0,
+ "1975": 28158672896.0,
+ "1980": 28158672896.0,
+ "1985": 28158672896.0,
+ "1990": 28158672896.0,
+ "1995": 28158672896.0,
+ "2000": 28158672896.0,
+ "2005": 28158672896.0,
+ "2010": 28158672896.0,
+ "2015": 28158672896.0,
+ "2020": 28158672896.0,
+ "2025": 28158672896.0,
+ "2030": 28158672896.0,
+ "2035": 28158672896.0,
+ "2040": 28158672896.0,
+ "2045": 28158672896.0,
+ "2050": 28158672896.0,
+ "2055": 28158672896.0,
+ "2060": 28158672896.0,
+ "2065": 28158672896.0,
+ "2070": 28158672896.0,
+ "2075": 28158672896.0,
+ "2080": 28158672896.0,
+ "2085": 28158672896.0,
+ "2090": 28158672896.0,
+ "2095": 28158672896.0,
+ "2100": 28158672896.0,
+ "2105": 28158672896.0,
+ "2110": 28158672896.0,
+ "2115": 28158672896.0,
+ "2120": 28158672896.0,
+ "2125": 28158672896.0,
+ "2130": 28158672896.0,
+ "2135": 28158672896.0,
+ "2140": 28158672896.0,
+ "2145": 28158672896.0,
+ "2150": 28158672896.0,
+ "2155": 28158672896.0,
+ "2160": 28158672896.0,
+ "2165": 28158672896.0,
+ "2170": 28158672896.0,
+ "2175": 28158672896.0,
+ "2180": 28158672896.0,
+ "2185": 28158672896.0,
+ "2190": 28158672896.0,
+ "2195": 28158672896.0,
+ "2200": 28158672896.0,
+ "2205": 28158672896.0,
+ "2210": 28158672896.0,
+ "2215": 28158672896.0,
+ "2220": 28158672896.0,
+ "2225": 28158672896.0,
+ "2230": 28158672896.0,
+ "2235": 28158672896.0,
+ "2240": 28158672896.0,
+ "2245": 28158672896.0,
+ "2250": 28158672896.0,
+ "2255": 28158672896.0,
+ "2260": 28158672896.0,
+ "2265": 28158672896.0,
+ "2270": 28158672896.0,
+ "2275": 28158672896.0,
+ "2280": 28158672896.0,
+ "2285": 28158672896.0,
+ "2290": 28158672896.0,
+ "2295": 28158672896.0,
+ "2300": 28158672896.0,
+ "2305": 28158672896.0,
+ "2310": 28158672896.0,
+ "2315": 28158672896.0,
+ "2320": 28158672896.0,
+ "2325": 28158672896.0,
+ "2330": 28158672896.0,
+ "2335": 28158672896.0,
+ "2340": 28158672896.0,
+ "2345": 28158672896.0,
+ "2350": 28158672896.0,
+ "2355": 28158672896.0,
+ "2360": 28158672896.0,
+ "2365": 28158672896.0,
+ "2370": 28158672896.0,
+ "2375": 28158672896.0,
+ "2380": 28158672896.0,
+ "2385": 28158672896.0,
+ "2390": 28158672896.0,
+ "2395": 28158672896.0,
+ "2400": 28158672896.0,
+ "2405": 28158672896.0,
+ "2410": 28158672896.0,
+ "2415": 28158672896.0,
+ "2420": 28158672896.0,
+ "2425": 28158672896.0,
+ "2430": 28158672896.0,
+ "2435": 28158672896.0,
+ "2440": 28158672896.0,
+ "2445": 28158672896.0,
+ "2450": 28158672896.0,
+ "2455": 28158672896.0,
+ "2460": 28158672896.0,
+ "2465": 28158672896.0,
+ "2470": 28158672896.0,
+ "2475": 28158672896.0,
+ "2480": 28158672896.0,
+ "2485": 28158672896.0,
+ "2490": 28158672896.0,
+ "2495": 28158672896.0,
+ "2500": 28158672896.0,
+ "2505": 28158672896.0,
+ "2510": 28158672896.0,
+ "2515": 28158672896.0,
+ "2520": 28158672896.0,
+ "2525": 28158672896.0,
+ "2530": 28158672896.0,
+ "2535": 28158672896.0,
+ "2540": 28158672896.0,
+ "2545": 28158672896.0,
+ "2550": 28158672896.0,
+ "2555": 28158672896.0,
+ "2560": 28158672896.0,
+ "2565": 28158672896.0,
+ "2570": 28158672896.0,
+ "2575": 28158672896.0,
+ "2580": 28158672896.0,
+ "2585": 28158672896.0,
+ "2590": 28158672896.0,
+ "2595": 28158672896.0,
+ "2600": 28158672896.0,
+ "2605": 28158672896.0,
+ "2610": 28158672896.0,
+ "2615": 28158672896.0,
+ "2620": 28158672896.0,
+ "2625": 28158672896.0,
+ "2630": 28158672896.0,
+ "2635": 28158672896.0,
+ "2640": 28158672896.0,
+ "2645": 28158672896.0,
+ "2650": 28158672896.0,
+ "2655": 28158672896.0,
+ "2660": 28158672896.0,
+ "2665": 28158672896.0,
+ "2670": 28158672896.0,
+ "2675": 28158672896.0,
+ "2680": 28158672896.0,
+ "2685": 28158672896.0,
+ "2690": 28158672896.0,
+ "2695": 28158672896.0,
+ "2700": 28158672896.0,
+ "2705": 28158672896.0,
+ "2710": 28158672896.0,
+ "2715": 28158672896.0,
+ "2720": 28158672896.0,
+ "2725": 28158672896.0,
+ "2730": 28158672896.0,
+ "2735": 28158672896.0,
+ "2740": 28158672896.0,
+ "2745": 28158672896.0,
+ "2750": 28158672896.0,
+ "2755": 28158672896.0,
+ "2760": 28158672896.0,
+ "2765": 28158672896.0,
+ "2770": 28158672896.0,
+ "2775": 28158672896.0,
+ "2780": 28158672896.0,
+ "2785": 28158672896.0,
+ "2790": 28158672896.0,
+ "2795": 28158672896.0,
+ "2800": 28158672896.0,
+ "2805": 28158672896.0,
+ "2810": 28158672896.0,
+ "2815": 28158672896.0,
+ "2820": 28158672896.0,
+ "2825": 28158672896.0,
+ "2830": 28158672896.0,
+ "2835": 28158672896.0,
+ "2840": 28158672896.0,
+ "2845": 28158672896.0,
+ "2850": 28158672896.0,
+ "2855": 28158672896.0,
+ "2860": 28158672896.0,
+ "2865": 28158672896.0,
+ "2870": 28158672896.0,
+ "2875": 28158672896.0,
+ "2880": 28158672896.0,
+ "2885": 28158672896.0,
+ "2890": 28158672896.0,
+ "2895": 28158672896.0,
+ "2900": 28158672896.0,
+ "2905": 28158672896.0,
+ "2910": 28158672896.0,
+ "2915": 28158672896.0,
+ "2920": 28158672896.0,
+ "2925": 28158672896.0,
+ "2930": 28158672896.0,
+ "2935": 28158672896.0,
+ "2940": 28158672896.0,
+ "2945": 28158672896.0,
+ "2950": 28158672896.0,
+ "2955": 28158672896.0,
+ "2960": 28158672896.0,
+ "2965": 28158672896.0,
+ "2970": 28158672896.0,
+ "2975": 28158672896.0,
+ "2980": 28158672896.0,
+ "2985": 28158672896.0,
+ "2990": 28158672896.0,
+ "2995": 28158672896.0,
+ "3000": 28158672896.0,
+ "3005": 28158672896.0,
+ "3010": 28158672896.0,
+ "3015": 28158672896.0,
+ "3020": 28158672896.0,
+ "3025": 28158672896.0,
+ "3030": 28158672896.0,
+ "3035": 28158672896.0,
+ "3040": 28158672896.0,
+ "3045": 28158672896.0,
+ "3050": 28158672896.0,
+ "3055": 28158672896.0,
+ "3060": 28158672896.0,
+ "3065": 28158672896.0,
+ "3070": 28158672896.0,
+ "3075": 28158672896.0,
+ "3080": 28158672896.0,
+ "3085": 28158672896.0,
+ "3090": 28158672896.0,
+ "3095": 28158672896.0,
+ "3100": 28158672896.0,
+ "3105": 28158672896.0,
+ "3110": 28158672896.0,
+ "3115": 28158672896.0,
+ "3120": 28158672896.0,
+ "3125": 28158672896.0,
+ "3130": 28158672896.0,
+ "3135": 28158672896.0,
+ "3140": 28158672896.0,
+ "3145": 28158672896.0,
+ "3150": 28158672896.0,
+ "3155": 28158672896.0,
+ "3160": 28158672896.0,
+ "3165": 28158672896.0,
+ "3170": 28158672896.0,
+ "3175": 28158672896.0,
+ "3180": 28158672896.0,
+ "3185": 28158672896.0,
+ "3190": 28158672896.0,
+ "3195": 28158672896.0,
+ "3200": 28158672896.0,
+ "3205": 28158672896.0,
+ "3210": 28158672896.0,
+ "3215": 28158672896.0,
+ "3220": 28158672896.0,
+ "3225": 28158672896.0,
+ "3230": 28158672896.0,
+ "3235": 28158672896.0,
+ "3240": 28158672896.0,
+ "3245": 28158672896.0,
+ "3250": 28158672896.0,
+ "3255": 28158672896.0,
+ "3260": 28158672896.0,
+ "3265": 28158672896.0,
+ "3270": 28158672896.0,
+ "3275": 28158672896.0,
+ "3280": 28158672896.0,
+ "3285": 28158672896.0,
+ "3290": 28158672896.0,
+ "3295": 28158672896.0,
+ "3300": 28158672896.0,
+ "3305": 28158672896.0,
+ "3310": 28158672896.0,
+ "3315": 28158672896.0,
+ "3320": 28158672896.0,
+ "3325": 28158672896.0,
+ "3330": 28158672896.0,
+ "3335": 28158672896.0,
+ "3340": 28158672896.0,
+ "3345": 28158672896.0,
+ "3350": 28158672896.0,
+ "3355": 28158672896.0,
+ "3360": 28158672896.0,
+ "3365": 28158672896.0,
+ "3370": 28158672896.0,
+ "3375": 28158672896.0,
+ "3380": 28158672896.0,
+ "3385": 28158672896.0,
+ "3390": 28158672896.0,
+ "3395": 28158672896.0,
+ "3400": 28158672896.0,
+ "3405": 28158672896.0,
+ "3410": 28158672896.0,
+ "3415": 28158672896.0,
+ "3420": 28158672896.0,
+ "3425": 28158672896.0,
+ "3430": 28158672896.0,
+ "3435": 28158672896.0,
+ "3440": 28158672896.0,
+ "3445": 28158672896.0,
+ "3450": 28158672896.0,
+ "3455": 28158672896.0,
+ "3460": 28158672896.0,
+ "3465": 28158672896.0,
+ "3470": 28158672896.0,
+ "3475": 28158672896.0,
+ "3480": 28158672896.0,
+ "3485": 28158672896.0,
+ "3490": 28158672896.0,
+ "3495": 28158672896.0,
+ "3500": 28158672896.0,
+ "3505": 28158672896.0,
+ "3510": 28158672896.0,
+ "3515": 28158672896.0,
+ "3520": 28158672896.0,
+ "3525": 28158672896.0,
+ "3530": 28158672896.0,
+ "3535": 28158672896.0,
+ "3540": 28158672896.0,
+ "3545": 28158672896.0,
+ "3550": 28158672896.0,
+ "3555": 28158672896.0,
+ "3560": 28158672896.0,
+ "3565": 28158672896.0,
+ "3570": 28158672896.0,
+ "3575": 28158672896.0,
+ "3580": 28158672896.0,
+ "3585": 28158672896.0,
+ "3590": 28158672896.0,
+ "3595": 28158672896.0,
+ "3600": 28158672896.0,
+ "3605": 28158672896.0,
+ "3610": 28158672896.0,
+ "3615": 28158672896.0,
+ "3620": 28158672896.0,
+ "3625": 28158672896.0,
+ "3630": 28158672896.0,
+ "3635": 28158672896.0,
+ "3640": 28158672896.0,
+ "3645": 28158672896.0,
+ "3650": 28158672896.0,
+ "3655": 28158672896.0,
+ "3660": 28158672896.0,
+ "3665": 28158672896.0,
+ "3670": 28158672896.0,
+ "3675": 28158672896.0,
+ "3680": 28158672896.0,
+ "3685": 28158672896.0,
+ "3690": 28158672896.0,
+ "3695": 28158672896.0,
+ "3700": 28158672896.0,
+ "3705": 28158672896.0,
+ "3710": 28158672896.0,
+ "3715": 28158672896.0,
+ "3720": 28158672896.0,
+ "3725": 28158672896.0,
+ "3730": 28158672896.0,
+ "3735": 28158672896.0,
+ "3740": 28158672896.0,
+ "3745": 28158672896.0,
+ "3750": 28158672896.0,
+ "3755": 28158672896.0,
+ "3760": 28158672896.0,
+ "3765": 28158672896.0,
+ "3770": 28158672896.0,
+ "3775": 28158672896.0,
+ "3780": 28158672896.0,
+ "3785": 28158672896.0,
+ "3790": 28158672896.0,
+ "3795": 28158672896.0,
+ "3800": 28158672896.0,
+ "3805": 28158672896.0,
+ "3810": 28158672896.0,
+ "3815": 28158672896.0,
+ "3820": 28158672896.0,
+ "3825": 28158672896.0,
+ "3830": 28158672896.0,
+ "3835": 28158672896.0,
+ "3840": 28158672896.0,
+ "3845": 28158672896.0,
+ "3850": 28158672896.0,
+ "3855": 28158672896.0,
+ "3860": 28158672896.0,
+ "3865": 28158672896.0,
+ "3870": 28158672896.0,
+ "3875": 28158672896.0,
+ "3880": 28158672896.0,
+ "3885": 28158672896.0,
+ "3890": 28158672896.0,
+ "3895": 28158672896.0,
+ "3900": 28158672896.0,
+ "3905": 28158672896.0,
+ "3910": 28158672896.0,
+ "3915": 28158672896.0,
+ "3920": 28158672896.0,
+ "3925": 28158672896.0,
+ "3930": 28158672896.0,
+ "3935": 28158672896.0,
+ "3940": 28158672896.0,
+ "3945": 28158672896.0,
+ "3950": 28158672896.0,
+ "3955": 28158672896.0,
+ "3960": 28158672896.0,
+ "3965": 28158672896.0,
+ "3970": 28158672896.0,
+ "3975": 28158672896.0,
+ "3980": 28158672896.0,
+ "3985": 28158672896.0,
+ "3990": 28158672896.0,
+ "3995": 28158672896.0,
+ "4000": 28158672896.0,
+ "4005": 28158672896.0,
+ "4010": 28158672896.0,
+ "4015": 28158672896.0,
+ "4020": 28158672896.0,
+ "4025": 28158672896.0,
+ "4030": 28158672896.0,
+ "4035": 28158672896.0,
+ "4040": 28158672896.0,
+ "4045": 28158672896.0,
+ "4050": 28158672896.0,
+ "4055": 28158672896.0,
+ "4060": 28158672896.0,
+ "4065": 28158672896.0,
+ "4070": 28158672896.0,
+ "4075": 28158672896.0,
+ "4080": 28158672896.0,
+ "4085": 28158672896.0,
+ "4090": 28158672896.0,
+ "4095": 28158672896.0,
+ "4100": 28158672896.0,
+ "4105": 28158672896.0,
+ "4110": 28158672896.0,
+ "4115": 28158672896.0,
+ "4120": 28158672896.0,
+ "4125": 28158672896.0,
+ "4130": 28158672896.0,
+ "4135": 28158672896.0,
+ "4140": 28158672896.0,
+ "4145": 28158672896.0,
+ "4150": 28158672896.0,
+ "4155": 28158672896.0,
+ "4160": 28158672896.0,
+ "4165": 28158672896.0,
+ "4170": 28158672896.0,
+ "4175": 28158672896.0,
+ "4180": 28158672896.0,
+ "4185": 28158672896.0,
+ "4190": 28158672896.0,
+ "4195": 28158672896.0,
+ "4200": 28158672896.0,
+ "4205": 28158672896.0,
+ "4210": 28158672896.0,
+ "4215": 28158672896.0,
+ "4220": 28158672896.0,
+ "4225": 28158672896.0,
+ "4230": 28158672896.0,
+ "4235": 28158672896.0,
+ "4240": 28158672896.0,
+ "4245": 28158672896.0,
+ "4250": 28158672896.0,
+ "4255": 28158672896.0,
+ "4260": 28158672896.0,
+ "4265": 28158672896.0,
+ "4270": 28158672896.0,
+ "4275": 28158672896.0,
+ "4280": 28158672896.0,
+ "4285": 28158672896.0,
+ "4290": 28158672896.0,
+ "4295": 28158672896.0,
+ "4300": 28158672896.0,
+ "4305": 28158672896.0,
+ "4310": 28158672896.0,
+ "4315": 28158672896.0,
+ "4320": 28158672896.0,
+ "4325": 28158672896.0,
+ "4330": 28158672896.0,
+ "4335": 28158672896.0,
+ "4340": 28158672896.0,
+ "4345": 28158672896.0,
+ "4350": 28158672896.0,
+ "4355": 28158672896.0,
+ "4360": 28158672896.0,
+ "4365": 28158672896.0,
+ "4370": 28158672896.0,
+ "4375": 28158672896.0,
+ "4380": 28158672896.0,
+ "4385": 28158672896.0,
+ "4390": 28158672896.0,
+ "4395": 28158672896.0,
+ "4400": 28158672896.0,
+ "4405": 28158672896.0,
+ "4410": 28158672896.0,
+ "4415": 28158672896.0,
+ "4420": 28158672896.0,
+ "4425": 28158672896.0,
+ "4430": 28158672896.0,
+ "4435": 28158672896.0,
+ "4440": 28158672896.0,
+ "4445": 28158672896.0,
+ "4450": 28158672896.0,
+ "4455": 28158672896.0,
+ "4460": 28158672896.0,
+ "4465": 28158672896.0,
+ "4470": 28158672896.0,
+ "4475": 28158672896.0,
+ "4480": 28158672896.0,
+ "4485": 28158672896.0,
+ "4490": 28158672896.0,
+ "4495": 28158672896.0,
+ "4500": 28158672896.0,
+ "4505": 28158672896.0,
+ "4510": 28158672896.0,
+ "4515": 28158672896.0,
+ "4520": 28158672896.0,
+ "4525": 28158672896.0,
+ "4530": 28158672896.0,
+ "4535": 28158672896.0,
+ "4540": 28158672896.0,
+ "4545": 28158672896.0,
+ "4550": 28158672896.0,
+ "4555": 28158672896.0,
+ "4560": 28158672896.0,
+ "4565": 28158672896.0,
+ "4570": 28158672896.0,
+ "4575": 28158672896.0,
+ "4580": 28158672896.0,
+ "4585": 28158672896.0,
+ "4590": 28158672896.0,
+ "4595": 28158672896.0,
+ "4600": 28158672896.0,
+ "4605": 28158672896.0,
+ "4610": 28158672896.0,
+ "4615": 28158672896.0,
+ "4620": 28158672896.0,
+ "4625": 28158672896.0,
+ "4630": 28158672896.0,
+ "4635": 28158672896.0,
+ "4640": 28158672896.0,
+ "4645": 28158672896.0,
+ "4650": 28158672896.0,
+ "4655": 28158672896.0,
+ "4660": 28158672896.0,
+ "4665": 28158672896.0,
+ "4670": 28158672896.0,
+ "4675": 28158672896.0,
+ "4680": 28158672896.0,
+ "4685": 28158672896.0,
+ "4690": 28158672896.0,
+ "4695": 28158672896.0,
+ "4700": 28158672896.0,
+ "4705": 28158672896.0,
+ "4710": 28158672896.0,
+ "4715": 28158672896.0,
+ "4720": 28158672896.0,
+ "4725": 28158672896.0,
+ "4730": 28158672896.0,
+ "4735": 28158672896.0,
+ "4740": 28158672896.0,
+ "4745": 28158672896.0,
+ "4750": 28158672896.0,
+ "4755": 28158672896.0,
+ "4760": 28158672896.0,
+ "4765": 28158672896.0,
+ "4770": 28158672896.0,
+ "4775": 28158672896.0,
+ "4780": 28158672896.0,
+ "4785": 28158672896.0,
+ "4790": 28158672896.0,
+ "4795": 28158672896.0,
+ "4800": 28158672896.0,
+ "4805": 28158672896.0,
+ "4810": 28158672896.0,
+ "4815": 28158672896.0,
+ "4820": 28158672896.0,
+ "4825": 28158672896.0,
+ "4830": 28158672896.0,
+ "4835": 28158672896.0,
+ "4840": 28158672896.0,
+ "4845": 28158672896.0,
+ "4850": 28158672896.0,
+ "4855": 28158672896.0,
+ "4860": 28158672896.0,
+ "4865": 28158672896.0,
+ "4870": 28158672896.0,
+ "4875": 28158672896.0,
+ "4880": 28158672896.0,
+ "4885": 28158672896.0,
+ "4890": 28158672896.0,
+ "4895": 28158672896.0,
+ "4900": 28158672896.0,
+ "4905": 28158672896.0,
+ "4910": 28158672896.0,
+ "4915": 28158672896.0,
+ "4920": 28158672896.0,
+ "4925": 28158672896.0,
+ "4930": 28158672896.0,
+ "4935": 28158672896.0,
+ "4940": 28158672896.0,
+ "4945": 28158672896.0,
+ "4950": 28158672896.0,
+ "4955": 28158672896.0,
+ "4960": 28158672896.0,
+ "4965": 28158672896.0,
+ "4970": 28158672896.0,
+ "4975": 28158672896.0,
+ "4980": 28158672896.0,
+ "4985": 28158672896.0,
+ "4990": 28158672896.0,
+ "4995": 28158672896.0,
+ "5000": 28158672896.0,
+ "5005": 28158672896.0,
+ "5010": 28158672896.0,
+ "5015": 28158672896.0,
+ "5020": 28158672896.0,
+ "5025": 28158672896.0,
+ "5030": 28158672896.0,
+ "5035": 28158672896.0,
+ "5040": 28158672896.0,
+ "5045": 28158672896.0,
+ "5050": 28158672896.0,
+ "5055": 28158672896.0,
+ "5060": 28158672896.0,
+ "5065": 28158672896.0,
+ "5070": 28158672896.0,
+ "5075": 28158672896.0,
+ "5080": 28158672896.0,
+ "5085": 28158672896.0,
+ "5090": 28158672896.0,
+ "5095": 28158672896.0,
+ "5100": 28158672896.0,
+ "5105": 28158672896.0,
+ "5110": 28158672896.0,
+ "5115": 28158672896.0,
+ "5120": 28158672896.0,
+ "5125": 28158672896.0,
+ "5130": 28158672896.0,
+ "5135": 28158672896.0,
+ "5140": 28158672896.0,
+ "5145": 28158672896.0,
+ "5150": 28158672896.0,
+ "5155": 28158672896.0,
+ "5160": 28158672896.0,
+ "5165": 28158672896.0,
+ "5170": 28158672896.0,
+ "5175": 28158672896.0,
+ "5180": 28158672896.0,
+ "5185": 28158672896.0,
+ "5190": 28158672896.0,
+ "5195": 28158672896.0,
+ "5200": 28158672896.0,
+ "5205": 28158672896.0,
+ "5210": 28158672896.0,
+ "5215": 28158672896.0,
+ "5220": 28158672896.0,
+ "5225": 28158672896.0,
+ "5230": 28158672896.0,
+ "5235": 28158672896.0,
+ "5240": 28158672896.0,
+ "5245": 28158672896.0,
+ "5250": 28158672896.0,
+ "5255": 28158672896.0,
+ "5260": 28158672896.0,
+ "5265": 28158672896.0,
+ "5270": 28158672896.0,
+ "5275": 28158672896.0,
+ "5280": 28158672896.0,
+ "5285": 28158672896.0,
+ "5290": 28158672896.0,
+ "5295": 28158672896.0,
+ "5300": 28158672896.0,
+ "5305": 28158672896.0,
+ "5310": 28158672896.0,
+ "5315": 28158672896.0,
+ "5320": 28158672896.0,
+ "5325": 28158672896.0,
+ "5330": 28158672896.0,
+ "5335": 28158672896.0,
+ "5340": 28158672896.0,
+ "5345": 28158672896.0,
+ "5350": 28158672896.0,
+ "5355": 28158672896.0,
+ "5360": 28158672896.0,
+ "5365": 28158672896.0,
+ "5370": 28158672896.0,
+ "5375": 28158672896.0,
+ "5380": 28158672896.0,
+ "5385": 28158672896.0,
+ "5390": 28158672896.0,
+ "5395": 28158672896.0,
+ "5400": 28158672896.0,
+ "5405": 28158672896.0,
+ "5410": 28158672896.0,
+ "5415": 28158672896.0,
+ "5420": 28158672896.0,
+ "5425": 28158672896.0,
+ "5430": 28158672896.0,
+ "5435": 28158672896.0,
+ "5440": 28158672896.0,
+ "5445": 28158672896.0,
+ "5450": 28158672896.0,
+ "5455": 28158672896.0,
+ "5460": 28158672896.0,
+ "5465": 28158672896.0,
+ "5470": 28158672896.0,
+ "5475": 28158672896.0,
+ "5480": 28158672896.0,
+ "5485": 28158672896.0,
+ "5490": 28158672896.0,
+ "5495": 28158672896.0,
+ "5500": 28158672896.0,
+ "5505": 28158672896.0,
+ "5510": 28158672896.0,
+ "5515": 28158672896.0,
+ "5520": 28158672896.0,
+ "5525": 28158672896.0,
+ "5530": 28158672896.0,
+ "5535": 28158672896.0,
+ "5540": 28158672896.0,
+ "5545": 28158672896.0,
+ "5550": 28158672896.0,
+ "5555": 28158672896.0,
+ "5560": 28158672896.0,
+ "5565": 28158672896.0,
+ "5570": 28158672896.0,
+ "5575": 28158672896.0,
+ "5580": 28158672896.0,
+ "5585": 28158672896.0,
+ "5590": 28158672896.0,
+ "5595": 28158672896.0,
+ "5600": 28158672896.0,
+ "5605": 28158672896.0,
+ "5610": 28158672896.0,
+ "5615": 28158672896.0,
+ "5620": 28158672896.0,
+ "5625": 28158672896.0,
+ "5630": 28158672896.0,
+ "5635": 28158672896.0,
+ "5640": 28158672896.0,
+ "5645": 28158672896.0,
+ "5650": 28158672896.0,
+ "5655": 28158672896.0,
+ "5660": 28158672896.0,
+ "5665": 28158672896.0,
+ "5670": 28158672896.0,
+ "5675": 28158672896.0,
+ "5680": 28158672896.0,
+ "5685": 28158672896.0,
+ "5690": 28158672896.0,
+ "5695": 28158672896.0,
+ "5700": 28158672896.0,
+ "5705": 28158672896.0,
+ "5710": 28158672896.0,
+ "5715": 28158672896.0,
+ "5720": 28158672896.0,
+ "5725": 28158672896.0,
+ "5730": 28158672896.0,
+ "5735": 28158672896.0,
+ "5740": 28158672896.0,
+ "5745": 28158672896.0,
+ "5750": 28158672896.0,
+ "5755": 28158672896.0,
+ "5760": 28158672896.0,
+ "5765": 28158672896.0,
+ "5770": 28158672896.0,
+ "5775": 28158672896.0,
+ "5780": 28158672896.0,
+ "5785": 28158672896.0,
+ "5790": 28158672896.0,
+ "5795": 28158672896.0,
+ "5800": 28158672896.0,
+ "5805": 28158672896.0,
+ "5810": 28158672896.0,
+ "5815": 28158672896.0,
+ "5820": 28158672896.0,
+ "5825": 28158672896.0,
+ "5830": 28158672896.0,
+ "5835": 28158672896.0,
+ "5840": 28158672896.0,
+ "5845": 28158672896.0,
+ "5850": 28158672896.0,
+ "5855": 28158672896.0,
+ "5860": 28158672896.0,
+ "5865": 28158672896.0,
+ "5870": 28158672896.0,
+ "5875": 28158672896.0,
+ "5880": 28158672896.0,
+ "5885": 28158672896.0,
+ "5890": 28158672896.0,
+ "5895": 28158672896.0,
+ "5900": 28158672896.0,
+ "5905": 28158672896.0,
+ "5910": 28158672896.0,
+ "5915": 28158672896.0,
+ "5920": 28158672896.0,
+ "5925": 28158672896.0,
+ "5930": 28158672896.0,
+ "5935": 28158672896.0,
+ "5940": 28158672896.0,
+ "5945": 28158672896.0,
+ "5950": 28158672896.0,
+ "5955": 28158672896.0,
+ "5960": 28158672896.0,
+ "5965": 28158672896.0,
+ "5970": 28158672896.0,
+ "5975": 28158672896.0,
+ "5980": 28158672896.0,
+ "5985": 28158672896.0,
+ "5990": 28158672896.0,
+ "5995": 28158672896.0,
+ "6000": 28158672896.0,
+ "6005": 28158672896.0,
+ "6010": 28158672896.0,
+ "6015": 28158672896.0,
+ "6020": 28158672896.0,
+ "6025": 28158672896.0,
+ "6030": 28158672896.0,
+ "6035": 28158672896.0,
+ "6040": 28158672896.0,
+ "6045": 28158672896.0,
+ "6050": 28158672896.0,
+ "6055": 28158672896.0,
+ "6060": 28158672896.0,
+ "6065": 28158672896.0,
+ "6070": 28158672896.0,
+ "6075": 28158672896.0,
+ "6080": 28158672896.0,
+ "6085": 28158672896.0,
+ "6090": 28158672896.0,
+ "6095": 28158672896.0,
+ "6100": 28158672896.0,
+ "6105": 28158672896.0,
+ "6110": 28158672896.0,
+ "6115": 28158672896.0,
+ "6120": 28158672896.0,
+ "6125": 28158672896.0,
+ "6130": 28158672896.0,
+ "6135": 28158672896.0,
+ "6140": 28158672896.0,
+ "6145": 28158672896.0,
+ "6150": 28158672896.0,
+ "6155": 28158672896.0,
+ "6160": 28158672896.0,
+ "6165": 28158672896.0,
+ "6170": 28158672896.0,
+ "6175": 28158672896.0,
+ "6180": 28158672896.0,
+ "6185": 28158672896.0,
+ "6190": 28158672896.0,
+ "6195": 28158672896.0,
+ "6200": 28158672896.0,
+ "6205": 28158672896.0,
+ "6210": 28158672896.0,
+ "6215": 28158672896.0,
+ "6220": 28158672896.0,
+ "6225": 28158672896.0,
+ "6230": 28158672896.0,
+ "6235": 28158672896.0,
+ "6240": 28158672896.0,
+ "6245": 28158672896.0,
+ "6250": 28158672896.0,
+ "6255": 28158672896.0,
+ "6260": 28158672896.0,
+ "6265": 28158672896.0,
+ "6270": 28158672896.0,
+ "6275": 28158672896.0,
+ "6280": 28158672896.0,
+ "6285": 28158672896.0,
+ "6290": 28158672896.0,
+ "6295": 28158672896.0,
+ "6300": 28158672896.0,
+ "6305": 28158672896.0,
+ "6310": 28158672896.0,
+ "6315": 28158672896.0,
+ "6320": 28158672896.0,
+ "6325": 28158672896.0,
+ "6330": 28158672896.0,
+ "6335": 28158672896.0,
+ "6340": 28158672896.0,
+ "6345": 28158672896.0,
+ "6350": 28158672896.0,
+ "6355": 28158672896.0,
+ "6360": 28158672896.0,
+ "6365": 28158672896.0,
+ "6370": 28158672896.0,
+ "6375": 28158672896.0,
+ "6380": 28158672896.0,
+ "6385": 28158672896.0,
+ "6390": 28158672896.0,
+ "6395": 28158672896.0,
+ "6400": 28158672896.0,
+ "6405": 28158672896.0,
+ "6410": 28158672896.0,
+ "6415": 28158672896.0,
+ "6420": 28158672896.0,
+ "6425": 28158672896.0,
+ "6430": 28158672896.0,
+ "6435": 28158672896.0,
+ "6440": 28158672896.0,
+ "6445": 28158672896.0,
+ "6450": 28158672896.0,
+ "6455": 28158672896.0,
+ "6460": 28158672896.0,
+ "6465": 28158672896.0,
+ "6470": 28158672896.0,
+ "6475": 28158672896.0,
+ "6480": 28158672896.0,
+ "6485": 28158672896.0,
+ "6490": 28158672896.0,
+ "6495": 28158672896.0,
+ "6500": 28158672896.0,
+ "6505": 28158672896.0,
+ "6510": 28158672896.0,
+ "6515": 28158672896.0,
+ "6520": 28158672896.0,
+ "6525": 28158672896.0,
+ "6530": 28158672896.0,
+ "6535": 28158672896.0,
+ "6540": 28158672896.0,
+ "6545": 28158672896.0,
+ "6550": 28158672896.0,
+ "6555": 28158672896.0,
+ "6560": 28158672896.0,
+ "6565": 28158672896.0,
+ "6570": 28158672896.0,
+ "6575": 28158672896.0,
+ "6580": 28158672896.0,
+ "6585": 28158672896.0,
+ "6590": 28158672896.0,
+ "6595": 28158672896.0,
+ "6600": 28158672896.0,
+ "6605": 28158672896.0,
+ "6610": 28158672896.0,
+ "6615": 28158672896.0,
+ "6620": 28158672896.0,
+ "6625": 28158672896.0,
+ "6630": 28158672896.0,
+ "6635": 28158672896.0,
+ "6640": 28158672896.0,
+ "6645": 28158672896.0,
+ "6650": 28158672896.0,
+ "6655": 28158672896.0,
+ "6660": 28158672896.0,
+ "6665": 28158672896.0,
+ "6670": 28158672896.0,
+ "6675": 28158672896.0,
+ "6680": 28158672896.0,
+ "6685": 28158672896.0,
+ "6690": 28158672896.0,
+ "6695": 28158672896.0,
+ "6700": 28158672896.0,
+ "6705": 28158672896.0,
+ "6710": 28158672896.0,
+ "6715": 28158672896.0,
+ "6720": 28158672896.0,
+ "6725": 28158672896.0,
+ "6730": 28158672896.0,
+ "6735": 28158672896.0,
+ "6740": 28158672896.0,
+ "6745": 28158672896.0,
+ "6750": 28158672896.0,
+ "6755": 28158672896.0,
+ "6760": 28158672896.0,
+ "6765": 28158672896.0,
+ "6770": 28158672896.0,
+ "6775": 28158672896.0,
+ "6780": 28158672896.0,
+ "6785": 28158672896.0,
+ "6790": 28158672896.0,
+ "6795": 28158672896.0,
+ "6800": 28158672896.0,
+ "6805": 28158672896.0,
+ "6810": 28158672896.0,
+ "6815": 28158672896.0,
+ "6820": 28158672896.0,
+ "6825": 28158672896.0,
+ "6830": 28158672896.0,
+ "6835": 28158672896.0,
+ "6840": 28158672896.0,
+ "6845": 28158672896.0,
+ "6850": 28158672896.0,
+ "6855": 28158672896.0,
+ "6860": 28158672896.0,
+ "6865": 28158672896.0,
+ "6870": 28158672896.0,
+ "6875": 28158672896.0,
+ "6880": 28158672896.0,
+ "6885": 28158672896.0,
+ "6890": 28158672896.0,
+ "6895": 28158672896.0,
+ "6900": 28158672896.0,
+ "6905": 28158672896.0,
+ "6910": 28158672896.0,
+ "6915": 28158672896.0,
+ "6920": 28158672896.0,
+ "6925": 28158672896.0,
+ "6930": 28158672896.0,
+ "6935": 28158672896.0,
+ "6940": 28158672896.0,
+ "6945": 28158672896.0,
+ "6950": 28158672896.0,
+ "6955": 28158672896.0,
+ "6960": 28158672896.0,
+ "6965": 28158672896.0,
+ "6970": 28158672896.0,
+ "6975": 28158672896.0,
+ "6980": 28158672896.0,
+ "6985": 28158672896.0,
+ "6990": 28158672896.0,
+ "6995": 28158672896.0,
+ "7000": 28158672896.0,
+ "7005": 28158672896.0,
+ "7010": 28158672896.0,
+ "7015": 28158672896.0,
+ "7020": 28158672896.0,
+ "7025": 28158672896.0,
+ "7030": 28158672896.0,
+ "7035": 28158672896.0,
+ "7040": 28158672896.0,
+ "7045": 28158672896.0,
+ "7050": 28158672896.0,
+ "7055": 28158672896.0,
+ "7060": 28158672896.0,
+ "7065": 28158672896.0,
+ "7070": 28158672896.0,
+ "7075": 28158672896.0,
+ "7080": 28158672896.0,
+ "7085": 28158672896.0,
+ "7090": 28158672896.0,
+ "7095": 28158672896.0,
+ "7100": 28158672896.0,
+ "7105": 28158672896.0,
+ "7110": 28158672896.0,
+ "7115": 28158672896.0,
+ "7120": 28158672896.0,
+ "7125": 28158672896.0,
+ "7130": 28158672896.0,
+ "7135": 28158672896.0,
+ "7140": 28158672896.0,
+ "7145": 28158672896.0,
+ "7150": 28158672896.0,
+ "7155": 28158672896.0,
+ "7160": 28158672896.0,
+ "7165": 28158672896.0,
+ "7170": 28158672896.0,
+ "7175": 28158672896.0,
+ "7180": 28158672896.0,
+ "7185": 28158672896.0,
+ "7190": 28158672896.0,
+ "7195": 28158672896.0,
+ "7200": 28158672896.0,
+ "7205": 28158672896.0,
+ "7210": 28158672896.0,
+ "7215": 28158672896.0,
+ "7220": 28158672896.0,
+ "7225": 28158672896.0,
+ "7230": 28158672896.0,
+ "7235": 28158672896.0,
+ "7240": 28158672896.0,
+ "7245": 28158672896.0,
+ "7250": 28158672896.0,
+ "7255": 28158672896.0,
+ "7260": 28158672896.0,
+ "7265": 28158672896.0,
+ "7270": 28158672896.0,
+ "7275": 28158672896.0,
+ "7280": 28158672896.0,
+ "7285": 28158672896.0,
+ "7290": 28158672896.0,
+ "7295": 28158672896.0,
+ "7300": 28158672896.0,
+ "7305": 28158672896.0,
+ "7310": 28158672896.0,
+ "7315": 28158672896.0,
+ "7320": 28158672896.0,
+ "7325": 28158672896.0,
+ "7330": 28158672896.0,
+ "7335": 28158672896.0,
+ "7340": 28158672896.0,
+ "7345": 28158672896.0,
+ "7350": 28158672896.0,
+ "7355": 28158672896.0,
+ "7360": 28158672896.0,
+ "7365": 28158672896.0,
+ "7370": 28158672896.0,
+ "7375": 28158672896.0,
+ "7380": 28158672896.0,
+ "7385": 28158672896.0,
+ "7390": 28158672896.0,
+ "7395": 28158672896.0,
+ "7400": 28158672896.0,
+ "7405": 28158672896.0,
+ "7410": 28158672896.0,
+ "7415": 28158672896.0,
+ "7420": 28158672896.0,
+ "7425": 28158672896.0,
+ "7430": 28158672896.0,
+ "7435": 28158672896.0,
+ "7440": 28158672896.0,
+ "7445": 28158672896.0,
+ "7450": 28158672896.0,
+ "7455": 28158672896.0,
+ "7460": 28158672896.0,
+ "7465": 28158672896.0,
+ "7470": 28158672896.0,
+ "7475": 28158672896.0,
+ "7480": 28158672896.0,
+ "7485": 28158672896.0,
+ "7490": 28158672896.0,
+ "7495": 28158672896.0,
+ "7500": 28158672896.0,
+ "7505": 28158672896.0,
+ "7510": 28158672896.0,
+ "7515": 28158672896.0,
+ "7520": 28158672896.0,
+ "7525": 28158672896.0,
+ "7530": 28158672896.0,
+ "7535": 28158672896.0,
+ "7540": 28158672896.0,
+ "7545": 28158672896.0,
+ "7550": 28158672896.0,
+ "7555": 28158672896.0,
+ "7560": 28158672896.0,
+ "7565": 28158672896.0,
+ "7570": 28158672896.0,
+ "7575": 28158672896.0,
+ "7580": 28158672896.0,
+ "7585": 28158672896.0,
+ "7590": 28158672896.0,
+ "7595": 28158672896.0,
+ "7600": 28158672896.0,
+ "7605": 28158672896.0,
+ "7610": 28158672896.0,
+ "7615": 28158672896.0,
+ "7620": 28158672896.0,
+ "7625": 28158672896.0,
+ "7630": 28158672896.0,
+ "7635": 28158672896.0,
+ "7640": 28158672896.0,
+ "7645": 28158672896.0,
+ "7650": 28158672896.0,
+ "7655": 28158672896.0,
+ "7660": 28158672896.0,
+ "7665": 28158672896.0,
+ "7670": 28158672896.0,
+ "7675": 28158672896.0,
+ "7680": 28158672896.0,
+ "7685": 28158672896.0,
+ "7690": 28158672896.0,
+ "7695": 28158672896.0,
+ "7700": 28158672896.0,
+ "7705": 28158672896.0,
+ "7710": 28158672896.0,
+ "7715": 28158672896.0,
+ "7720": 28158672896.0,
+ "7725": 28158672896.0,
+ "7730": 28158672896.0,
+ "7735": 28158672896.0,
+ "7740": 28158672896.0,
+ "7745": 28158672896.0,
+ "7750": 28158672896.0,
+ "7755": 28158672896.0,
+ "7760": 28158672896.0,
+ "7765": 28158672896.0,
+ "7770": 28158672896.0,
+ "7775": 28158672896.0,
+ "7780": 28158672896.0,
+ "7785": 28158672896.0,
+ "7790": 28158672896.0,
+ "7795": 28158672896.0,
+ "7800": 28158672896.0,
+ "7805": 28158672896.0,
+ "7810": 28158672896.0,
+ "7815": 28158672896.0,
+ "7820": 28158672896.0,
+ "7825": 28158672896.0,
+ "7830": 28158672896.0,
+ "7835": 28158672896.0,
+ "7840": 28158672896.0,
+ "7845": 28158672896.0,
+ "7850": 28158672896.0,
+ "7855": 28158672896.0,
+ "7860": 28158672896.0,
+ "7865": 28158672896.0,
+ "7870": 28158672896.0,
+ "7875": 28158672896.0,
+ "7880": 28158672896.0,
+ "7885": 28158672896.0,
+ "7890": 28158672896.0,
+ "7895": 28158672896.0,
+ "7900": 28158672896.0,
+ "7905": 28158672896.0,
+ "7910": 28158672896.0,
+ "7915": 28158672896.0,
+ "7920": 28158672896.0,
+ "7925": 28158672896.0,
+ "7930": 28158672896.0,
+ "7935": 28158672896.0,
+ "7940": 28158672896.0,
+ "7945": 28158672896.0,
+ "7950": 28158672896.0,
+ "7955": 28158672896.0,
+ "7960": 28158672896.0,
+ "7965": 28158672896.0,
+ "7970": 28158672896.0,
+ "7975": 28158672896.0,
+ "7980": 28158672896.0,
+ "7985": 28158672896.0,
+ "7990": 28158672896.0,
+ "7995": 28158672896.0,
+ "8000": 28158672896.0,
+ "8005": 28158672896.0,
+ "8010": 28158672896.0,
+ "8015": 28158672896.0,
+ "8020": 28158672896.0,
+ "8025": 28158672896.0,
+ "8030": 28158672896.0,
+ "8035": 28158672896.0,
+ "8040": 28158672896.0,
+ "8045": 28158672896.0,
+ "8050": 28158672896.0,
+ "8055": 28158672896.0,
+ "8060": 28158672896.0,
+ "8065": 28158672896.0,
+ "8070": 28158672896.0,
+ "8075": 28158672896.0,
+ "8080": 28158672896.0,
+ "8085": 28158672896.0,
+ "8090": 28158672896.0,
+ "8095": 28158672896.0,
+ "8100": 28158672896.0,
+ "8105": 28158672896.0,
+ "8110": 28158672896.0,
+ "8115": 28158672896.0,
+ "8120": 28158672896.0,
+ "8125": 28158672896.0,
+ "8130": 28158672896.0,
+ "8135": 28158672896.0,
+ "8140": 28158672896.0,
+ "8145": 28158672896.0,
+ "8150": 28158672896.0,
+ "8155": 28158672896.0,
+ "8160": 28158672896.0,
+ "8165": 28158672896.0,
+ "8170": 28158672896.0,
+ "8175": 28158672896.0,
+ "8180": 28158672896.0,
+ "8185": 28158672896.0,
+ "8190": 28158672896.0,
+ "8195": 28158672896.0,
+ "8200": 28158672896.0,
+ "8205": 28158672896.0,
+ "8210": 28158672896.0,
+ "8215": 28158672896.0,
+ "8220": 28158672896.0,
+ "8225": 28158672896.0,
+ "8230": 28158672896.0,
+ "8235": 28158672896.0,
+ "8240": 28158672896.0,
+ "8245": 28158672896.0,
+ "8250": 28158672896.0,
+ "8255": 28158672896.0,
+ "8260": 28158672896.0,
+ "8265": 28158672896.0,
+ "8270": 28158672896.0,
+ "8275": 28158672896.0,
+ "8280": 28158672896.0,
+ "8285": 28158672896.0,
+ "8290": 28158672896.0,
+ "8295": 28158672896.0,
+ "8300": 28158672896.0,
+ "8305": 28158672896.0,
+ "8310": 28158672896.0,
+ "8315": 28158672896.0,
+ "8320": 28158672896.0,
+ "8325": 28158672896.0,
+ "8330": 28158672896.0,
+ "8335": 28158672896.0,
+ "8340": 28158672896.0,
+ "8345": 28158672896.0,
+ "8350": 28158672896.0,
+ "8355": 28158672896.0,
+ "8360": 28158672896.0,
+ "8365": 28158672896.0,
+ "8370": 28158672896.0,
+ "8375": 28158672896.0,
+ "8380": 28158672896.0,
+ "8385": 28158672896.0,
+ "8390": 28158672896.0,
+ "8395": 28158672896.0,
+ "8400": 28158672896.0,
+ "8405": 28158672896.0,
+ "8410": 28158672896.0,
+ "8415": 28158672896.0,
+ "8420": 28158672896.0,
+ "8425": 28158672896.0,
+ "8430": 28158672896.0,
+ "8435": 28158672896.0,
+ "8440": 28158672896.0,
+ "8445": 28158672896.0,
+ "8450": 28158672896.0,
+ "8455": 28158672896.0,
+ "8460": 28158672896.0,
+ "8465": 28158672896.0,
+ "8470": 28158672896.0,
+ "8475": 28158672896.0,
+ "8480": 28158672896.0,
+ "8485": 28158672896.0,
+ "8490": 28158672896.0,
+ "8495": 28158672896.0,
+ "8500": 28158672896.0,
+ "8505": 28158672896.0,
+ "8510": 28158672896.0,
+ "8515": 28158672896.0,
+ "8520": 28158672896.0,
+ "8525": 28158672896.0,
+ "8530": 28158672896.0,
+ "8535": 28158672896.0,
+ "8540": 28158672896.0,
+ "8545": 28158672896.0,
+ "8550": 28158672896.0,
+ "8555": 28158672896.0,
+ "8560": 28158672896.0,
+ "8565": 28158672896.0,
+ "8570": 28158672896.0,
+ "8575": 28158672896.0,
+ "8580": 28158672896.0,
+ "8585": 28158672896.0,
+ "8590": 28158672896.0,
+ "8595": 28158672896.0,
+ "8600": 28158672896.0,
+ "8605": 28158672896.0,
+ "8610": 28158672896.0,
+ "8615": 28158672896.0,
+ "8620": 28158672896.0,
+ "8625": 28158672896.0,
+ "8630": 28158672896.0,
+ "8635": 28158672896.0,
+ "8640": 28158672896.0,
+ "8645": 28158672896.0,
+ "8650": 28158672896.0,
+ "8655": 28158672896.0,
+ "8660": 28158672896.0,
+ "8665": 28158672896.0,
+ "8670": 28158672896.0,
+ "8675": 28158672896.0,
+ "8680": 28158672896.0,
+ "8685": 28158672896.0,
+ "8690": 28158672896.0,
+ "8695": 28158672896.0,
+ "8700": 28158672896.0,
+ "8705": 28158672896.0,
+ "8710": 28158672896.0,
+ "8715": 28158672896.0,
+ "8720": 28158672896.0,
+ "8725": 28158672896.0,
+ "8730": 28158672896.0,
+ "8735": 28158672896.0,
+ "8740": 28158672896.0,
+ "8745": 28158672896.0,
+ "8750": 28158672896.0,
+ "8755": 28158672896.0,
+ "8760": 28158672896.0,
+ "8765": 28158672896.0,
+ "8770": 28158672896.0,
+ "8775": 28158672896.0,
+ "8780": 28158672896.0,
+ "8785": 28158672896.0,
+ "8790": 28158672896.0,
+ "8795": 28158672896.0,
+ "8800": 28158672896.0,
+ "8805": 28158672896.0,
+ "8810": 28156907520.0,
+ "8815": 28156907520.0,
+ "8820": 28156907520.0,
+ "8825": 28156907520.0,
+ "8830": 28156907520.0,
+ "8835": 28156907520.0,
+ "8840": 28156907520.0,
+ "8845": 28156907520.0,
+ "8850": 28156907520.0,
+ "8855": 28156907520.0,
+ "8860": 28156907520.0,
+ "8865": 28156907520.0,
+ "8870": 28156907520.0,
+ "8875": 28156907520.0,
+ "8880": 28156907520.0,
+ "8885": 28156907520.0,
+ "8890": 28156907520.0,
+ "8895": 28156907520.0,
+ "8900": 28156907520.0,
+ "8905": 28156907520.0,
+ "8910": 28156907520.0,
+ "8915": 28156907520.0,
+ "8920": 28156907520.0,
+ "8925": 28156907520.0,
+ "8930": 28156907520.0,
+ "8935": 28156907520.0,
+ "8940": 28156907520.0,
+ "8945": 28156907520.0,
+ "8950": 28156907520.0,
+ "8955": 28156907520.0,
+ "8960": 28156907520.0,
+ "8965": 28156907520.0,
+ "8970": 28156907520.0,
+ "8975": 28156907520.0,
+ "8980": 28156907520.0,
+ "8985": 28156907520.0,
+ "8990": 28156907520.0,
+ "8995": 28156907520.0,
+ "9000": 28156907520.0,
+ "9005": 28156907520.0,
+ "9010": 28156907520.0,
+ "9015": 28156907520.0,
+ "9020": 28156907520.0,
+ "9025": 28156907520.0,
+ "9030": 28156907520.0,
+ "9035": 28156907520.0,
+ "9040": 28156907520.0,
+ "9045": 28156907520.0,
+ "9050": 28156907520.0,
+ "9055": 28156907520.0,
+ "9060": 28156907520.0,
+ "9065": 28156907520.0,
+ "9070": 28156907520.0,
+ "9075": 28156907520.0,
+ "9080": 28156907520.0,
+ "9085": 28156907520.0,
+ "9090": 28156907520.0,
+ "9095": 28156907520.0,
+ "9100": 28156907520.0,
+ "9105": 28156907520.0,
+ "9110": 28156907520.0,
+ "9115": 28156907520.0,
+ "9120": 28156907520.0,
+ "9125": 28156907520.0,
+ "9130": 28156907520.0,
+ "9135": 28156907520.0,
+ "9140": 28156907520.0,
+ "9145": 28156907520.0,
+ "9150": 28156907520.0,
+ "9155": 28156907520.0,
+ "9160": 28156907520.0,
+ "9165": 28156907520.0,
+ "9170": 28156907520.0,
+ "9175": 28156907520.0,
+ "9180": 28156907520.0,
+ "9185": 28156907520.0,
+ "9190": 28156907520.0,
+ "9195": 28156907520.0,
+ "9200": 28156907520.0,
+ "9205": 28156907520.0,
+ "9210": 28156907520.0,
+ "9215": 28156907520.0,
+ "9220": 28156907520.0,
+ "9225": 28156907520.0,
+ "9230": 28156907520.0,
+ "9235": 28156907520.0,
+ "9240": 28156907520.0,
+ "9245": 28156907520.0,
+ "9250": 28156907520.0,
+ "9255": 28156907520.0,
+ "9260": 28156907520.0,
+ "9265": 28156907520.0,
+ "9270": 28156907520.0,
+ "9275": 28156907520.0,
+ "9280": 28156907520.0,
+ "9285": 28156907520.0,
+ "9290": 28156907520.0,
+ "9295": 28156907520.0,
+ "9300": 28156907520.0,
+ "9305": 28156907520.0,
+ "9310": 28156907520.0,
+ "9315": 28156907520.0,
+ "9320": 28156907520.0,
+ "9325": 28156907520.0,
+ "9330": 28156907520.0,
+ "9335": 28156907520.0,
+ "9340": 28156907520.0,
+ "9345": 28156907520.0,
+ "9350": 28156907520.0,
+ "9355": 28156907520.0,
+ "9360": 28156907520.0,
+ "9365": 28156907520.0,
+ "9370": 28156907520.0,
+ "9375": 28156907520.0,
+ "9380": 28156907520.0,
+ "9385": 28156907520.0,
+ "9390": 28156907520.0,
+ "9395": 28156907520.0,
+ "9400": 28156907520.0,
+ "9405": 28156907520.0,
+ "9410": 28156907520.0,
+ "9415": 28156907520.0,
+ "9420": 28156907520.0,
+ "9425": 28156907520.0,
+ "9430": 28156907520.0,
+ "9435": 28156907520.0,
+ "9440": 28156907520.0,
+ "9445": 28156907520.0,
+ "9450": 28156907520.0,
+ "9455": 28156907520.0,
+ "9460": 28156907520.0,
+ "9465": 28156907520.0,
+ "9470": 28156907520.0,
+ "9475": 28156907520.0,
+ "9480": 28156907520.0,
+ "9485": 28156907520.0,
+ "9490": 28156907520.0,
+ "9495": 28156907520.0,
+ "9500": 28156907520.0,
+ "9505": 28156907520.0,
+ "9510": 28156907520.0,
+ "9515": 28156907520.0,
+ "9520": 28156907520.0,
+ "9525": 28156907520.0,
+ "9530": 28156907520.0,
+ "9535": 28156907520.0,
+ "9540": 28156907520.0,
+ "9545": 28156907520.0,
+ "9550": 28156907520.0,
+ "9555": 28156907520.0,
+ "9560": 28156907520.0,
+ "9565": 28156907520.0,
+ "9570": 28156907520.0,
+ "9575": 28156907520.0,
+ "9580": 28156907520.0,
+ "9585": 28156907520.0,
+ "9590": 28156907520.0,
+ "9595": 28156907520.0,
+ "9600": 28156907520.0,
+ "9605": 28156907520.0,
+ "9610": 28156907520.0,
+ "9615": 28156907520.0,
+ "9620": 28156907520.0,
+ "9625": 28156907520.0,
+ "9630": 28156907520.0,
+ "9635": 28156907520.0,
+ "9640": 28156907520.0,
+ "9645": 28156907520.0,
+ "9650": 28156907520.0,
+ "9655": 28156907520.0,
+ "9660": 28156907520.0,
+ "9665": 28156907520.0,
+ "9670": 28156907520.0,
+ "9675": 28156907520.0,
+ "9680": 28156907520.0,
+ "9685": 28156907520.0,
+ "9690": 28156907520.0,
+ "9695": 28156907520.0,
+ "9700": 28156907520.0,
+ "9705": 28156907520.0,
+ "9710": 28156907520.0,
+ "9715": 28156907520.0,
+ "9720": 28156907520.0,
+ "9725": 28156907520.0,
+ "9730": 28156907520.0,
+ "9735": 28156907520.0,
+ "9740": 28156907520.0,
+ "9745": 28156907520.0,
+ "9750": 28156907520.0,
+ "9755": 28156907520.0,
+ "9760": 28156907520.0,
+ "9765": 28156907520.0,
+ "9770": 28156907520.0,
+ "9775": 28156907520.0,
+ "9780": 28156907520.0,
+ "9785": 28156907520.0,
+ "9790": 28156907520.0,
+ "9795": 28156907520.0,
+ "9800": 28156907520.0,
+ "9805": 28156907520.0,
+ "9810": 28156907520.0,
+ "9815": 28156907520.0,
+ "9820": 28156907520.0,
+ "9825": 28156907520.0,
+ "9830": 28156907520.0,
+ "9835": 28156907520.0,
+ "9840": 28156907520.0,
+ "9845": 28156907520.0,
+ "9850": 28156907520.0,
+ "9855": 28156907520.0,
+ "9860": 28156907520.0,
+ "9865": 28156907520.0,
+ "9870": 28156907520.0,
+ "9875": 28156907520.0,
+ "9880": 28156907520.0,
+ "9885": 28156907520.0,
+ "9890": 28156907520.0,
+ "9895": 28156907520.0,
+ "9900": 28156907520.0,
+ "9905": 28156907520.0,
+ "9910": 28156907520.0,
+ "9915": 28156907520.0,
+ "9920": 28156907520.0,
+ "9925": 28156907520.0,
+ "9930": 28156907520.0,
+ "9935": 28156907520.0,
+ "9940": 28156907520.0,
+ "9945": 28156907520.0,
+ "9950": 28156907520.0,
+ "9955": 28156907520.0,
+ "9960": 28156907520.0,
+ "9965": 28156907520.0,
+ "9970": 28156907520.0,
+ "9975": 28156907520.0,
+ "9980": 28156907520.0,
+ "9985": 28156907520.0,
+ "9990": 28156907520.0,
+ "9995": 28156907520.0,
+ "10000": 28156907520.0,
+ "10005": 28156907520.0,
+ "10010": 28156907520.0,
+ "10015": 28156907520.0,
+ "10020": 28156907520.0,
+ "10025": 28156907520.0,
+ "10030": 28156907520.0,
+ "10035": 28156907520.0,
+ "10040": 28156907520.0,
+ "10045": 28156907520.0,
+ "10050": 28156907520.0,
+ "10055": 28156907520.0,
+ "10060": 28156907520.0,
+ "10065": 28156907520.0,
+ "10070": 28156907520.0,
+ "10075": 28156907520.0,
+ "10080": 28156907520.0,
+ "10085": 28156907520.0,
+ "10090": 28156907520.0,
+ "10095": 28156907520.0,
+ "10100": 28156907520.0,
+ "10105": 28156907520.0,
+ "10110": 28156907520.0,
+ "10115": 28156907520.0,
+ "10120": 28156907520.0,
+ "10125": 28156907520.0,
+ "10130": 28156907520.0,
+ "10135": 28156907520.0,
+ "10140": 28156907520.0,
+ "10145": 28156907520.0,
+ "10150": 28156907520.0,
+ "10155": 28156907520.0,
+ "10160": 28156907520.0,
+ "10165": 28156907520.0,
+ "10170": 28156907520.0,
+ "10175": 28156907520.0,
+ "10180": 28156907520.0,
+ "10185": 28156907520.0,
+ "10190": 28156907520.0,
+ "10195": 28156907520.0,
+ "10200": 28156907520.0
+ }
+ },
+ "iteration-time": {
+ "start_step": 1,
+ "end_step": 10200,
+ "step_interval": 5,
+ "values": {
+ "1": "nan",
+ "5": "nan",
+ "10": "nan",
+ "15": "nan",
+ "20": "nan",
+ "25": "nan",
+ "30": "nan",
+ "35": "nan",
+ "40": "nan",
+ "45": "nan",
+ "50": "nan",
+ "55": "nan",
+ "60": "nan",
+ "65": "nan",
+ "70": "nan",
+ "75": "nan",
+ "80": "nan",
+ "85": "nan",
+ "90": "nan",
+ "95": "nan",
+ "100": 1.72893,
+ "105": "nan",
+ "110": "nan",
+ "115": "nan",
+ "120": "nan",
+ "125": "nan",
+ "130": "nan",
+ "135": "nan",
+ "140": "nan",
+ "145": "nan",
+ "150": "nan",
+ "155": "nan",
+ "160": "nan",
+ "165": "nan",
+ "170": "nan",
+ "175": "nan",
+ "180": "nan",
+ "185": "nan",
+ "190": "nan",
+ "195": "nan",
+ "200": 1.43857,
+ "205": "nan",
+ "210": "nan",
+ "215": "nan",
+ "220": "nan",
+ "225": "nan",
+ "230": "nan",
+ "235": "nan",
+ "240": "nan",
+ "245": "nan",
+ "250": "nan",
+ "255": "nan",
+ "260": "nan",
+ "265": "nan",
+ "270": "nan",
+ "275": "nan",
+ "280": "nan",
+ "285": "nan",
+ "290": "nan",
+ "295": "nan",
+ "300": 1.43584,
+ "305": "nan",
+ "310": "nan",
+ "315": "nan",
+ "320": "nan",
+ "325": "nan",
+ "330": "nan",
+ "335": "nan",
+ "340": "nan",
+ "345": "nan",
+ "350": "nan",
+ "355": "nan",
+ "360": "nan",
+ "365": "nan",
+ "370": "nan",
+ "375": "nan",
+ "380": "nan",
+ "385": "nan",
+ "390": "nan",
+ "395": "nan",
+ "400": 1.43883,
+ "405": "nan",
+ "410": "nan",
+ "415": "nan",
+ "420": "nan",
+ "425": "nan",
+ "430": "nan",
+ "435": "nan",
+ "440": "nan",
+ "445": "nan",
+ "450": "nan",
+ "455": "nan",
+ "460": "nan",
+ "465": "nan",
+ "470": "nan",
+ "475": "nan",
+ "480": "nan",
+ "485": "nan",
+ "490": "nan",
+ "495": "nan",
+ "500": 1.43781,
+ "505": "nan",
+ "510": "nan",
+ "515": "nan",
+ "520": "nan",
+ "525": "nan",
+ "530": "nan",
+ "535": "nan",
+ "540": "nan",
+ "545": "nan",
+ "550": "nan",
+ "555": "nan",
+ "560": "nan",
+ "565": "nan",
+ "570": "nan",
+ "575": "nan",
+ "580": "nan",
+ "585": "nan",
+ "590": "nan",
+ "595": "nan",
+ "600": 1.43827,
+ "605": "nan",
+ "610": "nan",
+ "615": "nan",
+ "620": "nan",
+ "625": "nan",
+ "630": "nan",
+ "635": "nan",
+ "640": "nan",
+ "645": "nan",
+ "650": "nan",
+ "655": "nan",
+ "660": "nan",
+ "665": "nan",
+ "670": "nan",
+ "675": "nan",
+ "680": "nan",
+ "685": "nan",
+ "690": "nan",
+ "695": "nan",
+ "700": 1.4373,
+ "705": "nan",
+ "710": "nan",
+ "715": "nan",
+ "720": "nan",
+ "725": "nan",
+ "730": "nan",
+ "735": "nan",
+ "740": "nan",
+ "745": "nan",
+ "750": "nan",
+ "755": "nan",
+ "760": "nan",
+ "765": "nan",
+ "770": "nan",
+ "775": "nan",
+ "780": "nan",
+ "785": "nan",
+ "790": "nan",
+ "795": "nan",
+ "800": 1.43657,
+ "805": "nan",
+ "810": "nan",
+ "815": "nan",
+ "820": "nan",
+ "825": "nan",
+ "830": "nan",
+ "835": "nan",
+ "840": "nan",
+ "845": "nan",
+ "850": "nan",
+ "855": "nan",
+ "860": "nan",
+ "865": "nan",
+ "870": "nan",
+ "875": "nan",
+ "880": "nan",
+ "885": "nan",
+ "890": "nan",
+ "895": "nan",
+ "900": 1.44068,
+ "905": "nan",
+ "910": "nan",
+ "915": "nan",
+ "920": "nan",
+ "925": "nan",
+ "930": "nan",
+ "935": "nan",
+ "940": "nan",
+ "945": "nan",
+ "950": "nan",
+ "955": "nan",
+ "960": "nan",
+ "965": "nan",
+ "970": "nan",
+ "975": "nan",
+ "980": "nan",
+ "985": "nan",
+ "990": "nan",
+ "995": "nan",
+ "1000": 1.43583,
+ "1005": "nan",
+ "1010": "nan",
+ "1015": "nan",
+ "1020": "nan",
+ "1025": "nan",
+ "1030": "nan",
+ "1035": "nan",
+ "1040": "nan",
+ "1045": "nan",
+ "1050": "nan",
+ "1055": "nan",
+ "1060": "nan",
+ "1065": "nan",
+ "1070": "nan",
+ "1075": "nan",
+ "1080": "nan",
+ "1085": "nan",
+ "1090": "nan",
+ "1095": "nan",
+ "1100": 1.43072,
+ "1105": "nan",
+ "1110": "nan",
+ "1115": "nan",
+ "1120": "nan",
+ "1125": "nan",
+ "1130": "nan",
+ "1135": "nan",
+ "1140": "nan",
+ "1145": "nan",
+ "1150": "nan",
+ "1155": "nan",
+ "1160": "nan",
+ "1165": "nan",
+ "1170": "nan",
+ "1175": "nan",
+ "1180": "nan",
+ "1185": "nan",
+ "1190": "nan",
+ "1195": "nan",
+ "1200": 1.4292,
+ "1205": "nan",
+ "1210": "nan",
+ "1215": "nan",
+ "1220": "nan",
+ "1225": "nan",
+ "1230": "nan",
+ "1235": "nan",
+ "1240": "nan",
+ "1245": "nan",
+ "1250": "nan",
+ "1255": "nan",
+ "1260": "nan",
+ "1265": "nan",
+ "1270": "nan",
+ "1275": "nan",
+ "1280": "nan",
+ "1285": "nan",
+ "1290": "nan",
+ "1295": "nan",
+ "1300": 1.4269,
+ "1305": "nan",
+ "1310": "nan",
+ "1315": "nan",
+ "1320": "nan",
+ "1325": "nan",
+ "1330": "nan",
+ "1335": "nan",
+ "1340": "nan",
+ "1345": "nan",
+ "1350": "nan",
+ "1355": "nan",
+ "1360": "nan",
+ "1365": "nan",
+ "1370": "nan",
+ "1375": "nan",
+ "1380": "nan",
+ "1385": "nan",
+ "1390": "nan",
+ "1395": "nan",
+ "1400": 1.42766,
+ "1405": "nan",
+ "1410": "nan",
+ "1415": "nan",
+ "1420": "nan",
+ "1425": "nan",
+ "1430": "nan",
+ "1435": "nan",
+ "1440": "nan",
+ "1445": "nan",
+ "1450": "nan",
+ "1455": "nan",
+ "1460": "nan",
+ "1465": "nan",
+ "1470": "nan",
+ "1475": "nan",
+ "1480": "nan",
+ "1485": "nan",
+ "1490": "nan",
+ "1495": "nan",
+ "1500": 1.42106,
+ "1505": "nan",
+ "1510": "nan",
+ "1515": "nan",
+ "1520": "nan",
+ "1525": "nan",
+ "1530": "nan",
+ "1535": "nan",
+ "1540": "nan",
+ "1545": "nan",
+ "1550": "nan",
+ "1555": "nan",
+ "1560": "nan",
+ "1565": "nan",
+ "1570": "nan",
+ "1575": "nan",
+ "1580": "nan",
+ "1585": "nan",
+ "1590": "nan",
+ "1595": "nan",
+ "1600": 1.42409,
+ "1605": "nan",
+ "1610": "nan",
+ "1615": "nan",
+ "1620": "nan",
+ "1625": "nan",
+ "1630": "nan",
+ "1635": "nan",
+ "1640": "nan",
+ "1645": "nan",
+ "1650": "nan",
+ "1655": "nan",
+ "1660": "nan",
+ "1665": "nan",
+ "1670": "nan",
+ "1675": "nan",
+ "1680": "nan",
+ "1685": "nan",
+ "1690": "nan",
+ "1695": "nan",
+ "1700": 1.42073,
+ "1705": "nan",
+ "1710": "nan",
+ "1715": "nan",
+ "1720": "nan",
+ "1725": "nan",
+ "1730": "nan",
+ "1735": "nan",
+ "1740": "nan",
+ "1745": "nan",
+ "1750": "nan",
+ "1755": "nan",
+ "1760": "nan",
+ "1765": "nan",
+ "1770": "nan",
+ "1775": "nan",
+ "1780": "nan",
+ "1785": "nan",
+ "1790": "nan",
+ "1795": "nan",
+ "1800": 1.42098,
+ "1805": "nan",
+ "1810": "nan",
+ "1815": "nan",
+ "1820": "nan",
+ "1825": "nan",
+ "1830": "nan",
+ "1835": "nan",
+ "1840": "nan",
+ "1845": "nan",
+ "1850": "nan",
+ "1855": "nan",
+ "1860": "nan",
+ "1865": "nan",
+ "1870": "nan",
+ "1875": "nan",
+ "1880": "nan",
+ "1885": "nan",
+ "1890": "nan",
+ "1895": "nan",
+ "1900": 1.42066,
+ "1905": "nan",
+ "1910": "nan",
+ "1915": "nan",
+ "1920": "nan",
+ "1925": "nan",
+ "1930": "nan",
+ "1935": "nan",
+ "1940": "nan",
+ "1945": "nan",
+ "1950": "nan",
+ "1955": "nan",
+ "1960": "nan",
+ "1965": "nan",
+ "1970": "nan",
+ "1975": "nan",
+ "1980": "nan",
+ "1985": "nan",
+ "1990": "nan",
+ "1995": "nan",
+ "2000": 1.42267,
+ "2005": "nan",
+ "2010": "nan",
+ "2015": "nan",
+ "2020": "nan",
+ "2025": "nan",
+ "2030": "nan",
+ "2035": "nan",
+ "2040": "nan",
+ "2045": "nan",
+ "2050": "nan",
+ "2055": "nan",
+ "2060": "nan",
+ "2065": "nan",
+ "2070": "nan",
+ "2075": "nan",
+ "2080": "nan",
+ "2085": "nan",
+ "2090": "nan",
+ "2095": "nan",
+ "2100": 1.42131,
+ "2105": "nan",
+ "2110": "nan",
+ "2115": "nan",
+ "2120": "nan",
+ "2125": "nan",
+ "2130": "nan",
+ "2135": "nan",
+ "2140": "nan",
+ "2145": "nan",
+ "2150": "nan",
+ "2155": "nan",
+ "2160": "nan",
+ "2165": "nan",
+ "2170": "nan",
+ "2175": "nan",
+ "2180": "nan",
+ "2185": "nan",
+ "2190": "nan",
+ "2195": "nan",
+ "2200": 1.41946,
+ "2205": "nan",
+ "2210": "nan",
+ "2215": "nan",
+ "2220": "nan",
+ "2225": "nan",
+ "2230": "nan",
+ "2235": "nan",
+ "2240": "nan",
+ "2245": "nan",
+ "2250": "nan",
+ "2255": "nan",
+ "2260": "nan",
+ "2265": "nan",
+ "2270": "nan",
+ "2275": "nan",
+ "2280": "nan",
+ "2285": "nan",
+ "2290": "nan",
+ "2295": "nan",
+ "2300": 1.42191,
+ "2305": "nan",
+ "2310": "nan",
+ "2315": "nan",
+ "2320": "nan",
+ "2325": "nan",
+ "2330": "nan",
+ "2335": "nan",
+ "2340": "nan",
+ "2345": "nan",
+ "2350": "nan",
+ "2355": "nan",
+ "2360": "nan",
+ "2365": "nan",
+ "2370": "nan",
+ "2375": "nan",
+ "2380": "nan",
+ "2385": "nan",
+ "2390": "nan",
+ "2395": "nan",
+ "2400": 1.42003,
+ "2405": "nan",
+ "2410": "nan",
+ "2415": "nan",
+ "2420": "nan",
+ "2425": "nan",
+ "2430": "nan",
+ "2435": "nan",
+ "2440": "nan",
+ "2445": "nan",
+ "2450": "nan",
+ "2455": "nan",
+ "2460": "nan",
+ "2465": "nan",
+ "2470": "nan",
+ "2475": "nan",
+ "2480": "nan",
+ "2485": "nan",
+ "2490": "nan",
+ "2495": "nan",
+ "2500": 1.41919,
+ "2505": "nan",
+ "2510": "nan",
+ "2515": "nan",
+ "2520": "nan",
+ "2525": "nan",
+ "2530": "nan",
+ "2535": "nan",
+ "2540": "nan",
+ "2545": "nan",
+ "2550": "nan",
+ "2555": "nan",
+ "2560": "nan",
+ "2565": "nan",
+ "2570": "nan",
+ "2575": "nan",
+ "2580": "nan",
+ "2585": "nan",
+ "2590": "nan",
+ "2595": "nan",
+ "2600": 1.41875,
+ "2605": "nan",
+ "2610": "nan",
+ "2615": "nan",
+ "2620": "nan",
+ "2625": "nan",
+ "2630": "nan",
+ "2635": "nan",
+ "2640": "nan",
+ "2645": "nan",
+ "2650": "nan",
+ "2655": "nan",
+ "2660": "nan",
+ "2665": "nan",
+ "2670": "nan",
+ "2675": "nan",
+ "2680": "nan",
+ "2685": "nan",
+ "2690": "nan",
+ "2695": "nan",
+ "2700": 1.41854,
+ "2705": "nan",
+ "2710": "nan",
+ "2715": "nan",
+ "2720": "nan",
+ "2725": "nan",
+ "2730": "nan",
+ "2735": "nan",
+ "2740": "nan",
+ "2745": "nan",
+ "2750": "nan",
+ "2755": "nan",
+ "2760": "nan",
+ "2765": "nan",
+ "2770": "nan",
+ "2775": "nan",
+ "2780": "nan",
+ "2785": "nan",
+ "2790": "nan",
+ "2795": "nan",
+ "2800": 1.41791,
+ "2805": "nan",
+ "2810": "nan",
+ "2815": "nan",
+ "2820": "nan",
+ "2825": "nan",
+ "2830": "nan",
+ "2835": "nan",
+ "2840": "nan",
+ "2845": "nan",
+ "2850": "nan",
+ "2855": "nan",
+ "2860": "nan",
+ "2865": "nan",
+ "2870": "nan",
+ "2875": "nan",
+ "2880": "nan",
+ "2885": "nan",
+ "2890": "nan",
+ "2895": "nan",
+ "2900": 1.41567,
+ "2905": "nan",
+ "2910": "nan",
+ "2915": "nan",
+ "2920": "nan",
+ "2925": "nan",
+ "2930": "nan",
+ "2935": "nan",
+ "2940": "nan",
+ "2945": "nan",
+ "2950": "nan",
+ "2955": "nan",
+ "2960": "nan",
+ "2965": "nan",
+ "2970": "nan",
+ "2975": "nan",
+ "2980": "nan",
+ "2985": "nan",
+ "2990": "nan",
+ "2995": "nan",
+ "3000": 1.4195,
+ "3005": "nan",
+ "3010": "nan",
+ "3015": "nan",
+ "3020": "nan",
+ "3025": "nan",
+ "3030": "nan",
+ "3035": "nan",
+ "3040": "nan",
+ "3045": "nan",
+ "3050": "nan",
+ "3055": "nan",
+ "3060": "nan",
+ "3065": "nan",
+ "3070": "nan",
+ "3075": "nan",
+ "3080": "nan",
+ "3085": "nan",
+ "3090": "nan",
+ "3095": "nan",
+ "3100": 1.42148,
+ "3105": "nan",
+ "3110": "nan",
+ "3115": "nan",
+ "3120": "nan",
+ "3125": "nan",
+ "3130": "nan",
+ "3135": "nan",
+ "3140": "nan",
+ "3145": "nan",
+ "3150": "nan",
+ "3155": "nan",
+ "3160": "nan",
+ "3165": "nan",
+ "3170": "nan",
+ "3175": "nan",
+ "3180": "nan",
+ "3185": "nan",
+ "3190": "nan",
+ "3195": "nan",
+ "3200": 1.41644,
+ "3205": "nan",
+ "3210": "nan",
+ "3215": "nan",
+ "3220": "nan",
+ "3225": "nan",
+ "3230": "nan",
+ "3235": "nan",
+ "3240": "nan",
+ "3245": "nan",
+ "3250": "nan",
+ "3255": "nan",
+ "3260": "nan",
+ "3265": "nan",
+ "3270": "nan",
+ "3275": "nan",
+ "3280": "nan",
+ "3285": "nan",
+ "3290": "nan",
+ "3295": "nan",
+ "3300": 1.41612,
+ "3305": "nan",
+ "3310": "nan",
+ "3315": "nan",
+ "3320": "nan",
+ "3325": "nan",
+ "3330": "nan",
+ "3335": "nan",
+ "3340": "nan",
+ "3345": "nan",
+ "3350": "nan",
+ "3355": "nan",
+ "3360": "nan",
+ "3365": "nan",
+ "3370": "nan",
+ "3375": "nan",
+ "3380": "nan",
+ "3385": "nan",
+ "3390": "nan",
+ "3395": "nan",
+ "3400": 1.41832,
+ "3405": "nan",
+ "3410": "nan",
+ "3415": "nan",
+ "3420": "nan",
+ "3425": "nan",
+ "3430": "nan",
+ "3435": "nan",
+ "3440": "nan",
+ "3445": "nan",
+ "3450": "nan",
+ "3455": "nan",
+ "3460": "nan",
+ "3465": "nan",
+ "3470": "nan",
+ "3475": "nan",
+ "3480": "nan",
+ "3485": "nan",
+ "3490": "nan",
+ "3495": "nan",
+ "3500": 1.41619,
+ "3505": "nan",
+ "3510": "nan",
+ "3515": "nan",
+ "3520": "nan",
+ "3525": "nan",
+ "3530": "nan",
+ "3535": "nan",
+ "3540": "nan",
+ "3545": "nan",
+ "3550": "nan",
+ "3555": "nan",
+ "3560": "nan",
+ "3565": "nan",
+ "3570": "nan",
+ "3575": "nan",
+ "3580": "nan",
+ "3585": "nan",
+ "3590": "nan",
+ "3595": "nan",
+ "3600": 1.41975,
+ "3605": "nan",
+ "3610": "nan",
+ "3615": "nan",
+ "3620": "nan",
+ "3625": "nan",
+ "3630": "nan",
+ "3635": "nan",
+ "3640": "nan",
+ "3645": "nan",
+ "3650": "nan",
+ "3655": "nan",
+ "3660": "nan",
+ "3665": "nan",
+ "3670": "nan",
+ "3675": "nan",
+ "3680": "nan",
+ "3685": "nan",
+ "3690": "nan",
+ "3695": "nan",
+ "3700": 1.41987,
+ "3705": "nan",
+ "3710": "nan",
+ "3715": "nan",
+ "3720": "nan",
+ "3725": "nan",
+ "3730": "nan",
+ "3735": "nan",
+ "3740": "nan",
+ "3745": "nan",
+ "3750": "nan",
+ "3755": "nan",
+ "3760": "nan",
+ "3765": "nan",
+ "3770": "nan",
+ "3775": "nan",
+ "3780": "nan",
+ "3785": "nan",
+ "3790": "nan",
+ "3795": "nan",
+ "3800": 1.41734,
+ "3805": "nan",
+ "3810": "nan",
+ "3815": "nan",
+ "3820": "nan",
+ "3825": "nan",
+ "3830": "nan",
+ "3835": "nan",
+ "3840": "nan",
+ "3845": "nan",
+ "3850": "nan",
+ "3855": "nan",
+ "3860": "nan",
+ "3865": "nan",
+ "3870": "nan",
+ "3875": "nan",
+ "3880": "nan",
+ "3885": "nan",
+ "3890": "nan",
+ "3895": "nan",
+ "3900": 1.41853,
+ "3905": "nan",
+ "3910": "nan",
+ "3915": "nan",
+ "3920": "nan",
+ "3925": "nan",
+ "3930": "nan",
+ "3935": "nan",
+ "3940": "nan",
+ "3945": "nan",
+ "3950": "nan",
+ "3955": "nan",
+ "3960": "nan",
+ "3965": "nan",
+ "3970": "nan",
+ "3975": "nan",
+ "3980": "nan",
+ "3985": "nan",
+ "3990": "nan",
+ "3995": "nan",
+ "4000": 1.41699,
+ "4005": "nan",
+ "4010": "nan",
+ "4015": "nan",
+ "4020": "nan",
+ "4025": "nan",
+ "4030": "nan",
+ "4035": "nan",
+ "4040": "nan",
+ "4045": "nan",
+ "4050": "nan",
+ "4055": "nan",
+ "4060": "nan",
+ "4065": "nan",
+ "4070": "nan",
+ "4075": "nan",
+ "4080": "nan",
+ "4085": "nan",
+ "4090": "nan",
+ "4095": "nan",
+ "4100": 1.41688,
+ "4105": "nan",
+ "4110": "nan",
+ "4115": "nan",
+ "4120": "nan",
+ "4125": "nan",
+ "4130": "nan",
+ "4135": "nan",
+ "4140": "nan",
+ "4145": "nan",
+ "4150": "nan",
+ "4155": "nan",
+ "4160": "nan",
+ "4165": "nan",
+ "4170": "nan",
+ "4175": "nan",
+ "4180": "nan",
+ "4185": "nan",
+ "4190": "nan",
+ "4195": "nan",
+ "4200": 1.41758,
+ "4205": "nan",
+ "4210": "nan",
+ "4215": "nan",
+ "4220": "nan",
+ "4225": "nan",
+ "4230": "nan",
+ "4235": "nan",
+ "4240": "nan",
+ "4245": "nan",
+ "4250": "nan",
+ "4255": "nan",
+ "4260": "nan",
+ "4265": "nan",
+ "4270": "nan",
+ "4275": "nan",
+ "4280": "nan",
+ "4285": "nan",
+ "4290": "nan",
+ "4295": "nan",
+ "4300": 1.41698,
+ "4305": "nan",
+ "4310": "nan",
+ "4315": "nan",
+ "4320": "nan",
+ "4325": "nan",
+ "4330": "nan",
+ "4335": "nan",
+ "4340": "nan",
+ "4345": "nan",
+ "4350": "nan",
+ "4355": "nan",
+ "4360": "nan",
+ "4365": "nan",
+ "4370": "nan",
+ "4375": "nan",
+ "4380": "nan",
+ "4385": "nan",
+ "4390": "nan",
+ "4395": "nan",
+ "4400": 1.41728,
+ "4405": "nan",
+ "4410": "nan",
+ "4415": "nan",
+ "4420": "nan",
+ "4425": "nan",
+ "4430": "nan",
+ "4435": "nan",
+ "4440": "nan",
+ "4445": "nan",
+ "4450": "nan",
+ "4455": "nan",
+ "4460": "nan",
+ "4465": "nan",
+ "4470": "nan",
+ "4475": "nan",
+ "4480": "nan",
+ "4485": "nan",
+ "4490": "nan",
+ "4495": "nan",
+ "4500": 1.41647,
+ "4505": "nan",
+ "4510": "nan",
+ "4515": "nan",
+ "4520": "nan",
+ "4525": "nan",
+ "4530": "nan",
+ "4535": "nan",
+ "4540": "nan",
+ "4545": "nan",
+ "4550": "nan",
+ "4555": "nan",
+ "4560": "nan",
+ "4565": "nan",
+ "4570": "nan",
+ "4575": "nan",
+ "4580": "nan",
+ "4585": "nan",
+ "4590": "nan",
+ "4595": "nan",
+ "4600": 1.42068,
+ "4605": "nan",
+ "4610": "nan",
+ "4615": "nan",
+ "4620": "nan",
+ "4625": "nan",
+ "4630": "nan",
+ "4635": "nan",
+ "4640": "nan",
+ "4645": "nan",
+ "4650": "nan",
+ "4655": "nan",
+ "4660": "nan",
+ "4665": "nan",
+ "4670": "nan",
+ "4675": "nan",
+ "4680": "nan",
+ "4685": "nan",
+ "4690": "nan",
+ "4695": "nan",
+ "4700": 1.41828,
+ "4705": "nan",
+ "4710": "nan",
+ "4715": "nan",
+ "4720": "nan",
+ "4725": "nan",
+ "4730": "nan",
+ "4735": "nan",
+ "4740": "nan",
+ "4745": "nan",
+ "4750": "nan",
+ "4755": "nan",
+ "4760": "nan",
+ "4765": "nan",
+ "4770": "nan",
+ "4775": "nan",
+ "4780": "nan",
+ "4785": "nan",
+ "4790": "nan",
+ "4795": "nan",
+ "4800": 1.42135,
+ "4805": "nan",
+ "4810": "nan",
+ "4815": "nan",
+ "4820": "nan",
+ "4825": "nan",
+ "4830": "nan",
+ "4835": "nan",
+ "4840": "nan",
+ "4845": "nan",
+ "4850": "nan",
+ "4855": "nan",
+ "4860": "nan",
+ "4865": "nan",
+ "4870": "nan",
+ "4875": "nan",
+ "4880": "nan",
+ "4885": "nan",
+ "4890": "nan",
+ "4895": "nan",
+ "4900": 1.41613,
+ "4905": "nan",
+ "4910": "nan",
+ "4915": "nan",
+ "4920": "nan",
+ "4925": "nan",
+ "4930": "nan",
+ "4935": "nan",
+ "4940": "nan",
+ "4945": "nan",
+ "4950": "nan",
+ "4955": "nan",
+ "4960": "nan",
+ "4965": "nan",
+ "4970": "nan",
+ "4975": "nan",
+ "4980": "nan",
+ "4985": "nan",
+ "4990": "nan",
+ "4995": "nan",
+ "5000": 1.41976,
+ "5005": "nan",
+ "5010": "nan",
+ "5015": "nan",
+ "5020": "nan",
+ "5025": "nan",
+ "5030": "nan",
+ "5035": "nan",
+ "5040": "nan",
+ "5045": "nan",
+ "5050": "nan",
+ "5055": "nan",
+ "5060": "nan",
+ "5065": "nan",
+ "5070": "nan",
+ "5075": "nan",
+ "5080": "nan",
+ "5085": "nan",
+ "5090": "nan",
+ "5095": "nan",
+ "5100": 1.42195,
+ "5105": "nan",
+ "5110": "nan",
+ "5115": "nan",
+ "5120": "nan",
+ "5125": "nan",
+ "5130": "nan",
+ "5135": "nan",
+ "5140": "nan",
+ "5145": "nan",
+ "5150": "nan",
+ "5155": "nan",
+ "5160": "nan",
+ "5165": "nan",
+ "5170": "nan",
+ "5175": "nan",
+ "5180": "nan",
+ "5185": "nan",
+ "5190": "nan",
+ "5195": "nan",
+ "5200": 1.41591,
+ "5205": "nan",
+ "5210": "nan",
+ "5215": "nan",
+ "5220": "nan",
+ "5225": "nan",
+ "5230": "nan",
+ "5235": "nan",
+ "5240": "nan",
+ "5245": "nan",
+ "5250": "nan",
+ "5255": "nan",
+ "5260": "nan",
+ "5265": "nan",
+ "5270": "nan",
+ "5275": "nan",
+ "5280": "nan",
+ "5285": "nan",
+ "5290": "nan",
+ "5295": "nan",
+ "5300": 1.41729,
+ "5305": "nan",
+ "5310": "nan",
+ "5315": "nan",
+ "5320": "nan",
+ "5325": "nan",
+ "5330": "nan",
+ "5335": "nan",
+ "5340": "nan",
+ "5345": "nan",
+ "5350": "nan",
+ "5355": "nan",
+ "5360": "nan",
+ "5365": "nan",
+ "5370": "nan",
+ "5375": "nan",
+ "5380": "nan",
+ "5385": "nan",
+ "5390": "nan",
+ "5395": "nan",
+ "5400": 1.42072,
+ "5405": "nan",
+ "5410": "nan",
+ "5415": "nan",
+ "5420": "nan",
+ "5425": "nan",
+ "5430": "nan",
+ "5435": "nan",
+ "5440": "nan",
+ "5445": "nan",
+ "5450": "nan",
+ "5455": "nan",
+ "5460": "nan",
+ "5465": "nan",
+ "5470": "nan",
+ "5475": "nan",
+ "5480": "nan",
+ "5485": "nan",
+ "5490": "nan",
+ "5495": "nan",
+ "5500": 1.42092,
+ "5505": "nan",
+ "5510": "nan",
+ "5515": "nan",
+ "5520": "nan",
+ "5525": "nan",
+ "5530": "nan",
+ "5535": "nan",
+ "5540": "nan",
+ "5545": "nan",
+ "5550": "nan",
+ "5555": "nan",
+ "5560": "nan",
+ "5565": "nan",
+ "5570": "nan",
+ "5575": "nan",
+ "5580": "nan",
+ "5585": "nan",
+ "5590": "nan",
+ "5595": "nan",
+ "5600": 1.41992,
+ "5605": "nan",
+ "5610": "nan",
+ "5615": "nan",
+ "5620": "nan",
+ "5625": "nan",
+ "5630": "nan",
+ "5635": "nan",
+ "5640": "nan",
+ "5645": "nan",
+ "5650": "nan",
+ "5655": "nan",
+ "5660": "nan",
+ "5665": "nan",
+ "5670": "nan",
+ "5675": "nan",
+ "5680": "nan",
+ "5685": "nan",
+ "5690": "nan",
+ "5695": "nan",
+ "5700": 1.41833,
+ "5705": "nan",
+ "5710": "nan",
+ "5715": "nan",
+ "5720": "nan",
+ "5725": "nan",
+ "5730": "nan",
+ "5735": "nan",
+ "5740": "nan",
+ "5745": "nan",
+ "5750": "nan",
+ "5755": "nan",
+ "5760": "nan",
+ "5765": "nan",
+ "5770": "nan",
+ "5775": "nan",
+ "5780": "nan",
+ "5785": "nan",
+ "5790": "nan",
+ "5795": "nan",
+ "5800": 1.41738,
+ "5805": "nan",
+ "5810": "nan",
+ "5815": "nan",
+ "5820": "nan",
+ "5825": "nan",
+ "5830": "nan",
+ "5835": "nan",
+ "5840": "nan",
+ "5845": "nan",
+ "5850": "nan",
+ "5855": "nan",
+ "5860": "nan",
+ "5865": "nan",
+ "5870": "nan",
+ "5875": "nan",
+ "5880": "nan",
+ "5885": "nan",
+ "5890": "nan",
+ "5895": "nan",
+ "5900": 1.42112,
+ "5905": "nan",
+ "5910": "nan",
+ "5915": "nan",
+ "5920": "nan",
+ "5925": "nan",
+ "5930": "nan",
+ "5935": "nan",
+ "5940": "nan",
+ "5945": "nan",
+ "5950": "nan",
+ "5955": "nan",
+ "5960": "nan",
+ "5965": "nan",
+ "5970": "nan",
+ "5975": "nan",
+ "5980": "nan",
+ "5985": "nan",
+ "5990": "nan",
+ "5995": "nan",
+ "6000": 1.42017,
+ "6005": "nan",
+ "6010": "nan",
+ "6015": "nan",
+ "6020": "nan",
+ "6025": "nan",
+ "6030": "nan",
+ "6035": "nan",
+ "6040": "nan",
+ "6045": "nan",
+ "6050": "nan",
+ "6055": "nan",
+ "6060": "nan",
+ "6065": "nan",
+ "6070": "nan",
+ "6075": "nan",
+ "6080": "nan",
+ "6085": "nan",
+ "6090": "nan",
+ "6095": "nan",
+ "6100": 1.41819,
+ "6105": "nan",
+ "6110": "nan",
+ "6115": "nan",
+ "6120": "nan",
+ "6125": "nan",
+ "6130": "nan",
+ "6135": "nan",
+ "6140": "nan",
+ "6145": "nan",
+ "6150": "nan",
+ "6155": "nan",
+ "6160": "nan",
+ "6165": "nan",
+ "6170": "nan",
+ "6175": "nan",
+ "6180": "nan",
+ "6185": "nan",
+ "6190": "nan",
+ "6195": "nan",
+ "6200": 1.41783,
+ "6205": "nan",
+ "6210": "nan",
+ "6215": "nan",
+ "6220": "nan",
+ "6225": "nan",
+ "6230": "nan",
+ "6235": "nan",
+ "6240": "nan",
+ "6245": "nan",
+ "6250": "nan",
+ "6255": "nan",
+ "6260": "nan",
+ "6265": "nan",
+ "6270": "nan",
+ "6275": "nan",
+ "6280": "nan",
+ "6285": "nan",
+ "6290": "nan",
+ "6295": "nan",
+ "6300": 1.41997,
+ "6305": "nan",
+ "6310": "nan",
+ "6315": "nan",
+ "6320": "nan",
+ "6325": "nan",
+ "6330": "nan",
+ "6335": "nan",
+ "6340": "nan",
+ "6345": "nan",
+ "6350": "nan",
+ "6355": "nan",
+ "6360": "nan",
+ "6365": "nan",
+ "6370": "nan",
+ "6375": "nan",
+ "6380": "nan",
+ "6385": "nan",
+ "6390": "nan",
+ "6395": "nan",
+ "6400": 1.41757,
+ "6405": "nan",
+ "6410": "nan",
+ "6415": "nan",
+ "6420": "nan",
+ "6425": "nan",
+ "6430": "nan",
+ "6435": "nan",
+ "6440": "nan",
+ "6445": "nan",
+ "6450": "nan",
+ "6455": "nan",
+ "6460": "nan",
+ "6465": "nan",
+ "6470": "nan",
+ "6475": "nan",
+ "6480": "nan",
+ "6485": "nan",
+ "6490": "nan",
+ "6495": "nan",
+ "6500": 1.42002,
+ "6505": "nan",
+ "6510": "nan",
+ "6515": "nan",
+ "6520": "nan",
+ "6525": "nan",
+ "6530": "nan",
+ "6535": "nan",
+ "6540": "nan",
+ "6545": "nan",
+ "6550": "nan",
+ "6555": "nan",
+ "6560": "nan",
+ "6565": "nan",
+ "6570": "nan",
+ "6575": "nan",
+ "6580": "nan",
+ "6585": "nan",
+ "6590": "nan",
+ "6595": "nan",
+ "6600": 1.41769,
+ "6605": "nan",
+ "6610": "nan",
+ "6615": "nan",
+ "6620": "nan",
+ "6625": "nan",
+ "6630": "nan",
+ "6635": "nan",
+ "6640": "nan",
+ "6645": "nan",
+ "6650": "nan",
+ "6655": "nan",
+ "6660": "nan",
+ "6665": "nan",
+ "6670": "nan",
+ "6675": "nan",
+ "6680": "nan",
+ "6685": "nan",
+ "6690": "nan",
+ "6695": "nan",
+ "6700": 1.50538,
+ "6705": "nan",
+ "6710": "nan",
+ "6715": "nan",
+ "6720": "nan",
+ "6725": "nan",
+ "6730": "nan",
+ "6735": "nan",
+ "6740": "nan",
+ "6745": "nan",
+ "6750": "nan",
+ "6755": "nan",
+ "6760": "nan",
+ "6765": "nan",
+ "6770": "nan",
+ "6775": "nan",
+ "6780": "nan",
+ "6785": "nan",
+ "6790": "nan",
+ "6795": "nan",
+ "6800": 1.41809,
+ "6805": "nan",
+ "6810": "nan",
+ "6815": "nan",
+ "6820": "nan",
+ "6825": "nan",
+ "6830": "nan",
+ "6835": "nan",
+ "6840": "nan",
+ "6845": "nan",
+ "6850": "nan",
+ "6855": "nan",
+ "6860": "nan",
+ "6865": "nan",
+ "6870": "nan",
+ "6875": "nan",
+ "6880": "nan",
+ "6885": "nan",
+ "6890": "nan",
+ "6895": "nan",
+ "6900": 1.41692,
+ "6905": "nan",
+ "6910": "nan",
+ "6915": "nan",
+ "6920": "nan",
+ "6925": "nan",
+ "6930": "nan",
+ "6935": "nan",
+ "6940": "nan",
+ "6945": "nan",
+ "6950": "nan",
+ "6955": "nan",
+ "6960": "nan",
+ "6965": "nan",
+ "6970": "nan",
+ "6975": "nan",
+ "6980": "nan",
+ "6985": "nan",
+ "6990": "nan",
+ "6995": "nan",
+ "7000": 1.41814,
+ "7005": "nan",
+ "7010": "nan",
+ "7015": "nan",
+ "7020": "nan",
+ "7025": "nan",
+ "7030": "nan",
+ "7035": "nan",
+ "7040": "nan",
+ "7045": "nan",
+ "7050": "nan",
+ "7055": "nan",
+ "7060": "nan",
+ "7065": "nan",
+ "7070": "nan",
+ "7075": "nan",
+ "7080": "nan",
+ "7085": "nan",
+ "7090": "nan",
+ "7095": "nan",
+ "7100": 1.41682,
+ "7105": "nan",
+ "7110": "nan",
+ "7115": "nan",
+ "7120": "nan",
+ "7125": "nan",
+ "7130": "nan",
+ "7135": "nan",
+ "7140": "nan",
+ "7145": "nan",
+ "7150": "nan",
+ "7155": "nan",
+ "7160": "nan",
+ "7165": "nan",
+ "7170": "nan",
+ "7175": "nan",
+ "7180": "nan",
+ "7185": "nan",
+ "7190": "nan",
+ "7195": "nan",
+ "7200": 1.41907,
+ "7205": "nan",
+ "7210": "nan",
+ "7215": "nan",
+ "7220": "nan",
+ "7225": "nan",
+ "7230": "nan",
+ "7235": "nan",
+ "7240": "nan",
+ "7245": "nan",
+ "7250": "nan",
+ "7255": "nan",
+ "7260": "nan",
+ "7265": "nan",
+ "7270": "nan",
+ "7275": "nan",
+ "7280": "nan",
+ "7285": "nan",
+ "7290": "nan",
+ "7295": "nan",
+ "7300": 1.44244,
+ "7305": "nan",
+ "7310": "nan",
+ "7315": "nan",
+ "7320": "nan",
+ "7325": "nan",
+ "7330": "nan",
+ "7335": "nan",
+ "7340": "nan",
+ "7345": "nan",
+ "7350": "nan",
+ "7355": "nan",
+ "7360": "nan",
+ "7365": "nan",
+ "7370": "nan",
+ "7375": "nan",
+ "7380": "nan",
+ "7385": "nan",
+ "7390": "nan",
+ "7395": "nan",
+ "7400": 1.41831,
+ "7405": "nan",
+ "7410": "nan",
+ "7415": "nan",
+ "7420": "nan",
+ "7425": "nan",
+ "7430": "nan",
+ "7435": "nan",
+ "7440": "nan",
+ "7445": "nan",
+ "7450": "nan",
+ "7455": "nan",
+ "7460": "nan",
+ "7465": "nan",
+ "7470": "nan",
+ "7475": "nan",
+ "7480": "nan",
+ "7485": "nan",
+ "7490": "nan",
+ "7495": "nan",
+ "7500": 1.41688,
+ "7505": "nan",
+ "7510": "nan",
+ "7515": "nan",
+ "7520": "nan",
+ "7525": "nan",
+ "7530": "nan",
+ "7535": "nan",
+ "7540": "nan",
+ "7545": "nan",
+ "7550": "nan",
+ "7555": "nan",
+ "7560": "nan",
+ "7565": "nan",
+ "7570": "nan",
+ "7575": "nan",
+ "7580": "nan",
+ "7585": "nan",
+ "7590": "nan",
+ "7595": "nan",
+ "7600": 1.41519,
+ "7605": "nan",
+ "7610": "nan",
+ "7615": "nan",
+ "7620": "nan",
+ "7625": "nan",
+ "7630": "nan",
+ "7635": "nan",
+ "7640": "nan",
+ "7645": "nan",
+ "7650": "nan",
+ "7655": "nan",
+ "7660": "nan",
+ "7665": "nan",
+ "7670": "nan",
+ "7675": "nan",
+ "7680": "nan",
+ "7685": "nan",
+ "7690": "nan",
+ "7695": "nan",
+ "7700": 1.41777,
+ "7705": "nan",
+ "7710": "nan",
+ "7715": "nan",
+ "7720": "nan",
+ "7725": "nan",
+ "7730": "nan",
+ "7735": "nan",
+ "7740": "nan",
+ "7745": "nan",
+ "7750": "nan",
+ "7755": "nan",
+ "7760": "nan",
+ "7765": "nan",
+ "7770": "nan",
+ "7775": "nan",
+ "7780": "nan",
+ "7785": "nan",
+ "7790": "nan",
+ "7795": "nan",
+ "7800": 1.41752,
+ "7805": "nan",
+ "7810": "nan",
+ "7815": "nan",
+ "7820": "nan",
+ "7825": "nan",
+ "7830": "nan",
+ "7835": "nan",
+ "7840": "nan",
+ "7845": "nan",
+ "7850": "nan",
+ "7855": "nan",
+ "7860": "nan",
+ "7865": "nan",
+ "7870": "nan",
+ "7875": "nan",
+ "7880": "nan",
+ "7885": "nan",
+ "7890": "nan",
+ "7895": "nan",
+ "7900": 1.41496,
+ "7905": "nan",
+ "7910": "nan",
+ "7915": "nan",
+ "7920": "nan",
+ "7925": "nan",
+ "7930": "nan",
+ "7935": "nan",
+ "7940": "nan",
+ "7945": "nan",
+ "7950": "nan",
+ "7955": "nan",
+ "7960": "nan",
+ "7965": "nan",
+ "7970": "nan",
+ "7975": "nan",
+ "7980": "nan",
+ "7985": "nan",
+ "7990": "nan",
+ "7995": "nan",
+ "8000": 1.41445,
+ "8005": "nan",
+ "8010": "nan",
+ "8015": "nan",
+ "8020": "nan",
+ "8025": "nan",
+ "8030": "nan",
+ "8035": "nan",
+ "8040": "nan",
+ "8045": "nan",
+ "8050": "nan",
+ "8055": "nan",
+ "8060": "nan",
+ "8065": "nan",
+ "8070": "nan",
+ "8075": "nan",
+ "8080": "nan",
+ "8085": "nan",
+ "8090": "nan",
+ "8095": "nan",
+ "8100": 1.41546,
+ "8105": "nan",
+ "8110": "nan",
+ "8115": "nan",
+ "8120": "nan",
+ "8125": "nan",
+ "8130": "nan",
+ "8135": "nan",
+ "8140": "nan",
+ "8145": "nan",
+ "8150": "nan",
+ "8155": "nan",
+ "8160": "nan",
+ "8165": "nan",
+ "8170": "nan",
+ "8175": "nan",
+ "8180": "nan",
+ "8185": "nan",
+ "8190": "nan",
+ "8195": "nan",
+ "8200": 1.41496,
+ "8205": "nan",
+ "8210": "nan",
+ "8215": "nan",
+ "8220": "nan",
+ "8225": "nan",
+ "8230": "nan",
+ "8235": "nan",
+ "8240": "nan",
+ "8245": "nan",
+ "8250": "nan",
+ "8255": "nan",
+ "8260": "nan",
+ "8265": "nan",
+ "8270": "nan",
+ "8275": "nan",
+ "8280": "nan",
+ "8285": "nan",
+ "8290": "nan",
+ "8295": "nan",
+ "8300": 1.41627,
+ "8305": "nan",
+ "8310": "nan",
+ "8315": "nan",
+ "8320": "nan",
+ "8325": "nan",
+ "8330": "nan",
+ "8335": "nan",
+ "8340": "nan",
+ "8345": "nan",
+ "8350": "nan",
+ "8355": "nan",
+ "8360": "nan",
+ "8365": "nan",
+ "8370": "nan",
+ "8375": "nan",
+ "8380": "nan",
+ "8385": "nan",
+ "8390": "nan",
+ "8395": "nan",
+ "8400": 1.41435,
+ "8405": "nan",
+ "8410": "nan",
+ "8415": "nan",
+ "8420": "nan",
+ "8425": "nan",
+ "8430": "nan",
+ "8435": "nan",
+ "8440": "nan",
+ "8445": "nan",
+ "8450": "nan",
+ "8455": "nan",
+ "8460": "nan",
+ "8465": "nan",
+ "8470": "nan",
+ "8475": "nan",
+ "8480": "nan",
+ "8485": "nan",
+ "8490": "nan",
+ "8495": "nan",
+ "8500": 1.41723,
+ "8505": "nan",
+ "8510": "nan",
+ "8515": "nan",
+ "8520": "nan",
+ "8525": "nan",
+ "8530": "nan",
+ "8535": "nan",
+ "8540": "nan",
+ "8545": "nan",
+ "8550": "nan",
+ "8555": "nan",
+ "8560": "nan",
+ "8565": "nan",
+ "8570": "nan",
+ "8575": "nan",
+ "8580": "nan",
+ "8585": "nan",
+ "8590": "nan",
+ "8595": "nan",
+ "8600": 1.41437,
+ "8605": "nan",
+ "8610": "nan",
+ "8615": "nan",
+ "8620": "nan",
+ "8625": "nan",
+ "8630": "nan",
+ "8635": "nan",
+ "8640": "nan",
+ "8645": "nan",
+ "8650": "nan",
+ "8655": "nan",
+ "8660": "nan",
+ "8665": "nan",
+ "8670": "nan",
+ "8675": "nan",
+ "8680": "nan",
+ "8685": "nan",
+ "8690": "nan",
+ "8695": "nan",
+ "8700": 1.41717,
+ "8705": "nan",
+ "8710": "nan",
+ "8715": "nan",
+ "8720": "nan",
+ "8725": "nan",
+ "8730": "nan",
+ "8735": "nan",
+ "8740": "nan",
+ "8745": "nan",
+ "8750": "nan",
+ "8755": "nan",
+ "8760": "nan",
+ "8765": "nan",
+ "8770": "nan",
+ "8775": "nan",
+ "8780": "nan",
+ "8785": "nan",
+ "8790": "nan",
+ "8795": "nan",
+ "8800": 1.41626,
+ "8805": "nan",
+ "8810": "nan",
+ "8815": "nan",
+ "8820": "nan",
+ "8825": "nan",
+ "8830": "nan",
+ "8835": "nan",
+ "8840": "nan",
+ "8845": "nan",
+ "8850": "nan",
+ "8855": "nan",
+ "8860": "nan",
+ "8865": "nan",
+ "8870": "nan",
+ "8875": "nan",
+ "8880": "nan",
+ "8885": "nan",
+ "8890": "nan",
+ "8895": "nan",
+ "8900": 1.72345,
+ "8905": "nan",
+ "8910": "nan",
+ "8915": "nan",
+ "8920": "nan",
+ "8925": "nan",
+ "8930": "nan",
+ "8935": "nan",
+ "8940": "nan",
+ "8945": "nan",
+ "8950": "nan",
+ "8955": "nan",
+ "8960": "nan",
+ "8965": "nan",
+ "8970": "nan",
+ "8975": "nan",
+ "8980": "nan",
+ "8985": "nan",
+ "8990": "nan",
+ "8995": "nan",
+ "9000": 1.42136,
+ "9005": "nan",
+ "9010": "nan",
+ "9015": "nan",
+ "9020": "nan",
+ "9025": "nan",
+ "9030": "nan",
+ "9035": "nan",
+ "9040": "nan",
+ "9045": "nan",
+ "9050": "nan",
+ "9055": "nan",
+ "9060": "nan",
+ "9065": "nan",
+ "9070": "nan",
+ "9075": "nan",
+ "9080": "nan",
+ "9085": "nan",
+ "9090": "nan",
+ "9095": "nan",
+ "9100": 1.41472,
+ "9105": "nan",
+ "9110": "nan",
+ "9115": "nan",
+ "9120": "nan",
+ "9125": "nan",
+ "9130": "nan",
+ "9135": "nan",
+ "9140": "nan",
+ "9145": "nan",
+ "9150": "nan",
+ "9155": "nan",
+ "9160": "nan",
+ "9165": "nan",
+ "9170": "nan",
+ "9175": "nan",
+ "9180": "nan",
+ "9185": "nan",
+ "9190": "nan",
+ "9195": "nan",
+ "9200": 1.41768,
+ "9205": "nan",
+ "9210": "nan",
+ "9215": "nan",
+ "9220": "nan",
+ "9225": "nan",
+ "9230": "nan",
+ "9235": "nan",
+ "9240": "nan",
+ "9245": "nan",
+ "9250": "nan",
+ "9255": "nan",
+ "9260": "nan",
+ "9265": "nan",
+ "9270": "nan",
+ "9275": "nan",
+ "9280": "nan",
+ "9285": "nan",
+ "9290": "nan",
+ "9295": "nan",
+ "9300": 1.42052,
+ "9305": "nan",
+ "9310": "nan",
+ "9315": "nan",
+ "9320": "nan",
+ "9325": "nan",
+ "9330": "nan",
+ "9335": "nan",
+ "9340": "nan",
+ "9345": "nan",
+ "9350": "nan",
+ "9355": "nan",
+ "9360": "nan",
+ "9365": "nan",
+ "9370": "nan",
+ "9375": "nan",
+ "9380": "nan",
+ "9385": "nan",
+ "9390": "nan",
+ "9395": "nan",
+ "9400": 1.41618,
+ "9405": "nan",
+ "9410": "nan",
+ "9415": "nan",
+ "9420": "nan",
+ "9425": "nan",
+ "9430": "nan",
+ "9435": "nan",
+ "9440": "nan",
+ "9445": "nan",
+ "9450": "nan",
+ "9455": "nan",
+ "9460": "nan",
+ "9465": "nan",
+ "9470": "nan",
+ "9475": "nan",
+ "9480": "nan",
+ "9485": "nan",
+ "9490": "nan",
+ "9495": "nan",
+ "9500": 1.41637,
+ "9505": "nan",
+ "9510": "nan",
+ "9515": "nan",
+ "9520": "nan",
+ "9525": "nan",
+ "9530": "nan",
+ "9535": "nan",
+ "9540": "nan",
+ "9545": "nan",
+ "9550": "nan",
+ "9555": "nan",
+ "9560": "nan",
+ "9565": "nan",
+ "9570": "nan",
+ "9575": "nan",
+ "9580": "nan",
+ "9585": "nan",
+ "9590": "nan",
+ "9595": "nan",
+ "9600": 1.41511,
+ "9605": "nan",
+ "9610": "nan",
+ "9615": "nan",
+ "9620": "nan",
+ "9625": "nan",
+ "9630": "nan",
+ "9635": "nan",
+ "9640": "nan",
+ "9645": "nan",
+ "9650": "nan",
+ "9655": "nan",
+ "9660": "nan",
+ "9665": "nan",
+ "9670": "nan",
+ "9675": "nan",
+ "9680": "nan",
+ "9685": "nan",
+ "9690": "nan",
+ "9695": "nan",
+ "9700": 1.41737,
+ "9705": "nan",
+ "9710": "nan",
+ "9715": "nan",
+ "9720": "nan",
+ "9725": "nan",
+ "9730": "nan",
+ "9735": "nan",
+ "9740": "nan",
+ "9745": "nan",
+ "9750": "nan",
+ "9755": "nan",
+ "9760": "nan",
+ "9765": "nan",
+ "9770": "nan",
+ "9775": "nan",
+ "9780": "nan",
+ "9785": "nan",
+ "9790": "nan",
+ "9795": "nan",
+ "9800": 1.41703,
+ "9805": "nan",
+ "9810": "nan",
+ "9815": "nan",
+ "9820": "nan",
+ "9825": "nan",
+ "9830": "nan",
+ "9835": "nan",
+ "9840": "nan",
+ "9845": "nan",
+ "9850": "nan",
+ "9855": "nan",
+ "9860": "nan",
+ "9865": "nan",
+ "9870": "nan",
+ "9875": "nan",
+ "9880": "nan",
+ "9885": "nan",
+ "9890": "nan",
+ "9895": "nan",
+ "9900": 1.4201,
+ "9905": "nan",
+ "9910": "nan",
+ "9915": "nan",
+ "9920": "nan",
+ "9925": "nan",
+ "9930": "nan",
+ "9935": "nan",
+ "9940": "nan",
+ "9945": "nan",
+ "9950": "nan",
+ "9955": "nan",
+ "9960": "nan",
+ "9965": "nan",
+ "9970": "nan",
+ "9975": "nan",
+ "9980": "nan",
+ "9985": "nan",
+ "9990": "nan",
+ "9995": "nan",
+ "10000": 1.42216,
+ "10005": "nan",
+ "10010": "nan",
+ "10015": "nan",
+ "10020": "nan",
+ "10025": "nan",
+ "10030": "nan",
+ "10035": "nan",
+ "10040": "nan",
+ "10045": "nan",
+ "10050": "nan",
+ "10055": "nan",
+ "10060": "nan",
+ "10065": "nan",
+ "10070": "nan",
+ "10075": "nan",
+ "10080": "nan",
+ "10085": "nan",
+ "10090": "nan",
+ "10095": "nan",
+ "10100": 1.43065,
+ "10105": "nan",
+ "10110": "nan",
+ "10115": "nan",
+ "10120": "nan",
+ "10125": "nan",
+ "10130": "nan",
+ "10135": "nan",
+ "10140": "nan",
+ "10145": "nan",
+ "10150": "nan",
+ "10155": "nan",
+ "10160": "nan",
+ "10165": "nan",
+ "10170": "nan",
+ "10175": "nan",
+ "10180": "nan",
+ "10185": "nan",
+ "10190": "nan",
+ "10195": "nan",
+ "10200": 1.41657
+ }
+ }
+}
\ No newline at end of file
diff --git a/tests/functional_tests/test_cases/gpt/gpt3_15b_8t_release_sm/model_config.yaml b/tests/functional_tests/test_cases/gpt/gpt3_15b_8t_release_sm/model_config.yaml
index 8b437ba75e7..32386558710 100644
--- a/tests/functional_tests/test_cases/gpt/gpt3_15b_8t_release_sm/model_config.yaml
+++ b/tests/functional_tests/test_cases/gpt/gpt3_15b_8t_release_sm/model_config.yaml
@@ -23,7 +23,7 @@ MODEL_ARGS:
--micro-batch-size: 4
--rampup-batch-size: "[384 384 97656250]"
--global-batch-size: 1152
- --train-samples: 4882812
+ --train-samples: 19531250
--manual-gc: true
# Transformer Engine args
--transformer-impl: transformer_engine
@@ -68,9 +68,10 @@ MODEL_ARGS:
--eval-iters: 32
--eval-interval: 2000
# Add checkpointing args
- --load: ${CHECKPOINT_LOAD_PATH}
--save: ${CHECKPOINT_SAVE_PATH}
- --save-interval: 1000
+ --load: ${CHECKPOINT_LOAD_PATH}
+ --save-interval: 5000
+ --save-retain-interval: 10000
# Add initialization args
--init-method-std: 0.0134
# Add logging args
@@ -86,7 +87,7 @@ MODEL_ARGS:
--wandb-exp-name: ${WANDB_EXPERIMENT}
# Add mixed precision args
--bf16: true
- --exit-interval: 13000
+ --exit-interval: 10200
--wandb-save-dir: ${WANDB_SAVE_PATH}
METRICS:
- "iteration-time"
diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_dist_optimizer_overlap_grad_reduce_param_gather_overlap_optimizer/golden_values_dev_dgx_gb200.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_dist_optimizer_overlap_grad_reduce_param_gather_overlap_optimizer/golden_values_dev_dgx_gb200.json
new file mode 100644
index 00000000000..f023ed07c99
--- /dev/null
+++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_dist_optimizer_overlap_grad_reduce_param_gather_overlap_optimizer/golden_values_dev_dgx_gb200.json
@@ -0,0 +1,287 @@
+{
+ "lm loss": {
+ "start_step": 1,
+ "end_step": 50,
+ "step_interval": 1,
+ "values": {
+ "1": 10.82558,
+ "2": 10.83322,
+ "3": 10.82737,
+ "4": 10.79588,
+ "5": 10.85708,
+ "6": 10.86392,
+ "7": 10.8269,
+ "8": 10.82588,
+ "9": 10.83699,
+ "10": 10.79719,
+ "11": 10.87851,
+ "12": 10.85797,
+ "13": 10.85368,
+ "14": 10.87548,
+ "15": 10.79177,
+ "16": 10.80301,
+ "17": 10.7745,
+ "18": 10.80399,
+ "19": 10.79365,
+ "20": 10.69588,
+ "21": 10.6855,
+ "22": 10.53152,
+ "23": 10.70658,
+ "24": 10.57319,
+ "25": 10.51545,
+ "26": 10.59076,
+ "27": 10.60738,
+ "28": 10.57025,
+ "29": 10.58904,
+ "30": 10.34674,
+ "31": 10.07736,
+ "32": 10.46317,
+ "33": 10.45705,
+ "34": 10.19923,
+ "35": 10.25593,
+ "36": 10.21246,
+ "37": 10.34689,
+ "38": 10.18008,
+ "39": 10.40796,
+ "40": 10.07602,
+ "41": 10.12935,
+ "42": 10.21132,
+ "43": 9.81692,
+ "44": 9.94027,
+ "45": 9.817,
+ "46": 9.80608,
+ "47": 10.12473,
+ "48": 9.84047,
+ "49": 9.50975,
+ "50": 9.88932
+ }
+ },
+ "num-zeros": {
+ "start_step": 1,
+ "end_step": 50,
+ "step_interval": 1,
+ "values": {
+ "1": 1691.0,
+ "2": 1553.0,
+ "3": 1673.0,
+ "4": 1760.0,
+ "5": 1852.0,
+ "6": 1861.0,
+ "7": 1852.0,
+ "8": 1755.0,
+ "9": 1952.0,
+ "10": 1427.0,
+ "11": 1857.0,
+ "12": 1820.0,
+ "13": 1948.0,
+ "14": 1828.0,
+ "15": 1913.0,
+ "16": 1881.0,
+ "17": 1770.0,
+ "18": 1683.0,
+ "19": 1784.0,
+ "20": 1714.0,
+ "21": 1969.0,
+ "22": 1701.0,
+ "23": 1972.0,
+ "24": 1545.0,
+ "25": 1537.0,
+ "26": 1650.0,
+ "27": 1770.0,
+ "28": 1889.0,
+ "29": 1946.0,
+ "30": 2031.0,
+ "31": 1511.0,
+ "32": 1848.0,
+ "33": 2009.0,
+ "34": 1749.0,
+ "35": 1978.0,
+ "36": 1926.0,
+ "37": 2358.0,
+ "38": 2036.0,
+ "39": 2202.0,
+ "40": 2015.0,
+ "41": 2184.0,
+ "42": 2304.0,
+ "43": 2079.0,
+ "44": 2042.0,
+ "45": 2082.0,
+ "46": 2206.0,
+ "47": 2417.0,
+ "48": 2284.0,
+ "49": 2231.0,
+ "50": 2430.0
+ }
+ },
+ "mem-allocated-bytes": {
+ "start_step": 1,
+ "end_step": 50,
+ "step_interval": 1,
+ "values": {
+ "1": 552193536.0,
+ "2": 552193536.0,
+ "3": 552193536.0,
+ "4": 553242112.0,
+ "5": 552193536.0,
+ "6": 553242112.0,
+ "7": 553242112.0,
+ "8": 552193536.0,
+ "9": 552193536.0,
+ "10": 552193536.0,
+ "11": 553242112.0,
+ "12": 552193536.0,
+ "13": 552193536.0,
+ "14": 552193536.0,
+ "15": 552193536.0,
+ "16": 553242112.0,
+ "17": 553242112.0,
+ "18": 552193536.0,
+ "19": 553242112.0,
+ "20": 552193536.0,
+ "21": 552193536.0,
+ "22": 552193536.0,
+ "23": 552193536.0,
+ "24": 552193536.0,
+ "25": 552193536.0,
+ "26": 552193536.0,
+ "27": 552193536.0,
+ "28": 552193536.0,
+ "29": 552193536.0,
+ "30": 552193536.0,
+ "31": 552193536.0,
+ "32": 552193536.0,
+ "33": 552193536.0,
+ "34": 552193536.0,
+ "35": 552193536.0,
+ "36": 552193536.0,
+ "37": 552193536.0,
+ "38": 552193536.0,
+ "39": 552193536.0,
+ "40": 552193536.0,
+ "41": 552193536.0,
+ "42": 552193536.0,
+ "43": 552193536.0,
+ "44": 552193536.0,
+ "45": 553242112.0,
+ "46": 552193536.0,
+ "47": 552193536.0,
+ "48": 552193536.0,
+ "49": 552193536.0,
+ "50": 552193536.0
+ }
+ },
+ "mem-max-allocated-bytes": {
+ "start_step": 1,
+ "end_step": 50,
+ "step_interval": 1,
+ "values": {
+ "1": 3798208000.0,
+ "2": 3942086144.0,
+ "3": 3942086144.0,
+ "4": 3942086144.0,
+ "5": 3942086144.0,
+ "6": 3942086144.0,
+ "7": 3942086144.0,
+ "8": 3942086144.0,
+ "9": 3942086144.0,
+ "10": 3942086144.0,
+ "11": 3942086144.0,
+ "12": 3942086144.0,
+ "13": 3942086144.0,
+ "14": 3942086144.0,
+ "15": 3942086144.0,
+ "16": 3942086144.0,
+ "17": 3942086144.0,
+ "18": 3942086144.0,
+ "19": 3942086144.0,
+ "20": 3942086144.0,
+ "21": 3942086144.0,
+ "22": 3942086144.0,
+ "23": 3942086144.0,
+ "24": 3942086144.0,
+ "25": 3942086144.0,
+ "26": 3942086144.0,
+ "27": 3942086144.0,
+ "28": 3942086144.0,
+ "29": 3942086144.0,
+ "30": 3942086144.0,
+ "31": 3942086144.0,
+ "32": 3942086144.0,
+ "33": 3942086144.0,
+ "34": 3942086144.0,
+ "35": 3942086144.0,
+ "36": 3942086144.0,
+ "37": 3942086144.0,
+ "38": 3942086144.0,
+ "39": 3942086144.0,
+ "40": 3942086144.0,
+ "41": 3942086144.0,
+ "42": 3942086144.0,
+ "43": 3942086144.0,
+ "44": 3942086144.0,
+ "45": 3942086144.0,
+ "46": 3942086144.0,
+ "47": 3942086144.0,
+ "48": 3942086144.0,
+ "49": 3942086144.0,
+ "50": 3942086144.0
+ }
+ },
+ "iteration-time": {
+ "start_step": 1,
+ "end_step": 50,
+ "step_interval": 1,
+ "values": {
+ "1": 11.06303,
+ "2": 0.15398,
+ "3": 0.27325,
+ "4": 0.13945,
+ "5": 0.25021,
+ "6": 0.16329,
+ "7": 0.27717,
+ "8": 0.18718,
+ "9": 0.12007,
+ "10": 0.21402,
+ "11": 0.2385,
+ "12": 0.61603,
+ "13": 0.24413,
+ "14": 0.18837,
+ "15": 0.14999,
+ "16": 0.12555,
+ "17": 0.24832,
+ "18": 0.1361,
+ "19": 0.13136,
+ "20": 0.27497,
+ "21": 0.22444,
+ "22": 0.11923,
+ "23": 0.11996,
+ "24": 0.25718,
+ "25": 0.20275,
+ "26": 0.35028,
+ "27": 0.11968,
+ "28": 0.23901,
+ "29": 0.12079,
+ "30": 0.12184,
+ "31": 0.21733,
+ "32": 0.28054,
+ "33": 0.11829,
+ "34": 0.17717,
+ "35": 0.1215,
+ "36": 0.27112,
+ "37": 0.22357,
+ "38": 0.12158,
+ "39": 0.12105,
+ "40": 0.12099,
+ "41": 0.21658,
+ "42": 0.22641,
+ "43": 0.12146,
+ "44": 0.1201,
+ "45": 0.253,
+ "46": 0.12142,
+ "47": 0.23268,
+ "48": 0.13569,
+ "49": 0.1302,
+ "50": 0.24153
+ }
+ }
+}
\ No newline at end of file
diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_resume_torch_dist_tunable_overlap/golden_values_lts_dgxa100_dracooci.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_resume_torch_dist_tunable_overlap/golden_values_lts_dgxa100_dracooci.json
index 1ba701443ce..7fa302274bf 100644
--- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_resume_torch_dist_tunable_overlap/golden_values_lts_dgxa100_dracooci.json
+++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_resume_torch_dist_tunable_overlap/golden_values_lts_dgxa100_dracooci.json
@@ -534,4 +534,4 @@
"100": 0.16898
}
}
-}
\ No newline at end of file
+}
diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_tunable_overlap/golden_values_lts_dgxa100_dracooci.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_tunable_overlap/golden_values_lts_dgxa100_dracooci.json
index d8ec5426bd1..363e94d8f52 100644
--- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_tunable_overlap/golden_values_lts_dgxa100_dracooci.json
+++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_tunable_overlap/golden_values_lts_dgxa100_dracooci.json
@@ -284,4 +284,4 @@
"50": 0.16165
}
}
-}
\ No newline at end of file
+}
diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp1_gdn/model_config.yaml b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp1_gdn/model_config.yaml
new file mode 100644
index 00000000000..37933a0e0a7
--- /dev/null
+++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp1_gdn/model_config.yaml
@@ -0,0 +1,82 @@
+ENV_VARS:
+ CUDA_DEVICE_MAX_CONNECTIONS: 1
+ NVTE_ALLOW_NONDETERMINISTIC_ALGO: 0
+ NCCL_ALGO: Ring
+ CUBLAS_WORKSPACE_CONFIG: :4096:8
+ ENABLE_LIGHTWEIGHT_MODE: true
+MODEL_ARGS:
+ # Add network size args
+ --untie-embeddings-and-output-weights: true
+ --num-layers: 6
+ --hidden-size: 512
+ --num-attention-heads: 8
+ --group-query-attention: true
+ --num-query-groups: 2
+ --swiglu: true
+ --position-embedding-type: rope
+ --rotary-percent: 0.5
+ --no-rope-fusion: true #TODO: We can remove this once upgrading to the DEV container
+ --apply-layernorm-1p: true
+ --attention-output-gate: true
+ --apply-wd-to-qk-layernorm: true
+ --experimental-attention-variant: gated_delta_net
+ --linear-attention-freq: 3
+ --linear-conv-kernel-dim: 4
+ --linear-key-head-dim: 64
+ --linear-value-head-dim: 64
+ --linear-num-key-heads: 4
+ --linear-num-value-heads: 8
+ # Add MoE args
+ --num-experts: 32
+ --moe-ffn-hidden-size: 64
+ --moe-shared-expert-intermediate-size: 64
+ --moe-shared-expert-gate: true
+ --moe-router-load-balancing-type: aux_loss
+ --moe-router-topk: 8
+ --disable-bias-linear: true
+ --moe-router-dtype: fp32
+ # Add logging args
+ --log-params-norm: true
+ --log-num-zeros-in-grad: true
+ --log-validation-ppl-to-tensorboard: true
+ --log-timers-to-tensorboard: true
+ --tensorboard-dir: ${TENSORBOARD_PATH}
+ --micro-batch-size: 4
+ --global-batch-size: 32
+ --seq-length: 1024
+ --max-position-embeddings: 1024
+ --train-iters: 50
+ --timing-log-level: 0
+ --lr-decay-iters: 320000
+ --save: ${CHECKPOINT_SAVE_PATH}
+ --load: ${CHECKPOINT_LOAD_PATH}
+ --data-path: ${DATA_PATH}/text/the_pile/shard00/my-gpt3_00_text_document
+ --vocab-file: ${DATA_PATH}/text/the_pile/shard00/bpe/vocab.json
+ --merge-file: ${DATA_PATH}/text/the_pile/shard00/bpe/merges.txt
+ --split: 949,50,1
+ --distributed-backend: nccl
+ --lr: 0.00015
+ --lr-decay-style: cosine
+ --min-lr: 1.0e-5
+ --weight-decay: 1e-2
+ --clip-grad: 1.0
+ --lr-warmup-fraction: .01
+ --log-interval: 1
+ --save-interval: 25
+ --eval-interval: 1000
+ --eval-iters: 10
+ --transformer-impl: transformer_engine
+ --tensor-model-parallel-size: 2
+ --pipeline-model-parallel-size: 1
+ --sequence-parallel: true
+ --untie-embeddings-and-output-weights: true
+ --deterministic-mode: true
+ --no-gradient-accumulation-fusion: true
+ --attention-softmax-in-fp32: true
+ --use-mcore-models: true
+ --ckpt-format: torch_dist
+ --data-cache-path: ${DATA_CACHE_PATH}
+ --bf16: true
+ --attention-backend: unfused
+ --log-memory-to-tensorboard: true
+TEST_TYPE: ckpt-resume
diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_cp2/golden_values_dev_dgx_h100.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_cp2/golden_values_dev_dgx_h100.json
index 02b4683ea0b..81005995dad 100644
--- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_cp2/golden_values_dev_dgx_h100.json
+++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_cp2/golden_values_dev_dgx_h100.json
@@ -4,55 +4,55 @@
"end_step": 50,
"step_interval": 1,
"values": {
- "1": 10.86535,
- "2": 10.85873,
- "3": 10.86284,
- "4": 10.84009,
+ "1": 10.86539,
+ "2": 10.85871,
+ "3": 10.86282,
+ "4": 10.84007,
"5": 10.87856,
- "6": 10.88856,
- "7": 10.86532,
- "8": 10.86017,
- "9": 10.8599,
- "10": 10.82981,
- "11": 10.8895,
- "12": 10.8751,
- "13": 10.87423,
+ "6": 10.88852,
+ "7": 10.86536,
+ "8": 10.86015,
+ "9": 10.85991,
+ "10": 10.82982,
+ "11": 10.88947,
+ "12": 10.87511,
+ "13": 10.87422,
"14": 10.89675,
- "15": 10.82054,
- "16": 10.82504,
+ "15": 10.82056,
+ "16": 10.82497,
"17": 10.78983,
"18": 10.81029,
- "19": 10.80535,
- "20": 10.70398,
- "21": 10.66993,
- "22": 10.50643,
- "23": 10.69004,
- "24": 10.56314,
- "25": 10.4942,
- "26": 10.56628,
- "27": 10.58025,
+ "19": 10.80528,
+ "20": 10.70396,
+ "21": 10.6699,
+ "22": 10.50641,
+ "23": 10.69006,
+ "24": 10.56312,
+ "25": 10.49418,
+ "26": 10.56627,
+ "27": 10.58023,
"28": 10.51571,
- "29": 10.55299,
- "30": 10.30549,
- "31": 10.02245,
- "32": 10.40614,
+ "29": 10.55296,
+ "30": 10.30551,
+ "31": 10.02244,
+ "32": 10.40618,
"33": 10.39874,
- "34": 10.13771,
+ "34": 10.1377,
"35": 10.20184,
- "36": 10.16052,
- "37": 10.28973,
- "38": 10.11474,
+ "36": 10.1605,
+ "37": 10.28975,
+ "38": 10.11483,
"39": 10.361,
- "40": 10.01903,
+ "40": 10.01905,
"41": 10.07292,
- "42": 10.14698,
- "43": 9.74687,
- "44": 9.87766,
- "45": 9.74966,
- "46": 9.73383,
- "47": 10.07535,
- "48": 9.78068,
- "49": 9.44784,
+ "42": 10.14697,
+ "43": 9.74684,
+ "44": 9.87763,
+ "45": 9.74962,
+ "46": 9.73382,
+ "47": 10.07536,
+ "48": 9.78071,
+ "49": 9.44783,
"50": 9.8399
}
},
@@ -61,56 +61,56 @@
"end_step": 50,
"step_interval": 1,
"values": {
- "1": 653.0,
- "2": 642.0,
- "3": 630.0,
- "4": 585.0,
- "5": 635.0,
- "6": 687.0,
- "7": 615.0,
- "8": 601.0,
- "9": 607.0,
- "10": 522.0,
- "11": 637.0,
- "12": 675.0,
- "13": 649.0,
- "14": 648.0,
- "15": 640.0,
- "16": 602.0,
- "17": 668.0,
- "18": 634.0,
- "19": 593.0,
- "20": 579.0,
- "21": 633.0,
- "22": 597.0,
- "23": 756.0,
- "24": 612.0,
- "25": 591.0,
- "26": 620.0,
- "27": 700.0,
- "28": 705.0,
- "29": 795.0,
- "30": 752.0,
- "31": 628.0,
- "32": 712.0,
- "33": 752.0,
- "34": 737.0,
- "35": 741.0,
- "36": 770.0,
- "37": 861.0,
- "38": 823.0,
- "39": 812.0,
- "40": 814.0,
- "41": 826.0,
- "42": 801.0,
- "43": 769.0,
- "44": 822.0,
- "45": 777.0,
- "46": 828.0,
- "47": 878.0,
- "48": 915.0,
- "49": 908.0,
- "50": 848.0
+ "1": 572.0,
+ "2": 656.0,
+ "3": 649.0,
+ "4": 631.0,
+ "5": 658.0,
+ "6": 636.0,
+ "7": 636.0,
+ "8": 542.0,
+ "9": 653.0,
+ "10": 551.0,
+ "11": 681.0,
+ "12": 642.0,
+ "13": 624.0,
+ "14": 658.0,
+ "15": 682.0,
+ "16": 659.0,
+ "17": 620.0,
+ "18": 603.0,
+ "19": 634.0,
+ "20": 639.0,
+ "21": 634.0,
+ "22": 602.0,
+ "23": 731.0,
+ "24": 620.0,
+ "25": 611.0,
+ "26": 626.0,
+ "27": 683.0,
+ "28": 668.0,
+ "29": 713.0,
+ "30": 712.0,
+ "31": 616.0,
+ "32": 786.0,
+ "33": 800.0,
+ "34": 702.0,
+ "35": 684.0,
+ "36": 664.0,
+ "37": 831.0,
+ "38": 802.0,
+ "39": 919.0,
+ "40": 802.0,
+ "41": 791.0,
+ "42": 840.0,
+ "43": 718.0,
+ "44": 756.0,
+ "45": 765.0,
+ "46": 809.0,
+ "47": 839.0,
+ "48": 827.0,
+ "49": 935.0,
+ "50": 839.0
}
},
"mem-allocated-bytes": {
@@ -118,56 +118,56 @@
"end_step": 50,
"step_interval": 1,
"values": {
- "1": 510689792.0,
- "2": 510689792.0,
- "3": 510689792.0,
- "4": 510689792.0,
- "5": 510689792.0,
- "6": 510689792.0,
- "7": 510689792.0,
- "8": 510689792.0,
- "9": 510689792.0,
- "10": 510689792.0,
- "11": 510689792.0,
- "12": 510689792.0,
- "13": 510689792.0,
- "14": 510689792.0,
- "15": 510689792.0,
- "16": 510689792.0,
- "17": 510689792.0,
- "18": 510689792.0,
- "19": 510689792.0,
- "20": 510689792.0,
- "21": 510689792.0,
- "22": 510689792.0,
- "23": 510689792.0,
- "24": 510689792.0,
- "25": 510689792.0,
- "26": 510689792.0,
- "27": 510689792.0,
- "28": 510689792.0,
- "29": 510689792.0,
- "30": 510689792.0,
- "31": 510689792.0,
- "32": 510689792.0,
- "33": 510689792.0,
- "34": 510689792.0,
- "35": 510689792.0,
- "36": 510689792.0,
- "37": 510689792.0,
- "38": 510689792.0,
- "39": 510689792.0,
- "40": 510689792.0,
- "41": 510689792.0,
- "42": 510689792.0,
- "43": 510689792.0,
- "44": 510689792.0,
- "45": 510689792.0,
- "46": 510689792.0,
- "47": 510689792.0,
- "48": 510689792.0,
- "49": 510689792.0,
- "50": 510689792.0
+ "1": 509641216.0,
+ "2": 509641216.0,
+ "3": 509641216.0,
+ "4": 509641216.0,
+ "5": 509641216.0,
+ "6": 509641216.0,
+ "7": 509641216.0,
+ "8": 509641216.0,
+ "9": 509641216.0,
+ "10": 509641216.0,
+ "11": 509641216.0,
+ "12": 509641216.0,
+ "13": 509641216.0,
+ "14": 509641216.0,
+ "15": 509641216.0,
+ "16": 509641216.0,
+ "17": 509641216.0,
+ "18": 509641216.0,
+ "19": 509641216.0,
+ "20": 509641216.0,
+ "21": 509641216.0,
+ "22": 509641216.0,
+ "23": 509641216.0,
+ "24": 509641216.0,
+ "25": 509641216.0,
+ "26": 509641216.0,
+ "27": 509641216.0,
+ "28": 509641216.0,
+ "29": 509641216.0,
+ "30": 509641216.0,
+ "31": 509641216.0,
+ "32": 509641216.0,
+ "33": 509641216.0,
+ "34": 509641216.0,
+ "35": 509641216.0,
+ "36": 509641216.0,
+ "37": 509641216.0,
+ "38": 509641216.0,
+ "39": 509641216.0,
+ "40": 509641216.0,
+ "41": 509641216.0,
+ "42": 509641216.0,
+ "43": 509641216.0,
+ "44": 509641216.0,
+ "45": 509641216.0,
+ "46": 509641216.0,
+ "47": 509641216.0,
+ "48": 509641216.0,
+ "49": 509641216.0,
+ "50": 509641216.0
}
},
"mem-max-allocated-bytes": {
@@ -175,56 +175,56 @@
"end_step": 50,
"step_interval": 1,
"values": {
- "1": 757801472.0,
- "2": 933156352.0,
- "3": 933156352.0,
- "4": 933156352.0,
- "5": 933156352.0,
- "6": 933156352.0,
- "7": 933156352.0,
- "8": 933156352.0,
- "9": 933156352.0,
- "10": 933156352.0,
- "11": 933156352.0,
- "12": 933156352.0,
- "13": 933156352.0,
- "14": 933156352.0,
- "15": 933156352.0,
- "16": 933156352.0,
- "17": 933156352.0,
- "18": 933156352.0,
- "19": 933156352.0,
- "20": 933156352.0,
- "21": 933156352.0,
- "22": 933156352.0,
- "23": 933156352.0,
- "24": 933156352.0,
- "25": 933156352.0,
- "26": 933156352.0,
- "27": 933156352.0,
- "28": 933156352.0,
- "29": 933156352.0,
- "30": 933156352.0,
- "31": 933156352.0,
- "32": 933156352.0,
- "33": 933156352.0,
- "34": 933156352.0,
- "35": 933156352.0,
- "36": 933156352.0,
- "37": 933156352.0,
- "38": 933156352.0,
- "39": 933156352.0,
- "40": 933156352.0,
- "41": 933156352.0,
- "42": 933156352.0,
- "43": 933156352.0,
- "44": 933156352.0,
- "45": 933156352.0,
- "46": 933156352.0,
- "47": 933156352.0,
- "48": 933156352.0,
- "49": 933156352.0,
- "50": 933156352.0
+ "1": 756751872.0,
+ "2": 932632064.0,
+ "3": 932632064.0,
+ "4": 932632064.0,
+ "5": 932632064.0,
+ "6": 932632064.0,
+ "7": 932632064.0,
+ "8": 932632064.0,
+ "9": 932632064.0,
+ "10": 933679616.0,
+ "11": 933679616.0,
+ "12": 933679616.0,
+ "13": 933679616.0,
+ "14": 933679616.0,
+ "15": 933679616.0,
+ "16": 933679616.0,
+ "17": 933679616.0,
+ "18": 933679616.0,
+ "19": 933679616.0,
+ "20": 933679616.0,
+ "21": 933679616.0,
+ "22": 933679616.0,
+ "23": 933679616.0,
+ "24": 933679616.0,
+ "25": 933679616.0,
+ "26": 933679616.0,
+ "27": 933679616.0,
+ "28": 933679616.0,
+ "29": 933679616.0,
+ "30": 933679616.0,
+ "31": 933679616.0,
+ "32": 933679616.0,
+ "33": 933679616.0,
+ "34": 933679616.0,
+ "35": 933679616.0,
+ "36": 933679616.0,
+ "37": 933679616.0,
+ "38": 933679616.0,
+ "39": 933679616.0,
+ "40": 933679616.0,
+ "41": 933679616.0,
+ "42": 933679616.0,
+ "43": 933679616.0,
+ "44": 933679616.0,
+ "45": 933680640.0,
+ "46": 933680640.0,
+ "47": 933680640.0,
+ "48": 933680640.0,
+ "49": 933680640.0,
+ "50": 933680640.0
}
},
"iteration-time": {
@@ -232,56 +232,56 @@
"end_step": 50,
"step_interval": 1,
"values": {
- "1": 15.78036,
- "2": 0.34723,
- "3": 0.33492,
- "4": 0.3292,
- "5": 0.33036,
- "6": 0.34971,
- "7": 0.33848,
- "8": 0.33262,
- "9": 0.34028,
- "10": 0.3518,
- "11": 0.34239,
- "12": 0.33211,
- "13": 0.32961,
- "14": 0.33263,
- "15": 0.32808,
- "16": 0.33152,
- "17": 0.33313,
- "18": 0.329,
- "19": 0.3317,
- "20": 0.33143,
- "21": 0.34166,
- "22": 0.33873,
- "23": 0.34817,
- "24": 0.3415,
- "25": 0.34495,
- "26": 0.32592,
- "27": 0.32935,
- "28": 0.33233,
- "29": 0.328,
- "30": 0.32746,
- "31": 0.3275,
- "32": 0.327,
- "33": 0.32765,
- "34": 0.32542,
- "35": 0.32703,
- "36": 0.33052,
- "37": 0.33413,
- "38": 0.32701,
- "39": 0.32816,
- "40": 0.32555,
- "41": 0.33676,
- "42": 0.33367,
- "43": 0.33748,
- "44": 0.33125,
- "45": 0.32793,
- "46": 0.33387,
- "47": 0.32628,
- "48": 0.32993,
- "49": 0.32747,
- "50": 0.327
+ "1": 42.02117,
+ "2": 0.34315,
+ "3": 0.31657,
+ "4": 0.29715,
+ "5": 0.29109,
+ "6": 0.28638,
+ "7": 0.28745,
+ "8": 0.29318,
+ "9": 0.30075,
+ "10": 0.29578,
+ "11": 0.30101,
+ "12": 0.29769,
+ "13": 0.2954,
+ "14": 0.2989,
+ "15": 0.29627,
+ "16": 0.29342,
+ "17": 0.29396,
+ "18": 0.29431,
+ "19": 0.29408,
+ "20": 0.29286,
+ "21": 0.29361,
+ "22": 0.29448,
+ "23": 0.29521,
+ "24": 0.29494,
+ "25": 0.29812,
+ "26": 0.29413,
+ "27": 0.2949,
+ "28": 0.29469,
+ "29": 0.29393,
+ "30": 0.29682,
+ "31": 0.2951,
+ "32": 0.29532,
+ "33": 0.29449,
+ "34": 0.29334,
+ "35": 0.29679,
+ "36": 0.29557,
+ "37": 0.29495,
+ "38": 0.29826,
+ "39": 0.29574,
+ "40": 0.2972,
+ "41": 0.29568,
+ "42": 0.29643,
+ "43": 0.29627,
+ "44": 0.29491,
+ "45": 0.29476,
+ "46": 0.29707,
+ "47": 0.35995,
+ "48": 0.28743,
+ "49": 0.28604,
+ "50": 0.28593
}
}
}
\ No newline at end of file
diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_cp2_calculate_per_token_loss/golden_values_dev_dgx_h100.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_cp2_calculate_per_token_loss/golden_values_dev_dgx_h100.json
index f2adbef4530..873d08f92a3 100644
--- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_cp2_calculate_per_token_loss/golden_values_dev_dgx_h100.json
+++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_cp2_calculate_per_token_loss/golden_values_dev_dgx_h100.json
@@ -4,55 +4,55 @@
"end_step": 50,
"step_interval": 1,
"values": {
- "1": 10.86535,
- "2": 10.85873,
- "3": 10.86284,
- "4": 10.84009,
+ "1": 10.86539,
+ "2": 10.85871,
+ "3": 10.86282,
+ "4": 10.84007,
"5": 10.87856,
- "6": 10.88856,
- "7": 10.86532,
- "8": 10.86017,
- "9": 10.8599,
- "10": 10.82981,
- "11": 10.8895,
- "12": 10.8751,
- "13": 10.87423,
+ "6": 10.88852,
+ "7": 10.86536,
+ "8": 10.86015,
+ "9": 10.85991,
+ "10": 10.82982,
+ "11": 10.88947,
+ "12": 10.87511,
+ "13": 10.87422,
"14": 10.89675,
- "15": 10.82054,
- "16": 10.82504,
+ "15": 10.82056,
+ "16": 10.82497,
"17": 10.78983,
"18": 10.81029,
- "19": 10.80535,
- "20": 10.70398,
- "21": 10.66993,
- "22": 10.50643,
- "23": 10.69004,
- "24": 10.56314,
- "25": 10.4942,
- "26": 10.56628,
- "27": 10.58025,
+ "19": 10.80528,
+ "20": 10.70396,
+ "21": 10.6699,
+ "22": 10.50641,
+ "23": 10.69006,
+ "24": 10.56312,
+ "25": 10.49418,
+ "26": 10.56627,
+ "27": 10.58023,
"28": 10.51571,
- "29": 10.55299,
- "30": 10.30549,
- "31": 10.02245,
- "32": 10.40614,
+ "29": 10.55296,
+ "30": 10.30551,
+ "31": 10.02244,
+ "32": 10.40618,
"33": 10.39874,
- "34": 10.13771,
+ "34": 10.1377,
"35": 10.20184,
- "36": 10.16052,
- "37": 10.28973,
- "38": 10.11474,
+ "36": 10.1605,
+ "37": 10.28975,
+ "38": 10.11483,
"39": 10.361,
- "40": 10.01903,
+ "40": 10.01905,
"41": 10.07292,
- "42": 10.14698,
- "43": 9.74687,
- "44": 9.87766,
- "45": 9.74966,
- "46": 9.73383,
- "47": 10.07535,
- "48": 9.78068,
- "49": 9.44784,
+ "42": 10.14697,
+ "43": 9.74684,
+ "44": 9.87763,
+ "45": 9.74962,
+ "46": 9.73382,
+ "47": 10.07536,
+ "48": 9.78071,
+ "49": 9.44783,
"50": 9.8399
}
},
@@ -61,56 +61,56 @@
"end_step": 50,
"step_interval": 1,
"values": {
- "1": 653.0,
- "2": 642.0,
- "3": 630.0,
- "4": 585.0,
- "5": 635.0,
- "6": 687.0,
- "7": 615.0,
- "8": 601.0,
- "9": 607.0,
- "10": 522.0,
- "11": 637.0,
- "12": 675.0,
- "13": 649.0,
- "14": 648.0,
- "15": 640.0,
- "16": 602.0,
- "17": 668.0,
- "18": 634.0,
- "19": 593.0,
- "20": 579.0,
- "21": 633.0,
- "22": 597.0,
- "23": 756.0,
- "24": 612.0,
- "25": 591.0,
- "26": 620.0,
- "27": 700.0,
- "28": 705.0,
- "29": 795.0,
- "30": 752.0,
- "31": 628.0,
- "32": 712.0,
- "33": 752.0,
- "34": 737.0,
- "35": 741.0,
- "36": 770.0,
- "37": 861.0,
- "38": 823.0,
- "39": 812.0,
- "40": 814.0,
- "41": 826.0,
- "42": 801.0,
- "43": 769.0,
- "44": 822.0,
- "45": 777.0,
- "46": 828.0,
- "47": 878.0,
- "48": 915.0,
- "49": 908.0,
- "50": 848.0
+ "1": 572.0,
+ "2": 656.0,
+ "3": 649.0,
+ "4": 631.0,
+ "5": 658.0,
+ "6": 636.0,
+ "7": 636.0,
+ "8": 542.0,
+ "9": 653.0,
+ "10": 551.0,
+ "11": 681.0,
+ "12": 642.0,
+ "13": 624.0,
+ "14": 658.0,
+ "15": 682.0,
+ "16": 659.0,
+ "17": 620.0,
+ "18": 603.0,
+ "19": 634.0,
+ "20": 639.0,
+ "21": 634.0,
+ "22": 602.0,
+ "23": 731.0,
+ "24": 620.0,
+ "25": 611.0,
+ "26": 626.0,
+ "27": 683.0,
+ "28": 668.0,
+ "29": 713.0,
+ "30": 712.0,
+ "31": 616.0,
+ "32": 786.0,
+ "33": 800.0,
+ "34": 702.0,
+ "35": 684.0,
+ "36": 664.0,
+ "37": 831.0,
+ "38": 802.0,
+ "39": 919.0,
+ "40": 802.0,
+ "41": 791.0,
+ "42": 840.0,
+ "43": 718.0,
+ "44": 756.0,
+ "45": 765.0,
+ "46": 809.0,
+ "47": 839.0,
+ "48": 827.0,
+ "49": 935.0,
+ "50": 839.0
}
},
"mem-allocated-bytes": {
@@ -118,56 +118,56 @@
"end_step": 50,
"step_interval": 1,
"values": {
- "1": 510689792.0,
- "2": 510689792.0,
- "3": 510689792.0,
- "4": 510689792.0,
- "5": 510689792.0,
- "6": 510689792.0,
- "7": 510689792.0,
- "8": 510689792.0,
- "9": 510689792.0,
- "10": 510689792.0,
- "11": 510689792.0,
- "12": 510689792.0,
- "13": 510689792.0,
- "14": 510689792.0,
- "15": 510689792.0,
- "16": 510689792.0,
- "17": 510689792.0,
- "18": 510689792.0,
- "19": 510689792.0,
- "20": 510689792.0,
- "21": 510689792.0,
- "22": 510689792.0,
- "23": 510689792.0,
- "24": 510689792.0,
- "25": 510689792.0,
- "26": 510689792.0,
- "27": 510689792.0,
- "28": 510689792.0,
- "29": 510689792.0,
- "30": 510689792.0,
- "31": 510689792.0,
- "32": 510689792.0,
- "33": 510689792.0,
- "34": 510689792.0,
- "35": 510689792.0,
- "36": 510689792.0,
- "37": 510689792.0,
- "38": 510689792.0,
- "39": 510689792.0,
- "40": 510689792.0,
- "41": 510689792.0,
- "42": 510689792.0,
- "43": 510689792.0,
- "44": 510689792.0,
- "45": 510689792.0,
- "46": 510689792.0,
- "47": 510689792.0,
- "48": 510689792.0,
- "49": 510689792.0,
- "50": 510689792.0
+ "1": 511214080.0,
+ "2": 511214080.0,
+ "3": 511214080.0,
+ "4": 511214080.0,
+ "5": 511214080.0,
+ "6": 511214080.0,
+ "7": 511214080.0,
+ "8": 511214080.0,
+ "9": 511214080.0,
+ "10": 511214080.0,
+ "11": 511214080.0,
+ "12": 511214080.0,
+ "13": 511214080.0,
+ "14": 511214080.0,
+ "15": 511214080.0,
+ "16": 511214080.0,
+ "17": 511214080.0,
+ "18": 511214080.0,
+ "19": 511214080.0,
+ "20": 511214080.0,
+ "21": 511214080.0,
+ "22": 511214080.0,
+ "23": 511214080.0,
+ "24": 511214080.0,
+ "25": 511214080.0,
+ "26": 511214080.0,
+ "27": 511214080.0,
+ "28": 511214080.0,
+ "29": 511214080.0,
+ "30": 511214080.0,
+ "31": 511214080.0,
+ "32": 511214080.0,
+ "33": 511214080.0,
+ "34": 511214080.0,
+ "35": 511214080.0,
+ "36": 511214080.0,
+ "37": 511214080.0,
+ "38": 511214080.0,
+ "39": 511214080.0,
+ "40": 511214080.0,
+ "41": 511214080.0,
+ "42": 511214080.0,
+ "43": 511214080.0,
+ "44": 511214080.0,
+ "45": 511214080.0,
+ "46": 511214080.0,
+ "47": 511214080.0,
+ "48": 511214080.0,
+ "49": 511214080.0,
+ "50": 511214080.0
}
},
"mem-max-allocated-bytes": {
@@ -175,56 +175,56 @@
"end_step": 50,
"step_interval": 1,
"values": {
- "1": 759898624.0,
- "2": 933156352.0,
- "3": 933156352.0,
- "4": 933156352.0,
- "5": 933156352.0,
- "6": 933156352.0,
- "7": 933156352.0,
- "8": 933156352.0,
- "9": 933156352.0,
- "10": 933156352.0,
- "11": 933156352.0,
- "12": 933156352.0,
- "13": 933156352.0,
- "14": 933156352.0,
- "15": 933156352.0,
- "16": 933156352.0,
- "17": 933156352.0,
- "18": 933156352.0,
- "19": 933156352.0,
- "20": 933156352.0,
- "21": 933156352.0,
- "22": 933156352.0,
- "23": 933156352.0,
- "24": 933156352.0,
- "25": 933156352.0,
- "26": 933156352.0,
- "27": 933156352.0,
- "28": 933156352.0,
- "29": 933156352.0,
- "30": 933156352.0,
- "31": 933156352.0,
- "32": 933156352.0,
- "33": 933156352.0,
- "34": 933156352.0,
- "35": 933156352.0,
- "36": 933156352.0,
- "37": 933156352.0,
- "38": 933156352.0,
- "39": 933156352.0,
- "40": 933156352.0,
- "41": 933156352.0,
- "42": 933156352.0,
- "43": 933156352.0,
- "44": 933156352.0,
- "45": 933156352.0,
- "46": 933156352.0,
- "47": 933156352.0,
- "48": 933156352.0,
- "49": 933156352.0,
- "50": 933156352.0
+ "1": 756753920.0,
+ "2": 935776768.0,
+ "3": 935777792.0,
+ "4": 935777792.0,
+ "5": 935777792.0,
+ "6": 935777792.0,
+ "7": 935777792.0,
+ "8": 935777792.0,
+ "9": 935777792.0,
+ "10": 935777792.0,
+ "11": 935777792.0,
+ "12": 935777792.0,
+ "13": 935777792.0,
+ "14": 935777792.0,
+ "15": 935777792.0,
+ "16": 935777792.0,
+ "17": 935777792.0,
+ "18": 935777792.0,
+ "19": 935777792.0,
+ "20": 935777792.0,
+ "21": 935777792.0,
+ "22": 935777792.0,
+ "23": 935777792.0,
+ "24": 935777792.0,
+ "25": 935777792.0,
+ "26": 935777792.0,
+ "27": 935777792.0,
+ "28": 935777792.0,
+ "29": 935777792.0,
+ "30": 935777792.0,
+ "31": 935777792.0,
+ "32": 935777792.0,
+ "33": 935777792.0,
+ "34": 935777792.0,
+ "35": 935777792.0,
+ "36": 935777792.0,
+ "37": 935777792.0,
+ "38": 935777792.0,
+ "39": 935777792.0,
+ "40": 935777792.0,
+ "41": 935777792.0,
+ "42": 935777792.0,
+ "43": 935777792.0,
+ "44": 935777792.0,
+ "45": 935777792.0,
+ "46": 935777792.0,
+ "47": 935777792.0,
+ "48": 935777792.0,
+ "49": 935777792.0,
+ "50": 935777792.0
}
},
"iteration-time": {
@@ -232,56 +232,56 @@
"end_step": 50,
"step_interval": 1,
"values": {
- "1": 16.72434,
- "2": 0.40342,
- "3": 0.32477,
- "4": 0.32459,
- "5": 0.32511,
- "6": 0.32478,
- "7": 0.32469,
- "8": 0.32479,
- "9": 0.32229,
- "10": 0.32534,
- "11": 0.32568,
- "12": 0.32325,
- "13": 0.3234,
- "14": 0.32735,
- "15": 0.32264,
- "16": 0.32664,
- "17": 0.32289,
- "18": 0.32328,
- "19": 0.32997,
- "20": 0.32955,
- "21": 0.32699,
- "22": 0.3292,
- "23": 0.32982,
- "24": 0.32452,
- "25": 0.32644,
- "26": 0.32596,
- "27": 0.32426,
- "28": 0.32527,
- "29": 0.32409,
- "30": 0.32549,
- "31": 0.32259,
- "32": 0.32488,
- "33": 0.32331,
- "34": 0.3242,
- "35": 0.3261,
- "36": 0.32048,
- "37": 0.32127,
- "38": 0.32479,
- "39": 0.32338,
- "40": 0.32137,
- "41": 0.32292,
- "42": 0.32202,
- "43": 0.32321,
- "44": 0.32105,
- "45": 0.32265,
- "46": 0.32148,
- "47": 0.32443,
- "48": 0.32158,
- "49": 0.32089,
- "50": 0.32389
+ "1": 44.927,
+ "2": 0.34811,
+ "3": 0.31209,
+ "4": 0.29049,
+ "5": 0.28904,
+ "6": 0.28728,
+ "7": 0.28884,
+ "8": 0.29393,
+ "9": 0.28153,
+ "10": 0.28717,
+ "11": 0.28861,
+ "12": 0.29265,
+ "13": 0.29015,
+ "14": 0.29189,
+ "15": 0.29081,
+ "16": 0.29742,
+ "17": 0.29933,
+ "18": 0.29528,
+ "19": 0.29058,
+ "20": 0.29304,
+ "21": 0.29307,
+ "22": 0.29297,
+ "23": 0.2889,
+ "24": 0.29028,
+ "25": 0.29626,
+ "26": 0.29321,
+ "27": 0.29347,
+ "28": 0.29303,
+ "29": 0.2812,
+ "30": 0.28971,
+ "31": 0.28878,
+ "32": 0.28499,
+ "33": 0.28119,
+ "34": 0.27908,
+ "35": 0.28101,
+ "36": 0.2794,
+ "37": 0.2798,
+ "38": 0.27799,
+ "39": 0.28519,
+ "40": 0.28246,
+ "41": 0.28126,
+ "42": 0.28572,
+ "43": 0.28647,
+ "44": 0.28772,
+ "45": 0.28736,
+ "46": 0.29677,
+ "47": 0.29247,
+ "48": 0.29174,
+ "49": 0.29182,
+ "50": 0.29085
}
}
}
\ No newline at end of file
diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_cp2_etp4_calculate_per_token_loss_dp_last/golden_values_dev_dgx_h100.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_cp2_etp4_calculate_per_token_loss_dp_last/golden_values_dev_dgx_h100.json
index f64661824cb..84e2331d673 100644
--- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_cp2_etp4_calculate_per_token_loss_dp_last/golden_values_dev_dgx_h100.json
+++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_cp2_etp4_calculate_per_token_loss_dp_last/golden_values_dev_dgx_h100.json
@@ -4,55 +4,55 @@
"end_step": 50,
"step_interval": 1,
"values": {
- "1": 10.86535,
- "2": 10.85873,
- "3": 10.86284,
- "4": 10.84009,
+ "1": 10.86539,
+ "2": 10.85871,
+ "3": 10.86282,
+ "4": 10.84007,
"5": 10.87856,
- "6": 10.88856,
- "7": 10.86532,
- "8": 10.86017,
- "9": 10.8599,
- "10": 10.82981,
- "11": 10.8895,
- "12": 10.8751,
- "13": 10.87423,
+ "6": 10.88852,
+ "7": 10.86536,
+ "8": 10.86015,
+ "9": 10.85991,
+ "10": 10.82982,
+ "11": 10.88947,
+ "12": 10.87511,
+ "13": 10.87422,
"14": 10.89675,
- "15": 10.82054,
- "16": 10.82504,
+ "15": 10.82056,
+ "16": 10.82497,
"17": 10.78983,
"18": 10.81029,
- "19": 10.80535,
- "20": 10.70398,
- "21": 10.66993,
- "22": 10.50643,
- "23": 10.69004,
- "24": 10.56314,
- "25": 10.4942,
- "26": 10.56628,
- "27": 10.58025,
+ "19": 10.80528,
+ "20": 10.70396,
+ "21": 10.6699,
+ "22": 10.50641,
+ "23": 10.69006,
+ "24": 10.56312,
+ "25": 10.49418,
+ "26": 10.56627,
+ "27": 10.58023,
"28": 10.51571,
- "29": 10.55299,
- "30": 10.30549,
- "31": 10.02245,
- "32": 10.40614,
+ "29": 10.55296,
+ "30": 10.30551,
+ "31": 10.02244,
+ "32": 10.40618,
"33": 10.39874,
- "34": 10.13771,
+ "34": 10.1377,
"35": 10.20184,
- "36": 10.16052,
- "37": 10.28973,
- "38": 10.11474,
+ "36": 10.1605,
+ "37": 10.28975,
+ "38": 10.11483,
"39": 10.361,
- "40": 10.01903,
+ "40": 10.01905,
"41": 10.07292,
- "42": 10.14698,
- "43": 9.74687,
- "44": 9.87766,
- "45": 9.74966,
- "46": 9.73383,
- "47": 10.07535,
- "48": 9.78068,
- "49": 9.44784,
+ "42": 10.14697,
+ "43": 9.74684,
+ "44": 9.87763,
+ "45": 9.74962,
+ "46": 9.73382,
+ "47": 10.07536,
+ "48": 9.78071,
+ "49": 9.44783,
"50": 9.8399
}
},
@@ -61,56 +61,56 @@
"end_step": 50,
"step_interval": 1,
"values": {
- "1": 653.0,
- "2": 642.0,
- "3": 630.0,
- "4": 585.0,
- "5": 635.0,
- "6": 687.0,
- "7": 615.0,
- "8": 601.0,
- "9": 607.0,
- "10": 522.0,
- "11": 637.0,
- "12": 675.0,
- "13": 649.0,
- "14": 648.0,
- "15": 640.0,
- "16": 602.0,
- "17": 668.0,
- "18": 634.0,
- "19": 593.0,
- "20": 579.0,
- "21": 633.0,
- "22": 597.0,
- "23": 756.0,
- "24": 612.0,
- "25": 591.0,
- "26": 620.0,
- "27": 700.0,
- "28": 705.0,
- "29": 795.0,
- "30": 752.0,
- "31": 628.0,
- "32": 712.0,
- "33": 752.0,
- "34": 737.0,
- "35": 741.0,
- "36": 770.0,
- "37": 861.0,
- "38": 823.0,
- "39": 812.0,
- "40": 814.0,
- "41": 826.0,
- "42": 801.0,
- "43": 769.0,
- "44": 822.0,
- "45": 777.0,
- "46": 828.0,
- "47": 878.0,
- "48": 915.0,
- "49": 908.0,
- "50": 848.0
+ "1": 572.0,
+ "2": 656.0,
+ "3": 649.0,
+ "4": 631.0,
+ "5": 658.0,
+ "6": 636.0,
+ "7": 636.0,
+ "8": 542.0,
+ "9": 653.0,
+ "10": 551.0,
+ "11": 681.0,
+ "12": 642.0,
+ "13": 624.0,
+ "14": 658.0,
+ "15": 682.0,
+ "16": 659.0,
+ "17": 620.0,
+ "18": 603.0,
+ "19": 634.0,
+ "20": 639.0,
+ "21": 634.0,
+ "22": 602.0,
+ "23": 731.0,
+ "24": 620.0,
+ "25": 611.0,
+ "26": 626.0,
+ "27": 683.0,
+ "28": 668.0,
+ "29": 713.0,
+ "30": 712.0,
+ "31": 616.0,
+ "32": 786.0,
+ "33": 800.0,
+ "34": 702.0,
+ "35": 684.0,
+ "36": 664.0,
+ "37": 831.0,
+ "38": 802.0,
+ "39": 919.0,
+ "40": 802.0,
+ "41": 791.0,
+ "42": 840.0,
+ "43": 718.0,
+ "44": 756.0,
+ "45": 765.0,
+ "46": 809.0,
+ "47": 839.0,
+ "48": 827.0,
+ "49": 935.0,
+ "50": 839.0
}
},
"mem-allocated-bytes": {
@@ -118,56 +118,56 @@
"end_step": 50,
"step_interval": 1,
"values": {
- "1": 510689792.0,
- "2": 510689792.0,
- "3": 510689792.0,
- "4": 510689792.0,
- "5": 510689792.0,
- "6": 510689792.0,
- "7": 510689792.0,
- "8": 510689792.0,
- "9": 510689792.0,
- "10": 510689792.0,
- "11": 510689792.0,
- "12": 510689792.0,
- "13": 510689792.0,
- "14": 510689792.0,
- "15": 510689792.0,
- "16": 510689792.0,
- "17": 510689792.0,
- "18": 510689792.0,
- "19": 510689792.0,
- "20": 510689792.0,
- "21": 510689792.0,
- "22": 510689792.0,
- "23": 510689792.0,
- "24": 510689792.0,
- "25": 510689792.0,
- "26": 510689792.0,
- "27": 510689792.0,
- "28": 510689792.0,
- "29": 510689792.0,
- "30": 510689792.0,
- "31": 510689792.0,
- "32": 510689792.0,
- "33": 510689792.0,
- "34": 510689792.0,
- "35": 510689792.0,
- "36": 510689792.0,
- "37": 510689792.0,
- "38": 510689792.0,
- "39": 510689792.0,
- "40": 510689792.0,
- "41": 510689792.0,
- "42": 510689792.0,
- "43": 510689792.0,
- "44": 510689792.0,
- "45": 510689792.0,
- "46": 510689792.0,
- "47": 510689792.0,
- "48": 510689792.0,
- "49": 510689792.0,
- "50": 510689792.0
+ "1": 511214080.0,
+ "2": 511214080.0,
+ "3": 511214080.0,
+ "4": 511214080.0,
+ "5": 511214080.0,
+ "6": 511214080.0,
+ "7": 511214080.0,
+ "8": 511214080.0,
+ "9": 511214080.0,
+ "10": 511214080.0,
+ "11": 511214080.0,
+ "12": 511214080.0,
+ "13": 511214080.0,
+ "14": 511214080.0,
+ "15": 511214080.0,
+ "16": 511214080.0,
+ "17": 511214080.0,
+ "18": 511214080.0,
+ "19": 511214080.0,
+ "20": 511214080.0,
+ "21": 511214080.0,
+ "22": 511214080.0,
+ "23": 511214080.0,
+ "24": 511214080.0,
+ "25": 511214080.0,
+ "26": 511214080.0,
+ "27": 511214080.0,
+ "28": 511214080.0,
+ "29": 511214080.0,
+ "30": 511214080.0,
+ "31": 511214080.0,
+ "32": 511214080.0,
+ "33": 511214080.0,
+ "34": 511214080.0,
+ "35": 511214080.0,
+ "36": 511214080.0,
+ "37": 511214080.0,
+ "38": 511214080.0,
+ "39": 511214080.0,
+ "40": 511214080.0,
+ "41": 511214080.0,
+ "42": 511214080.0,
+ "43": 511214080.0,
+ "44": 511214080.0,
+ "45": 511214080.0,
+ "46": 511214080.0,
+ "47": 511214080.0,
+ "48": 511214080.0,
+ "49": 511214080.0,
+ "50": 511214080.0
}
},
"mem-max-allocated-bytes": {
@@ -175,56 +175,56 @@
"end_step": 50,
"step_interval": 1,
"values": {
- "1": 759898624.0,
- "2": 933156352.0,
- "3": 933156352.0,
- "4": 933156352.0,
- "5": 933156352.0,
- "6": 933156352.0,
- "7": 933156352.0,
- "8": 933156352.0,
- "9": 933156352.0,
- "10": 933156352.0,
- "11": 933156352.0,
- "12": 933156352.0,
- "13": 933156352.0,
- "14": 933156352.0,
- "15": 933156352.0,
- "16": 933156352.0,
- "17": 933156352.0,
- "18": 933156352.0,
- "19": 933156352.0,
- "20": 933156352.0,
- "21": 933156352.0,
- "22": 933156352.0,
- "23": 933156352.0,
- "24": 933156352.0,
- "25": 933156352.0,
- "26": 933156352.0,
- "27": 933156352.0,
- "28": 933156352.0,
- "29": 933156352.0,
- "30": 933156352.0,
- "31": 933156352.0,
- "32": 933156352.0,
- "33": 933156352.0,
- "34": 933156352.0,
- "35": 933156352.0,
- "36": 933156352.0,
- "37": 933156352.0,
- "38": 933156352.0,
- "39": 933156352.0,
- "40": 933156352.0,
- "41": 933156352.0,
- "42": 933156352.0,
- "43": 933156352.0,
- "44": 933156352.0,
- "45": 933156352.0,
- "46": 933156352.0,
- "47": 933156352.0,
- "48": 933156352.0,
- "49": 933156352.0,
- "50": 933156352.0
+ "1": 759899136.0,
+ "2": 936824320.0,
+ "3": 936824832.0,
+ "4": 936824832.0,
+ "5": 936824832.0,
+ "6": 936824832.0,
+ "7": 936824832.0,
+ "8": 936824832.0,
+ "9": 936824832.0,
+ "10": 936824832.0,
+ "11": 936824832.0,
+ "12": 936824832.0,
+ "13": 936824832.0,
+ "14": 936824832.0,
+ "15": 936824832.0,
+ "16": 936824832.0,
+ "17": 936824832.0,
+ "18": 936824832.0,
+ "19": 936824832.0,
+ "20": 936824832.0,
+ "21": 936824832.0,
+ "22": 936824832.0,
+ "23": 936824832.0,
+ "24": 936824832.0,
+ "25": 936824832.0,
+ "26": 936824832.0,
+ "27": 936824832.0,
+ "28": 936824832.0,
+ "29": 936824832.0,
+ "30": 936824832.0,
+ "31": 936824832.0,
+ "32": 936824832.0,
+ "33": 936824832.0,
+ "34": 936824832.0,
+ "35": 936824832.0,
+ "36": 936824832.0,
+ "37": 936824832.0,
+ "38": 936824832.0,
+ "39": 936824832.0,
+ "40": 936824832.0,
+ "41": 936824832.0,
+ "42": 936824832.0,
+ "43": 936824832.0,
+ "44": 936824832.0,
+ "45": 936824832.0,
+ "46": 936824832.0,
+ "47": 936824832.0,
+ "48": 936824832.0,
+ "49": 936824832.0,
+ "50": 936824832.0
}
},
"iteration-time": {
@@ -232,56 +232,56 @@
"end_step": 50,
"step_interval": 1,
"values": {
- "1": 18.71096,
- "2": 0.39649,
- "3": 0.33228,
- "4": 0.33042,
- "5": 0.33036,
- "6": 0.3326,
- "7": 0.33962,
- "8": 0.37041,
- "9": 0.33077,
- "10": 0.33179,
- "11": 0.33053,
- "12": 0.33332,
- "13": 0.33149,
- "14": 0.32928,
- "15": 0.33252,
- "16": 0.3321,
- "17": 0.32661,
- "18": 0.32933,
- "19": 0.32718,
- "20": 0.32982,
- "21": 0.32827,
- "22": 0.3313,
- "23": 0.32836,
- "24": 0.3287,
- "25": 0.33025,
- "26": 0.32605,
- "27": 0.33501,
- "28": 0.32889,
- "29": 0.32971,
- "30": 0.3318,
- "31": 0.33458,
- "32": 0.33222,
- "33": 0.33434,
- "34": 0.3337,
- "35": 0.33221,
- "36": 0.32984,
- "37": 0.32779,
- "38": 0.33131,
- "39": 0.33056,
- "40": 0.32941,
- "41": 0.32351,
- "42": 0.32946,
- "43": 0.32913,
- "44": 0.3283,
- "45": 0.32845,
- "46": 0.32474,
- "47": 0.33097,
- "48": 0.32791,
- "49": 0.33143,
- "50": 0.33005
+ "1": 45.68343,
+ "2": 0.392,
+ "3": 0.35818,
+ "4": 0.28793,
+ "5": 0.28609,
+ "6": 0.28869,
+ "7": 0.28726,
+ "8": 0.28725,
+ "9": 0.28787,
+ "10": 0.2834,
+ "11": 0.28813,
+ "12": 0.28685,
+ "13": 0.28453,
+ "14": 0.28421,
+ "15": 0.28504,
+ "16": 0.28118,
+ "17": 0.28123,
+ "18": 0.28302,
+ "19": 0.28937,
+ "20": 0.28486,
+ "21": 0.28762,
+ "22": 0.28121,
+ "23": 0.28289,
+ "24": 0.28379,
+ "25": 0.28305,
+ "26": 0.28337,
+ "27": 0.28236,
+ "28": 0.28063,
+ "29": 0.27814,
+ "30": 0.2808,
+ "31": 0.27908,
+ "32": 0.28085,
+ "33": 0.28065,
+ "34": 0.28226,
+ "35": 0.28009,
+ "36": 0.2802,
+ "37": 0.28283,
+ "38": 0.27963,
+ "39": 0.28465,
+ "40": 0.28297,
+ "41": 0.28176,
+ "42": 0.28166,
+ "43": 0.2805,
+ "44": 0.28385,
+ "45": 0.28053,
+ "46": 0.27883,
+ "47": 0.28037,
+ "48": 0.28067,
+ "49": 0.27929,
+ "50": 0.27864
}
}
}
\ No newline at end of file
diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_cp2_etp4_dp_last/golden_values_dev_dgx_h100.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_cp2_etp4_dp_last/golden_values_dev_dgx_h100.json
index cd45ff021d9..e8b9cea88e0 100644
--- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_cp2_etp4_dp_last/golden_values_dev_dgx_h100.json
+++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_cp2_etp4_dp_last/golden_values_dev_dgx_h100.json
@@ -4,55 +4,55 @@
"end_step": 50,
"step_interval": 1,
"values": {
- "1": 10.86535,
- "2": 10.85873,
- "3": 10.86284,
- "4": 10.84009,
+ "1": 10.86539,
+ "2": 10.85871,
+ "3": 10.86282,
+ "4": 10.84007,
"5": 10.87856,
- "6": 10.88856,
- "7": 10.86532,
- "8": 10.86017,
- "9": 10.8599,
- "10": 10.82981,
- "11": 10.8895,
- "12": 10.8751,
- "13": 10.87423,
+ "6": 10.88852,
+ "7": 10.86536,
+ "8": 10.86015,
+ "9": 10.85991,
+ "10": 10.82982,
+ "11": 10.88947,
+ "12": 10.87511,
+ "13": 10.87422,
"14": 10.89675,
- "15": 10.82054,
- "16": 10.82504,
+ "15": 10.82056,
+ "16": 10.82497,
"17": 10.78983,
"18": 10.81029,
- "19": 10.80535,
- "20": 10.70398,
- "21": 10.66993,
- "22": 10.50643,
- "23": 10.69004,
- "24": 10.56314,
- "25": 10.4942,
- "26": 10.56628,
- "27": 10.58025,
+ "19": 10.80528,
+ "20": 10.70396,
+ "21": 10.6699,
+ "22": 10.50641,
+ "23": 10.69006,
+ "24": 10.56312,
+ "25": 10.49418,
+ "26": 10.56627,
+ "27": 10.58023,
"28": 10.51571,
- "29": 10.55299,
- "30": 10.30549,
- "31": 10.02245,
- "32": 10.40614,
+ "29": 10.55296,
+ "30": 10.30551,
+ "31": 10.02244,
+ "32": 10.40618,
"33": 10.39874,
- "34": 10.13771,
+ "34": 10.1377,
"35": 10.20184,
- "36": 10.16052,
- "37": 10.28973,
- "38": 10.11474,
+ "36": 10.1605,
+ "37": 10.28975,
+ "38": 10.11483,
"39": 10.361,
- "40": 10.01903,
+ "40": 10.01905,
"41": 10.07292,
- "42": 10.14698,
- "43": 9.74687,
- "44": 9.87766,
- "45": 9.74966,
- "46": 9.73383,
- "47": 10.07535,
- "48": 9.78068,
- "49": 9.44784,
+ "42": 10.14697,
+ "43": 9.74684,
+ "44": 9.87763,
+ "45": 9.74962,
+ "46": 9.73382,
+ "47": 10.07536,
+ "48": 9.78071,
+ "49": 9.44783,
"50": 9.8399
}
},
@@ -61,56 +61,56 @@
"end_step": 50,
"step_interval": 1,
"values": {
- "1": 653.0,
- "2": 642.0,
- "3": 630.0,
- "4": 585.0,
- "5": 635.0,
- "6": 687.0,
- "7": 615.0,
- "8": 601.0,
- "9": 607.0,
- "10": 522.0,
- "11": 637.0,
- "12": 675.0,
- "13": 649.0,
- "14": 648.0,
- "15": 640.0,
- "16": 602.0,
- "17": 668.0,
- "18": 634.0,
- "19": 593.0,
- "20": 579.0,
- "21": 633.0,
- "22": 597.0,
- "23": 756.0,
- "24": 612.0,
- "25": 591.0,
- "26": 620.0,
- "27": 700.0,
- "28": 705.0,
- "29": 795.0,
- "30": 752.0,
- "31": 628.0,
- "32": 712.0,
- "33": 752.0,
- "34": 737.0,
- "35": 741.0,
- "36": 770.0,
- "37": 861.0,
- "38": 823.0,
- "39": 812.0,
- "40": 814.0,
- "41": 826.0,
- "42": 801.0,
- "43": 769.0,
- "44": 822.0,
- "45": 777.0,
- "46": 828.0,
- "47": 878.0,
- "48": 915.0,
- "49": 908.0,
- "50": 848.0
+ "1": 572.0,
+ "2": 656.0,
+ "3": 649.0,
+ "4": 631.0,
+ "5": 658.0,
+ "6": 636.0,
+ "7": 636.0,
+ "8": 542.0,
+ "9": 653.0,
+ "10": 551.0,
+ "11": 681.0,
+ "12": 642.0,
+ "13": 624.0,
+ "14": 658.0,
+ "15": 682.0,
+ "16": 659.0,
+ "17": 620.0,
+ "18": 603.0,
+ "19": 634.0,
+ "20": 639.0,
+ "21": 634.0,
+ "22": 602.0,
+ "23": 731.0,
+ "24": 620.0,
+ "25": 611.0,
+ "26": 626.0,
+ "27": 683.0,
+ "28": 668.0,
+ "29": 713.0,
+ "30": 712.0,
+ "31": 616.0,
+ "32": 786.0,
+ "33": 800.0,
+ "34": 702.0,
+ "35": 684.0,
+ "36": 664.0,
+ "37": 831.0,
+ "38": 802.0,
+ "39": 919.0,
+ "40": 802.0,
+ "41": 791.0,
+ "42": 840.0,
+ "43": 718.0,
+ "44": 756.0,
+ "45": 765.0,
+ "46": 809.0,
+ "47": 839.0,
+ "48": 827.0,
+ "49": 935.0,
+ "50": 839.0
}
},
"mem-allocated-bytes": {
@@ -118,56 +118,56 @@
"end_step": 50,
"step_interval": 1,
"values": {
- "1": 510689792.0,
- "2": 510689792.0,
- "3": 510689792.0,
- "4": 510689792.0,
- "5": 510689792.0,
- "6": 510689792.0,
- "7": 510689792.0,
- "8": 510689792.0,
- "9": 510689792.0,
- "10": 510689792.0,
- "11": 510689792.0,
- "12": 510689792.0,
- "13": 510689792.0,
- "14": 510689792.0,
- "15": 510689792.0,
- "16": 510689792.0,
- "17": 510689792.0,
- "18": 510689792.0,
- "19": 510689792.0,
- "20": 510689792.0,
- "21": 510689792.0,
- "22": 510689792.0,
- "23": 510689792.0,
- "24": 510689792.0,
- "25": 510689792.0,
- "26": 510689792.0,
- "27": 510689792.0,
- "28": 510689792.0,
- "29": 510689792.0,
- "30": 510689792.0,
- "31": 510689792.0,
- "32": 510689792.0,
- "33": 510689792.0,
- "34": 510689792.0,
- "35": 510689792.0,
- "36": 510689792.0,
- "37": 510689792.0,
- "38": 510689792.0,
- "39": 510689792.0,
- "40": 510689792.0,
- "41": 510689792.0,
- "42": 510689792.0,
- "43": 510689792.0,
- "44": 510689792.0,
- "45": 510689792.0,
- "46": 510689792.0,
- "47": 510689792.0,
- "48": 510689792.0,
- "49": 510689792.0,
- "50": 510689792.0
+ "1": 511214080.0,
+ "2": 511214080.0,
+ "3": 511214080.0,
+ "4": 511214080.0,
+ "5": 511214080.0,
+ "6": 511214080.0,
+ "7": 511214080.0,
+ "8": 511214080.0,
+ "9": 511214080.0,
+ "10": 511214080.0,
+ "11": 511214080.0,
+ "12": 511214080.0,
+ "13": 511214080.0,
+ "14": 511214080.0,
+ "15": 511214080.0,
+ "16": 511214080.0,
+ "17": 511214080.0,
+ "18": 511214080.0,
+ "19": 511214080.0,
+ "20": 511214080.0,
+ "21": 511214080.0,
+ "22": 511214080.0,
+ "23": 511214080.0,
+ "24": 511214080.0,
+ "25": 511214080.0,
+ "26": 511214080.0,
+ "27": 511214080.0,
+ "28": 511214080.0,
+ "29": 511214080.0,
+ "30": 511214080.0,
+ "31": 511214080.0,
+ "32": 511214080.0,
+ "33": 511214080.0,
+ "34": 511214080.0,
+ "35": 511214080.0,
+ "36": 511214080.0,
+ "37": 511214080.0,
+ "38": 511214080.0,
+ "39": 511214080.0,
+ "40": 511214080.0,
+ "41": 511214080.0,
+ "42": 511214080.0,
+ "43": 511214080.0,
+ "44": 511214080.0,
+ "45": 511214080.0,
+ "46": 511214080.0,
+ "47": 511214080.0,
+ "48": 511214080.0,
+ "49": 511214080.0,
+ "50": 511214080.0
}
},
"mem-max-allocated-bytes": {
@@ -175,56 +175,56 @@
"end_step": 50,
"step_interval": 1,
"values": {
- "1": 759895552.0,
- "2": 933156352.0,
- "3": 933156352.0,
- "4": 933156352.0,
- "5": 933156352.0,
- "6": 933156352.0,
- "7": 933156352.0,
- "8": 933156352.0,
- "9": 933156352.0,
- "10": 933156352.0,
- "11": 933156352.0,
- "12": 933156352.0,
- "13": 933156352.0,
- "14": 933156352.0,
- "15": 933156352.0,
- "16": 933156352.0,
- "17": 933156352.0,
- "18": 933156352.0,
- "19": 933156352.0,
- "20": 933156352.0,
- "21": 933156352.0,
- "22": 933156352.0,
- "23": 933156352.0,
- "24": 933156352.0,
- "25": 933156352.0,
- "26": 933156352.0,
- "27": 933156352.0,
- "28": 933156352.0,
- "29": 933156352.0,
- "30": 933156352.0,
- "31": 933156352.0,
- "32": 933156352.0,
- "33": 934201856.0,
- "34": 934201856.0,
- "35": 934201856.0,
- "36": 934201856.0,
- "37": 934201856.0,
- "38": 934201856.0,
- "39": 934201856.0,
- "40": 934201856.0,
- "41": 934201856.0,
- "42": 934201856.0,
- "43": 934201856.0,
- "44": 934201856.0,
- "45": 934201856.0,
- "46": 934201856.0,
- "47": 934201856.0,
- "48": 934201856.0,
- "49": 934201856.0,
- "50": 934201856.0
+ "1": 757801984.0,
+ "2": 935777792.0,
+ "3": 935777792.0,
+ "4": 935777792.0,
+ "5": 935777792.0,
+ "6": 935777792.0,
+ "7": 935777792.0,
+ "8": 935777792.0,
+ "9": 935777792.0,
+ "10": 935777792.0,
+ "11": 935777792.0,
+ "12": 935777792.0,
+ "13": 935777792.0,
+ "14": 935777792.0,
+ "15": 935777792.0,
+ "16": 935777792.0,
+ "17": 935777792.0,
+ "18": 935777792.0,
+ "19": 935777792.0,
+ "20": 935777792.0,
+ "21": 935777792.0,
+ "22": 935777792.0,
+ "23": 935777792.0,
+ "24": 935777792.0,
+ "25": 935777792.0,
+ "26": 935777792.0,
+ "27": 935777792.0,
+ "28": 935777792.0,
+ "29": 935777792.0,
+ "30": 935777792.0,
+ "31": 935777792.0,
+ "32": 935777792.0,
+ "33": 935777792.0,
+ "34": 935777792.0,
+ "35": 935777792.0,
+ "36": 935777792.0,
+ "37": 935777792.0,
+ "38": 935777792.0,
+ "39": 935777792.0,
+ "40": 935777792.0,
+ "41": 935777792.0,
+ "42": 935777792.0,
+ "43": 935777792.0,
+ "44": 935777792.0,
+ "45": 935777792.0,
+ "46": 935777792.0,
+ "47": 935777792.0,
+ "48": 935777792.0,
+ "49": 935777792.0,
+ "50": 935777792.0
}
},
"iteration-time": {
@@ -232,56 +232,56 @@
"end_step": 50,
"step_interval": 1,
"values": {
- "1": 17.72917,
- "2": 0.36269,
- "3": 0.33585,
- "4": 0.33878,
- "5": 0.33758,
- "6": 0.33453,
- "7": 0.33628,
- "8": 0.33416,
- "9": 0.33309,
- "10": 0.33521,
- "11": 0.33536,
- "12": 0.33148,
- "13": 0.33565,
- "14": 0.33401,
- "15": 0.33029,
- "16": 0.33788,
- "17": 0.33302,
- "18": 0.33337,
- "19": 0.33761,
- "20": 0.33672,
- "21": 0.33256,
- "22": 0.3374,
- "23": 0.33652,
- "24": 0.33672,
- "25": 0.33982,
- "26": 0.3335,
- "27": 0.3328,
- "28": 0.33835,
- "29": 0.33338,
- "30": 0.33371,
- "31": 0.33991,
- "32": 0.33259,
- "33": 0.33537,
- "34": 0.33777,
- "35": 0.33494,
- "36": 0.33504,
- "37": 0.33915,
- "38": 0.33462,
- "39": 0.33387,
- "40": 0.33791,
- "41": 0.33426,
- "42": 0.33834,
- "43": 0.33785,
- "44": 0.32761,
- "45": 0.32857,
- "46": 0.33205,
- "47": 0.3355,
- "48": 0.33535,
- "49": 0.33792,
- "50": 0.33613
+ "1": 44.86787,
+ "2": 0.36349,
+ "3": 0.3142,
+ "4": 0.29456,
+ "5": 0.29609,
+ "6": 0.29566,
+ "7": 0.29467,
+ "8": 0.2899,
+ "9": 0.28864,
+ "10": 0.28994,
+ "11": 0.28355,
+ "12": 0.28608,
+ "13": 0.28278,
+ "14": 0.2823,
+ "15": 0.28087,
+ "16": 0.28237,
+ "17": 0.28556,
+ "18": 0.28363,
+ "19": 0.28381,
+ "20": 0.28356,
+ "21": 0.28235,
+ "22": 0.29036,
+ "23": 0.28491,
+ "24": 0.28322,
+ "25": 0.28412,
+ "26": 0.28352,
+ "27": 0.28643,
+ "28": 0.2853,
+ "29": 0.28809,
+ "30": 0.28258,
+ "31": 0.28114,
+ "32": 0.281,
+ "33": 0.28135,
+ "34": 0.27914,
+ "35": 0.28099,
+ "36": 0.28267,
+ "37": 0.28236,
+ "38": 0.28102,
+ "39": 0.31493,
+ "40": 0.28173,
+ "41": 0.28058,
+ "42": 0.28033,
+ "43": 0.28335,
+ "44": 0.28253,
+ "45": 0.28169,
+ "46": 0.28078,
+ "47": 0.28082,
+ "48": 0.2819,
+ "49": 0.28087,
+ "50": 0.28
}
}
}
\ No newline at end of file
diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp4_pp2_resume_torch_dist_reshard_8x1xNone/golden_values_lts_dgx_a100.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp4_pp2_resume_torch_dist_reshard_8x1xNone/golden_values_lts_dgx_a100.json
index cac9c570ec1..f273ff540d3 100644
--- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp4_pp2_resume_torch_dist_reshard_8x1xNone/golden_values_lts_dgx_a100.json
+++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp4_pp2_resume_torch_dist_reshard_8x1xNone/golden_values_lts_dgx_a100.json
@@ -4,106 +4,106 @@
"end_step": 100,
"step_interval": 1,
"values": {
- "1": 10.8583,
- "2": 10.85411,
- "3": 10.8543,
- "4": 10.84407,
- "5": 10.87282,
- "6": 10.8793,
- "7": 10.84658,
- "8": 10.86139,
- "9": 10.87078,
- "10": 10.83266,
- "11": 10.86332,
- "12": 10.87295,
- "13": 10.87798,
- "14": 10.88588,
- "15": 10.82104,
- "16": 10.82759,
- "17": 10.80303,
- "18": 10.82092,
- "19": 10.80032,
- "20": 10.71379,
- "21": 10.69818,
- "22": 10.57542,
- "23": 10.72119,
- "24": 10.60091,
- "25": 10.5476,
- "26": 10.61127,
- "27": 10.61393,
- "28": 10.57777,
- "29": 10.57888,
- "30": 10.36791,
- "31": 10.13451,
- "32": 10.47063,
- "33": 10.47371,
- "34": 10.23442,
- "35": 10.28457,
- "36": 10.23595,
- "37": 10.35351,
- "38": 10.20695,
- "39": 10.40581,
- "40": 10.08924,
- "41": 10.16388,
- "42": 10.22671,
- "43": 9.86336,
- "44": 9.98189,
- "45": 9.84555,
- "46": 9.85753,
- "47": 10.16884,
- "48": 9.86474,
- "49": 9.54712,
- "50": 9.91942,
- "51": 9.86179,
- "52": 9.76162,
- "53": 10.08383,
- "54": 9.96743,
- "55": 9.89199,
- "56": 9.63777,
- "57": 9.49339,
- "58": 9.83897,
- "59": 9.59641,
- "60": 9.50823,
- "61": 9.70513,
- "62": 9.99499,
- "63": 9.38054,
- "64": 9.78296,
- "65": 8.95946,
- "66": 9.71045,
- "67": 9.38075,
- "68": 9.78884,
- "69": 9.79451,
- "70": 9.73441,
- "71": 9.62146,
- "72": 9.58792,
- "73": 9.49657,
- "74": 8.9434,
- "75": 9.43112,
- "76": 9.09716,
- "77": 10.0681,
- "78": 9.73005,
- "79": 9.37764,
- "80": 9.41097,
- "81": 9.48622,
- "82": 9.69669,
- "83": 9.3163,
- "84": 9.42182,
- "85": 9.61516,
- "86": 9.07553,
- "87": 9.59851,
- "88": 9.75046,
- "89": 9.61112,
- "90": 9.82373,
- "91": 9.35278,
- "92": 9.36495,
- "93": 9.08811,
- "94": 8.83656,
- "95": 9.52256,
- "96": 9.52793,
- "97": 9.31634,
- "98": 9.67876,
- "99": 8.89321,
- "100": 9.40801
+ "1": 10.85936,
+ "2": 10.8548,
+ "3": 10.85199,
+ "4": 10.84317,
+ "5": 10.87247,
+ "6": 10.87857,
+ "7": 10.84622,
+ "8": 10.86369,
+ "9": 10.87211,
+ "10": 10.8311,
+ "11": 10.86068,
+ "12": 10.87273,
+ "13": 10.87992,
+ "14": 10.88657,
+ "15": 10.82029,
+ "16": 10.82684,
+ "17": 10.7998,
+ "18": 10.81985,
+ "19": 10.80035,
+ "20": 10.71399,
+ "21": 10.69893,
+ "22": 10.57449,
+ "23": 10.71973,
+ "24": 10.60285,
+ "25": 10.54611,
+ "26": 10.61041,
+ "27": 10.61227,
+ "28": 10.57731,
+ "29": 10.58005,
+ "30": 10.36705,
+ "31": 10.13447,
+ "32": 10.47127,
+ "33": 10.47454,
+ "34": 10.23198,
+ "35": 10.28443,
+ "36": 10.23436,
+ "37": 10.35346,
+ "38": 10.20696,
+ "39": 10.40599,
+ "40": 10.08972,
+ "41": 10.16331,
+ "42": 10.2256,
+ "43": 9.8639,
+ "44": 9.98246,
+ "45": 9.84548,
+ "46": 9.8581,
+ "47": 10.1689,
+ "48": 9.86658,
+ "49": 9.54555,
+ "50": 9.91937,
+ "51": 9.86074,
+ "52": 9.76116,
+ "53": 10.08415,
+ "54": 9.96563,
+ "55": 9.89123,
+ "56": 9.63923,
+ "57": 9.4936,
+ "58": 9.83871,
+ "59": 9.59623,
+ "60": 9.5091,
+ "61": 9.70544,
+ "62": 9.99513,
+ "63": 9.38104,
+ "64": 9.78222,
+ "65": 8.95962,
+ "66": 9.71006,
+ "67": 9.38013,
+ "68": 9.78827,
+ "69": 9.79425,
+ "70": 9.73517,
+ "71": 9.62218,
+ "72": 9.58801,
+ "73": 9.49714,
+ "74": 8.94242,
+ "75": 9.4322,
+ "76": 9.09757,
+ "77": 10.06853,
+ "78": 9.73055,
+ "79": 9.37759,
+ "80": 9.41116,
+ "81": 9.48631,
+ "82": 9.69758,
+ "83": 9.31674,
+ "84": 9.42151,
+ "85": 9.61502,
+ "86": 9.07627,
+ "87": 9.59887,
+ "88": 9.75047,
+ "89": 9.61233,
+ "90": 9.82363,
+ "91": 9.35377,
+ "92": 9.36525,
+ "93": 9.08833,
+ "94": 8.83614,
+ "95": 9.5226,
+ "96": 9.52736,
+ "97": 9.3169,
+ "98": 9.67961,
+ "99": 8.89276,
+ "100": 9.40803
}
},
"num-zeros": {
@@ -111,106 +111,106 @@
"end_step": 100,
"step_interval": 1,
"values": {
- "1": 1708.0,
- "2": 1804.0,
- "3": 1725.0,
- "4": 1881.0,
- "5": 2019.0,
- "6": 2015.0,
- "7": 2086.0,
- "8": 1730.0,
- "9": 2024.0,
- "10": 1515.0,
- "11": 2162.0,
- "12": 1847.0,
- "13": 2125.0,
- "14": 2050.0,
- "15": 1946.0,
- "16": 2000.0,
- "17": 1996.0,
- "18": 1874.0,
- "19": 2011.0,
- "20": 1771.0,
- "21": 2099.0,
- "22": 1892.0,
- "23": 2171.0,
- "24": 1834.0,
- "25": 1790.0,
- "26": 1803.0,
- "27": 1998.0,
- "28": 2211.0,
- "29": 2129.0,
- "30": 2147.0,
- "31": 1623.0,
- "32": 2174.0,
- "33": 2364.0,
- "34": 2035.0,
- "35": 2089.0,
- "36": 2202.0,
- "37": 2603.0,
- "38": 2468.0,
- "39": 2623.0,
- "40": 2383.0,
- "41": 2519.0,
- "42": 2522.0,
- "43": 2235.0,
- "44": 2275.0,
- "45": 2319.0,
- "46": 2632.0,
- "47": 2675.0,
- "48": 2697.0,
- "49": 2551.0,
- "50": 2814.0,
- "51": 2767.0,
- "52": 2804.0,
- "53": 3231.0,
- "54": 2905.0,
- "55": 2575.0,
- "56": 3077.0,
- "57": 2587.0,
- "58": 3346.0,
- "59": 3056.0,
- "60": 2695.0,
- "61": 3191.0,
- "62": 2637.0,
- "63": 2649.0,
- "64": 3176.0,
- "65": 2756.0,
- "66": 3481.0,
- "67": 2905.0,
- "68": 3114.0,
- "69": 3133.0,
- "70": 3533.0,
- "71": 3225.0,
- "72": 2621.0,
- "73": 3297.0,
- "74": 2145.0,
- "75": 2799.0,
- "76": 3354.0,
- "77": 3466.0,
- "78": 3485.0,
- "79": 3464.0,
- "80": 3614.0,
- "81": 4011.0,
- "82": 3694.0,
- "83": 3201.0,
- "84": 3655.0,
- "85": 3597.0,
- "86": 3096.0,
- "87": 4103.0,
- "88": 3306.0,
- "89": 3839.0,
- "90": 3352.0,
- "91": 2980.0,
- "92": 3452.0,
- "93": 2967.0,
- "94": 3773.0,
- "95": 3589.0,
- "96": 3800.0,
- "97": 3412.0,
- "98": 3998.0,
- "99": 3483.0,
- "100": 3651.0
+ "1": 1768.0,
+ "2": 1871.0,
+ "3": 1757.0,
+ "4": 1902.0,
+ "5": 2016.0,
+ "6": 1943.0,
+ "7": 1981.0,
+ "8": 1667.0,
+ "9": 1973.0,
+ "10": 1477.0,
+ "11": 2178.0,
+ "12": 1985.0,
+ "13": 2137.0,
+ "14": 2021.0,
+ "15": 1944.0,
+ "16": 2053.0,
+ "17": 1967.0,
+ "18": 1922.0,
+ "19": 2031.0,
+ "20": 1837.0,
+ "21": 2048.0,
+ "22": 1917.0,
+ "23": 2190.0,
+ "24": 1787.0,
+ "25": 1869.0,
+ "26": 1882.0,
+ "27": 2143.0,
+ "28": 2147.0,
+ "29": 2222.0,
+ "30": 2046.0,
+ "31": 1734.0,
+ "32": 2171.0,
+ "33": 2380.0,
+ "34": 2046.0,
+ "35": 2147.0,
+ "36": 2149.0,
+ "37": 2645.0,
+ "38": 2416.0,
+ "39": 2672.0,
+ "40": 2441.0,
+ "41": 2585.0,
+ "42": 2483.0,
+ "43": 2262.0,
+ "44": 2344.0,
+ "45": 2300.0,
+ "46": 2560.0,
+ "47": 2755.0,
+ "48": 2764.0,
+ "49": 2505.0,
+ "50": 2723.0,
+ "51": 2806.0,
+ "52": 2805.0,
+ "53": 3225.0,
+ "54": 3028.0,
+ "55": 2486.0,
+ "56": 3093.0,
+ "57": 2588.0,
+ "58": 3219.0,
+ "59": 3021.0,
+ "60": 2649.0,
+ "61": 3247.0,
+ "62": 2649.0,
+ "63": 2637.0,
+ "64": 3140.0,
+ "65": 3038.0,
+ "66": 3422.0,
+ "67": 2933.0,
+ "68": 3039.0,
+ "69": 3167.0,
+ "70": 3539.0,
+ "71": 3213.0,
+ "72": 2597.0,
+ "73": 3290.0,
+ "74": 2140.0,
+ "75": 2837.0,
+ "76": 3342.0,
+ "77": 3444.0,
+ "78": 3504.0,
+ "79": 3513.0,
+ "80": 3733.0,
+ "81": 4024.0,
+ "82": 3670.0,
+ "83": 3199.0,
+ "84": 3539.0,
+ "85": 3585.0,
+ "86": 2979.0,
+ "87": 3951.0,
+ "88": 3286.0,
+ "89": 3787.0,
+ "90": 3341.0,
+ "91": 3070.0,
+ "92": 3410.0,
+ "93": 2923.0,
+ "94": 3868.0,
+ "95": 3627.0,
+ "96": 3787.0,
+ "97": 3549.0,
+ "98": 4026.0,
+ "99": 3531.0,
+ "100": 3649.0
}
},
"mem-allocated-bytes": {
@@ -218,106 +218,106 @@
"end_step": 100,
"step_interval": 1,
"values": {
- "1": 232422400.0,
- "2": 232422400.0,
- "3": 232422400.0,
- "4": 232422400.0,
- "5": 232422400.0,
- "6": 233470976.0,
- "7": 232422400.0,
- "8": 233470976.0,
- "9": 232422400.0,
- "10": 232422400.0,
- "11": 232422400.0,
- "12": 232422400.0,
- "13": 232422400.0,
- "14": 233470976.0,
- "15": 232422400.0,
- "16": 232422400.0,
- "17": 232422400.0,
- "18": 232422400.0,
- "19": 232422400.0,
- "20": 232422400.0,
- "21": 232422400.0,
- "22": 232422400.0,
- "23": 232422400.0,
- "24": 232422400.0,
- "25": 232422400.0,
- "26": 232422400.0,
- "27": 232422400.0,
- "28": 232422400.0,
- "29": 232422400.0,
- "30": 232422400.0,
- "31": 232422400.0,
- "32": 232422400.0,
- "33": 232422400.0,
- "34": 232422400.0,
- "35": 232422400.0,
- "36": 232422400.0,
- "37": 232422400.0,
- "38": 232422400.0,
- "39": 232422400.0,
- "40": 232422400.0,
- "41": 232422400.0,
- "42": 232422400.0,
- "43": 232422400.0,
- "44": 232422400.0,
- "45": 232422400.0,
- "46": 232422400.0,
- "47": 232422400.0,
- "48": 232422400.0,
- "49": 233470976.0,
- "50": 232422400.0,
- "51": 232422400.0,
- "52": 232422400.0,
- "53": 232422400.0,
- "54": 232422400.0,
- "55": 233470976.0,
- "56": 232422400.0,
- "57": 233470976.0,
- "58": 232422400.0,
- "59": 232422400.0,
- "60": 232422400.0,
- "61": 232422400.0,
- "62": 232422400.0,
- "63": 232422400.0,
- "64": 232422400.0,
- "65": 232422400.0,
- "66": 232422400.0,
- "67": 232422400.0,
- "68": 232422400.0,
- "69": 232422400.0,
- "70": 232422400.0,
- "71": 232422400.0,
- "72": 232422400.0,
- "73": 232422400.0,
- "74": 232422400.0,
- "75": 232422400.0,
- "76": 232422400.0,
- "77": 232422400.0,
- "78": 232422400.0,
- "79": 232422400.0,
- "80": 232422400.0,
- "81": 232422400.0,
- "82": 232422400.0,
- "83": 232422400.0,
- "84": 232422400.0,
- "85": 232422400.0,
- "86": 232422400.0,
- "87": 232422400.0,
- "88": 232422400.0,
- "89": 232422400.0,
- "90": 232422400.0,
- "91": 232422400.0,
- "92": 232422400.0,
- "93": 232422400.0,
- "94": 232422400.0,
- "95": 232422400.0,
- "96": 232422400.0,
- "97": 232422400.0,
- "98": 232422400.0,
- "99": 233470976.0,
- "100": 232422400.0
+ "1": 232398336.0,
+ "2": 232398336.0,
+ "3": 232398336.0,
+ "4": 232398336.0,
+ "5": 232398336.0,
+ "6": 232398336.0,
+ "7": 232398336.0,
+ "8": 232398336.0,
+ "9": 232398336.0,
+ "10": 232398336.0,
+ "11": 232398336.0,
+ "12": 232398336.0,
+ "13": 232398336.0,
+ "14": 232398336.0,
+ "15": 232398336.0,
+ "16": 232398336.0,
+ "17": 232398336.0,
+ "18": 232398336.0,
+ "19": 232398336.0,
+ "20": 232398336.0,
+ "21": 232398336.0,
+ "22": 232398336.0,
+ "23": 232398336.0,
+ "24": 232398336.0,
+ "25": 232398336.0,
+ "26": 232398336.0,
+ "27": 232398336.0,
+ "28": 232398336.0,
+ "29": 232398336.0,
+ "30": 232398336.0,
+ "31": 232398336.0,
+ "32": 232398336.0,
+ "33": 232398336.0,
+ "34": 232398336.0,
+ "35": 232398336.0,
+ "36": 232398336.0,
+ "37": 232398336.0,
+ "38": 232398336.0,
+ "39": 232398336.0,
+ "40": 232398336.0,
+ "41": 232398336.0,
+ "42": 232398336.0,
+ "43": 232398336.0,
+ "44": 232398336.0,
+ "45": 232398336.0,
+ "46": 232398336.0,
+ "47": 232398336.0,
+ "48": 232398336.0,
+ "49": 232398336.0,
+ "50": 232398336.0,
+ "51": 232398336.0,
+ "52": 232398336.0,
+ "53": 232398336.0,
+ "54": 232398336.0,
+ "55": 232398336.0,
+ "56": 232398336.0,
+ "57": 232398336.0,
+ "58": 232398336.0,
+ "59": 232398336.0,
+ "60": 232398336.0,
+ "61": 232398336.0,
+ "62": 232398336.0,
+ "63": 232398336.0,
+ "64": 232398336.0,
+ "65": 232398336.0,
+ "66": 232398336.0,
+ "67": 232398336.0,
+ "68": 232398336.0,
+ "69": 232398336.0,
+ "70": 232398336.0,
+ "71": 232398336.0,
+ "72": 232398336.0,
+ "73": 232398336.0,
+ "74": 232398336.0,
+ "75": 232398336.0,
+ "76": 232398336.0,
+ "77": 232398336.0,
+ "78": 232398336.0,
+ "79": 232398336.0,
+ "80": 232398336.0,
+ "81": 232398336.0,
+ "82": 232398336.0,
+ "83": 232398336.0,
+ "84": 232398336.0,
+ "85": 232398336.0,
+ "86": 232398336.0,
+ "87": 232398336.0,
+ "88": 232398336.0,
+ "89": 232398336.0,
+ "90": 232398336.0,
+ "91": 232398336.0,
+ "92": 232398336.0,
+ "93": 232398336.0,
+ "94": 232398336.0,
+ "95": 232398336.0,
+ "96": 232398336.0,
+ "97": 232398336.0,
+ "98": 232398336.0,
+ "99": 232398336.0,
+ "100": 232398336.0
}
},
"mem-max-allocated-bytes": {
@@ -325,106 +325,106 @@
"end_step": 100,
"step_interval": 1,
"values": {
- "1": 683423744.0,
- "2": 773273600.0,
- "3": 773276672.0,
- "4": 773276672.0,
- "5": 773276672.0,
- "6": 773276672.0,
- "7": 773276672.0,
- "8": 773276672.0,
- "9": 773276672.0,
- "10": 773276672.0,
- "11": 773276672.0,
- "12": 773276672.0,
- "13": 773276672.0,
- "14": 773276672.0,
- "15": 773276672.0,
- "16": 773276672.0,
- "17": 773276672.0,
- "18": 773276672.0,
- "19": 773276672.0,
- "20": 773276672.0,
- "21": 773276672.0,
- "22": 773276672.0,
- "23": 773276672.0,
- "24": 773276672.0,
- "25": 773276672.0,
- "26": 773276672.0,
- "27": 773276672.0,
- "28": 773276672.0,
- "29": 773276672.0,
- "30": 773276672.0,
- "31": 773276672.0,
- "32": 773276672.0,
- "33": 773276672.0,
- "34": 773276672.0,
- "35": 773276672.0,
- "36": 773276672.0,
- "37": 773276672.0,
- "38": 773276672.0,
- "39": 773276672.0,
- "40": 773276672.0,
- "41": 773276672.0,
- "42": 773276672.0,
- "43": 773276672.0,
- "44": 773276672.0,
- "45": 773276672.0,
- "46": 773276672.0,
- "47": 773276672.0,
- "48": 773276672.0,
- "49": 773276672.0,
- "50": 775372800.0,
- "51": 775372800.0,
- "52": 775372800.0,
- "53": 775372800.0,
- "54": 775372800.0,
- "55": 775372800.0,
- "56": 775372800.0,
- "57": 775372800.0,
- "58": 775372800.0,
- "59": 775372800.0,
- "60": 775372800.0,
- "61": 775372800.0,
- "62": 775372800.0,
- "63": 775372800.0,
- "64": 775372800.0,
- "65": 775372800.0,
- "66": 775372800.0,
- "67": 775372800.0,
- "68": 775372800.0,
- "69": 775372800.0,
- "70": 775372800.0,
- "71": 775372800.0,
- "72": 775372800.0,
- "73": 775372800.0,
- "74": 775372800.0,
- "75": 775372800.0,
- "76": 775372800.0,
- "77": 775372800.0,
- "78": 775372800.0,
- "79": 775372800.0,
- "80": 775372800.0,
- "81": 775372800.0,
- "82": 775372800.0,
- "83": 775372800.0,
- "84": 775372800.0,
- "85": 775372800.0,
- "86": 775372800.0,
- "87": 775372800.0,
- "88": 775372800.0,
- "89": 775372800.0,
- "90": 775372800.0,
- "91": 775372800.0,
- "92": 775372800.0,
- "93": 775372800.0,
- "94": 775372800.0,
- "95": 775372800.0,
- "96": 775372800.0,
- "97": 775372800.0,
- "98": 775372800.0,
- "99": 775373312.0,
- "100": 775373312.0
+ "1": 685490688.0,
+ "2": 773246464.0,
+ "3": 773246464.0,
+ "4": 773246464.0,
+ "5": 773246464.0,
+ "6": 773246464.0,
+ "7": 773246464.0,
+ "8": 773246464.0,
+ "9": 773246464.0,
+ "10": 773246464.0,
+ "11": 773246464.0,
+ "12": 773246464.0,
+ "13": 773246464.0,
+ "14": 773246464.0,
+ "15": 773246464.0,
+ "16": 773246464.0,
+ "17": 773246464.0,
+ "18": 773246464.0,
+ "19": 773246464.0,
+ "20": 773246464.0,
+ "21": 773246464.0,
+ "22": 773246464.0,
+ "23": 773246464.0,
+ "24": 773246464.0,
+ "25": 773246464.0,
+ "26": 773246464.0,
+ "27": 773246464.0,
+ "28": 773246464.0,
+ "29": 773246464.0,
+ "30": 773246464.0,
+ "31": 773246464.0,
+ "32": 773246464.0,
+ "33": 773246464.0,
+ "34": 773246464.0,
+ "35": 773246464.0,
+ "36": 773246464.0,
+ "37": 773246464.0,
+ "38": 773246464.0,
+ "39": 773246464.0,
+ "40": 773246464.0,
+ "41": 773246464.0,
+ "42": 773246464.0,
+ "43": 773246464.0,
+ "44": 773246464.0,
+ "45": 773246464.0,
+ "46": 773246464.0,
+ "47": 773246464.0,
+ "48": 773246464.0,
+ "49": 773246464.0,
+ "50": 773246464.0,
+ "51": 773246464.0,
+ "52": 773246464.0,
+ "53": 773246464.0,
+ "54": 773246464.0,
+ "55": 773246464.0,
+ "56": 773246464.0,
+ "57": 773246464.0,
+ "58": 773246464.0,
+ "59": 773246464.0,
+ "60": 773246464.0,
+ "61": 773246464.0,
+ "62": 773246464.0,
+ "63": 773246464.0,
+ "64": 773246464.0,
+ "65": 773246464.0,
+ "66": 773246464.0,
+ "67": 773246464.0,
+ "68": 773246464.0,
+ "69": 773246464.0,
+ "70": 773246464.0,
+ "71": 773246464.0,
+ "72": 773246464.0,
+ "73": 773246464.0,
+ "74": 773246464.0,
+ "75": 773246464.0,
+ "76": 773246464.0,
+ "77": 773246464.0,
+ "78": 773246464.0,
+ "79": 773246464.0,
+ "80": 773246464.0,
+ "81": 773246464.0,
+ "82": 773246464.0,
+ "83": 773246464.0,
+ "84": 773246464.0,
+ "85": 773246464.0,
+ "86": 773246464.0,
+ "87": 773246464.0,
+ "88": 773246464.0,
+ "89": 773246464.0,
+ "90": 773246464.0,
+ "91": 773246464.0,
+ "92": 773246464.0,
+ "93": 773246464.0,
+ "94": 773246464.0,
+ "95": 773246464.0,
+ "96": 773246464.0,
+ "97": 773246464.0,
+ "98": 773246464.0,
+ "99": 773246464.0,
+ "100": 773246464.0
}
},
"iteration-time": {
@@ -432,106 +432,106 @@
"end_step": 100,
"step_interval": 1,
"values": {
- "1": 16.23173,
- "2": 0.48632,
- "3": 0.3184,
- "4": 0.31067,
- "5": 0.31575,
- "6": 0.3127,
- "7": 0.3096,
- "8": 0.31392,
- "9": 0.31591,
- "10": 0.30891,
- "11": 0.31209,
- "12": 0.31271,
- "13": 0.30582,
- "14": 0.31032,
- "15": 0.30879,
- "16": 0.3077,
- "17": 0.30689,
- "18": 0.30824,
- "19": 0.30953,
- "20": 0.30728,
- "21": 0.31141,
- "22": 0.31157,
- "23": 0.30569,
- "24": 0.30896,
- "25": 0.30916,
- "26": 0.30674,
- "27": 0.31017,
- "28": 0.30716,
- "29": 0.30734,
- "30": 0.30698,
- "31": 0.30881,
- "32": 0.3089,
- "33": 0.30647,
- "34": 0.3112,
- "35": 0.311,
- "36": 0.30632,
- "37": 0.30856,
- "38": 0.30986,
- "39": 0.30502,
- "40": 0.31035,
- "41": 0.306,
- "42": 0.30943,
- "43": 0.30773,
- "44": 0.30886,
- "45": 0.30942,
- "46": 0.30579,
- "47": 0.31121,
- "48": 0.31407,
- "49": 0.30981,
- "50": 0.30966,
- "51": 0.3347,
- "52": 0.35543,
- "53": 0.31067,
- "54": 0.30931,
- "55": 0.31517,
- "56": 0.30883,
- "57": 0.30908,
- "58": 0.31373,
- "59": 0.30746,
- "60": 0.31113,
- "61": 0.31473,
- "62": 0.30775,
- "63": 0.31034,
- "64": 0.31108,
- "65": 0.3103,
- "66": 0.3085,
- "67": 0.31036,
- "68": 0.31412,
- "69": 0.30947,
- "70": 0.30646,
- "71": 0.31133,
- "72": 0.30734,
- "73": 0.31043,
- "74": 0.31583,
- "75": 0.3074,
- "76": 0.30939,
- "77": 0.3182,
- "78": 0.30755,
- "79": 0.30953,
- "80": 0.3085,
- "81": 0.31023,
- "82": 0.30621,
- "83": 0.30705,
- "84": 0.31232,
- "85": 0.30864,
- "86": 0.31017,
- "87": 0.3124,
- "88": 0.30667,
- "89": 0.31086,
- "90": 0.31626,
- "91": 0.30744,
- "92": 0.30887,
- "93": 0.31054,
- "94": 0.31172,
- "95": 0.31164,
- "96": 0.31058,
- "97": 0.31089,
- "98": 0.30676,
- "99": 0.3105,
- "100": 0.31337
+ "1": 9.93671,
+ "2": 0.44025,
+ "3": 0.31978,
+ "4": 0.30044,
+ "5": 0.29939,
+ "6": 0.29882,
+ "7": 0.29791,
+ "8": 0.29478,
+ "9": 0.29711,
+ "10": 0.29556,
+ "11": 0.29815,
+ "12": 0.29967,
+ "13": 0.29479,
+ "14": 0.29726,
+ "15": 0.29661,
+ "16": 0.29615,
+ "17": 0.29592,
+ "18": 0.29568,
+ "19": 0.29536,
+ "20": 0.29486,
+ "21": 0.29478,
+ "22": 0.29533,
+ "23": 0.29472,
+ "24": 0.29577,
+ "25": 0.29612,
+ "26": 0.29259,
+ "27": 0.28753,
+ "28": 0.28697,
+ "29": 0.70578,
+ "30": 0.29095,
+ "31": 0.29056,
+ "32": 0.29195,
+ "33": 0.29198,
+ "34": 0.29205,
+ "35": 0.29049,
+ "36": 0.28947,
+ "37": 0.29052,
+ "38": 0.29096,
+ "39": 0.29096,
+ "40": 0.29115,
+ "41": 0.29128,
+ "42": 0.29068,
+ "43": 0.29094,
+ "44": 0.29228,
+ "45": 0.29059,
+ "46": 0.29108,
+ "47": 0.29102,
+ "48": 0.29077,
+ "49": 0.29062,
+ "50": 0.2902,
+ "51": 0.30007,
+ "52": 0.63804,
+ "53": 0.28911,
+ "54": 0.46416,
+ "55": 0.29262,
+ "56": 0.37133,
+ "57": 0.29216,
+ "58": 0.32564,
+ "59": 0.29296,
+ "60": 0.2903,
+ "61": 0.29162,
+ "62": 0.28953,
+ "63": 0.28969,
+ "64": 0.28976,
+ "65": 0.64598,
+ "66": 0.28891,
+ "67": 0.55309,
+ "68": 0.67465,
+ "69": 0.35714,
+ "70": 0.3918,
+ "71": 0.2878,
+ "72": 0.33397,
+ "73": 0.41898,
+ "74": 0.29045,
+ "75": 0.31982,
+ "76": 0.28797,
+ "77": 0.34091,
+ "78": 0.52101,
+ "79": 0.29094,
+ "80": 0.299,
+ "81": 0.43963,
+ "82": 0.28851,
+ "83": 0.38734,
+ "84": 0.38974,
+ "85": 0.38902,
+ "86": 0.69087,
+ "87": 0.37076,
+ "88": 0.29102,
+ "89": 0.55341,
+ "90": 0.54278,
+ "91": 0.28909,
+ "92": 0.31421,
+ "93": 0.29166,
+ "94": 0.29126,
+ "95": 0.32114,
+ "96": 0.29039,
+ "97": 0.30171,
+ "98": 0.29192,
+ "99": 0.29197,
+ "100": 0.31795
}
}
}
\ No newline at end of file
diff --git a/tests/functional_tests/test_cases/gpt/gpt3_weekly_dgx_b200_mcore_tp2_pp2_current_scaling_native_fp8_tp_pp_sp_tp_overlap/model_config.yaml b/tests/functional_tests/test_cases/gpt/gpt3_weekly_dgx_b200_mcore_tp2_pp2_current_scaling_native_fp8_tp_pp_sp_tp_overlap/model_config.yaml
index f6892ae5c24..048256c3504 100644
--- a/tests/functional_tests/test_cases/gpt/gpt3_weekly_dgx_b200_mcore_tp2_pp2_current_scaling_native_fp8_tp_pp_sp_tp_overlap/model_config.yaml
+++ b/tests/functional_tests/test_cases/gpt/gpt3_weekly_dgx_b200_mcore_tp2_pp2_current_scaling_native_fp8_tp_pp_sp_tp_overlap/model_config.yaml
@@ -41,7 +41,7 @@ MODEL_ARGS:
--pipeline-model-parallel-size: 2
--sequence-parallel: true
--tp-comm-overlap: true
- --tp-comm-overlap-cfg: tests/functional_tests/test_cases/gpt/gpt3_345m_weekly_dgx_b200_1N8G_mcore_tp2_pp2_current_scaling_native_fp8_tp_pp_sp_tp_overlap/tp_comm_overlap_cfg.yaml
+ --tp-comm-overlap-cfg: tests/functional_tests/test_cases/gpt/gpt3_mcore_tp2_pp2_current_scaling_native_fp8_tp_pp_sp_tp_overlap/tp_comm_overlap_cfg.yaml
--deterministic-mode: true
--no-gradient-accumulation-fusion: true
--fp8-format: hybrid
diff --git a/tests/functional_tests/test_cases/gpt/gpt3_weekly_dgx_h100_mcore_tp2_pp2_current_scaling_native_fp8_tp_pp_sp_tp_overlap/golden_values_dev_dgx_h100.json b/tests/functional_tests/test_cases/gpt/gpt3_weekly_dgx_h100_mcore_tp2_pp2_current_scaling_native_fp8_tp_pp_sp_tp_overlap/golden_values_dev_dgx_h100.json
new file mode 100644
index 00000000000..f56b5fa6f77
--- /dev/null
+++ b/tests/functional_tests/test_cases/gpt/gpt3_weekly_dgx_h100_mcore_tp2_pp2_current_scaling_native_fp8_tp_pp_sp_tp_overlap/golden_values_dev_dgx_h100.json
@@ -0,0 +1,10037 @@
+{
+ "lm loss": {
+ "start_step": 1,
+ "end_step": 2000,
+ "step_interval": 1,
+ "values": {
+ "1": 10.85954,
+ "2": 10.88017,
+ "3": 10.87732,
+ "4": 10.8999,
+ "5": 10.88699,
+ "6": 10.87335,
+ "7": 10.88219,
+ "8": 10.87225,
+ "9": 10.87277,
+ "10": 10.87494,
+ "11": 10.85221,
+ "12": 10.84405,
+ "13": 10.84222,
+ "14": 10.86461,
+ "15": 10.78656,
+ "16": 10.81059,
+ "17": 10.77436,
+ "18": 10.81246,
+ "19": 10.72203,
+ "20": 10.69596,
+ "21": 10.64272,
+ "22": 10.64956,
+ "23": 10.65288,
+ "24": 10.54233,
+ "25": 10.55491,
+ "26": 10.63818,
+ "27": 10.44117,
+ "28": 10.46928,
+ "29": 10.34986,
+ "30": 10.24645,
+ "31": 10.42625,
+ "32": 10.33791,
+ "33": 10.19559,
+ "34": 10.14074,
+ "35": 10.22182,
+ "36": 10.13202,
+ "37": 10.07533,
+ "38": 10.01538,
+ "39": 10.02986,
+ "40": 10.05768,
+ "41": 9.93219,
+ "42": 9.93962,
+ "43": 9.8498,
+ "44": 9.97902,
+ "45": 9.99946,
+ "46": 9.83276,
+ "47": 9.99696,
+ "48": 9.80958,
+ "49": 9.94884,
+ "50": 9.94537,
+ "51": 9.58197,
+ "52": 9.79331,
+ "53": 9.62548,
+ "54": 9.88686,
+ "55": 9.73482,
+ "56": 9.84492,
+ "57": 9.85708,
+ "58": 9.87627,
+ "59": 9.54205,
+ "60": 9.64489,
+ "61": 9.88334,
+ "62": 9.75928,
+ "63": 9.68107,
+ "64": 9.82461,
+ "65": 9.59476,
+ "66": 9.62868,
+ "67": 9.74002,
+ "68": 9.60205,
+ "69": 9.29216,
+ "70": 9.42139,
+ "71": 9.78753,
+ "72": 9.7124,
+ "73": 9.61815,
+ "74": 9.44773,
+ "75": 9.23898,
+ "76": 9.50824,
+ "77": 9.5795,
+ "78": 9.56058,
+ "79": 9.30801,
+ "80": 9.35768,
+ "81": 9.45813,
+ "82": 9.55358,
+ "83": 9.53407,
+ "84": 9.35442,
+ "85": 9.3992,
+ "86": 9.65282,
+ "87": 9.23449,
+ "88": 9.48753,
+ "89": 9.22214,
+ "90": 9.41067,
+ "91": 9.38753,
+ "92": 9.37682,
+ "93": 9.36024,
+ "94": 9.51507,
+ "95": 9.42125,
+ "96": 9.33616,
+ "97": 9.20399,
+ "98": 9.4954,
+ "99": 9.29284,
+ "100": 9.35905,
+ "101": 9.24757,
+ "102": 9.24676,
+ "103": 9.07735,
+ "104": 9.16669,
+ "105": 9.37858,
+ "106": 9.1496,
+ "107": 9.1756,
+ "108": 9.316,
+ "109": 9.29109,
+ "110": 9.36426,
+ "111": 9.17995,
+ "112": 9.23471,
+ "113": 9.35297,
+ "114": 9.35265,
+ "115": 9.32672,
+ "116": 9.00223,
+ "117": 9.06476,
+ "118": 9.06643,
+ "119": 9.22418,
+ "120": 9.08485,
+ "121": 9.19671,
+ "122": 9.14164,
+ "123": 9.25933,
+ "124": 9.45506,
+ "125": 9.21512,
+ "126": 9.06416,
+ "127": 9.01814,
+ "128": 9.22131,
+ "129": 8.98184,
+ "130": 9.13972,
+ "131": 9.15856,
+ "132": 9.03559,
+ "133": 8.85977,
+ "134": 9.18539,
+ "135": 8.88999,
+ "136": 9.16801,
+ "137": 9.15771,
+ "138": 9.23511,
+ "139": 9.09197,
+ "140": 8.87218,
+ "141": 9.29906,
+ "142": 9.19961,
+ "143": 9.1169,
+ "144": 9.24305,
+ "145": 9.10446,
+ "146": 8.98709,
+ "147": 8.98617,
+ "148": 9.13261,
+ "149": 9.06335,
+ "150": 9.01504,
+ "151": 8.92787,
+ "152": 8.8739,
+ "153": 9.06335,
+ "154": 9.17913,
+ "155": 9.13381,
+ "156": 9.04889,
+ "157": 9.15064,
+ "158": 9.04955,
+ "159": 9.03261,
+ "160": 8.88987,
+ "161": 9.04543,
+ "162": 8.89584,
+ "163": 8.84272,
+ "164": 8.97534,
+ "165": 8.93132,
+ "166": 8.65959,
+ "167": 8.83243,
+ "168": 8.81953,
+ "169": 8.6566,
+ "170": 9.04622,
+ "171": 8.72286,
+ "172": 8.82159,
+ "173": 8.91163,
+ "174": 8.84751,
+ "175": 8.70611,
+ "176": 8.75439,
+ "177": 8.7626,
+ "178": 8.7201,
+ "179": 8.64046,
+ "180": 8.74053,
+ "181": 8.69404,
+ "182": 8.72193,
+ "183": 9.08364,
+ "184": 8.6088,
+ "185": 8.88346,
+ "186": 8.74191,
+ "187": 8.56949,
+ "188": 8.67975,
+ "189": 8.86478,
+ "190": 8.53542,
+ "191": 8.66632,
+ "192": 8.61266,
+ "193": 8.57469,
+ "194": 8.75195,
+ "195": 8.59279,
+ "196": 8.77393,
+ "197": 8.74234,
+ "198": 8.62722,
+ "199": 8.77454,
+ "200": 8.73803,
+ "201": 8.66979,
+ "202": 8.54593,
+ "203": 8.54185,
+ "204": 8.71307,
+ "205": 8.2228,
+ "206": 8.8603,
+ "207": 8.68157,
+ "208": 8.70896,
+ "209": 8.75303,
+ "210": 8.57807,
+ "211": 8.84258,
+ "212": 8.49127,
+ "213": 8.57327,
+ "214": 8.51199,
+ "215": 8.5645,
+ "216": 8.50863,
+ "217": 8.53183,
+ "218": 8.52998,
+ "219": 8.64367,
+ "220": 8.54746,
+ "221": 8.39991,
+ "222": 8.50528,
+ "223": 8.43775,
+ "224": 8.53014,
+ "225": 8.57091,
+ "226": 8.4394,
+ "227": 8.67918,
+ "228": 8.38473,
+ "229": 8.45045,
+ "230": 8.49717,
+ "231": 8.49832,
+ "232": 8.49783,
+ "233": 8.49539,
+ "234": 8.63795,
+ "235": 8.55875,
+ "236": 8.39461,
+ "237": 8.48826,
+ "238": 8.30522,
+ "239": 8.562,
+ "240": 8.66952,
+ "241": 8.44144,
+ "242": 8.47219,
+ "243": 8.51768,
+ "244": 8.36825,
+ "245": 8.59274,
+ "246": 8.59497,
+ "247": 8.44008,
+ "248": 8.51279,
+ "249": 8.52035,
+ "250": 8.42183,
+ "251": 8.37751,
+ "252": 8.54393,
+ "253": 8.31454,
+ "254": 8.351,
+ "255": 8.29005,
+ "256": 8.20261,
+ "257": 8.394,
+ "258": 8.45386,
+ "259": 8.23708,
+ "260": 8.2437,
+ "261": 8.23617,
+ "262": 8.34919,
+ "263": 8.30683,
+ "264": 8.18831,
+ "265": 8.33481,
+ "266": 8.23369,
+ "267": 7.89923,
+ "268": 8.38063,
+ "269": 8.40466,
+ "270": 8.26271,
+ "271": 8.279,
+ "272": 8.32109,
+ "273": 8.13747,
+ "274": 8.09677,
+ "275": 8.01372,
+ "276": 7.92611,
+ "277": 8.24041,
+ "278": 8.05017,
+ "279": 7.96688,
+ "280": 7.75652,
+ "281": 8.10713,
+ "282": 8.15049,
+ "283": 8.15621,
+ "284": 8.10354,
+ "285": 8.07234,
+ "286": 7.90454,
+ "287": 7.9963,
+ "288": 8.24862,
+ "289": 8.17575,
+ "290": 8.13093,
+ "291": 8.25763,
+ "292": 8.08131,
+ "293": 8.12059,
+ "294": 7.98178,
+ "295": 7.97108,
+ "296": 8.24114,
+ "297": 7.79647,
+ "298": 8.04847,
+ "299": 7.94257,
+ "300": 7.85748,
+ "301": 8.01649,
+ "302": 7.95112,
+ "303": 7.99606,
+ "304": 7.96394,
+ "305": 8.00301,
+ "306": 7.98312,
+ "307": 7.99372,
+ "308": 8.00491,
+ "309": 8.01362,
+ "310": 7.97824,
+ "311": 7.9323,
+ "312": 7.89419,
+ "313": 7.84054,
+ "314": 7.83,
+ "315": 7.8335,
+ "316": 7.75122,
+ "317": 7.934,
+ "318": 7.98841,
+ "319": 7.83343,
+ "320": 7.57896,
+ "321": 7.75427,
+ "322": 7.83781,
+ "323": 7.7769,
+ "324": 7.91623,
+ "325": 7.80539,
+ "326": 7.65641,
+ "327": 7.86989,
+ "328": 7.79369,
+ "329": 7.89137,
+ "330": 7.7586,
+ "331": 7.52885,
+ "332": 7.81946,
+ "333": 7.84359,
+ "334": 7.68375,
+ "335": 7.69975,
+ "336": 7.91931,
+ "337": 7.65356,
+ "338": 7.90277,
+ "339": 7.7307,
+ "340": 7.7606,
+ "341": 7.70898,
+ "342": 7.82827,
+ "343": 7.61824,
+ "344": 7.58818,
+ "345": 7.61602,
+ "346": 7.46415,
+ "347": 7.5612,
+ "348": 7.68737,
+ "349": 7.58361,
+ "350": 7.65762,
+ "351": 7.75424,
+ "352": 7.711,
+ "353": 7.50477,
+ "354": 7.74925,
+ "355": 7.77011,
+ "356": 7.78305,
+ "357": 7.81855,
+ "358": 7.60031,
+ "359": 7.55187,
+ "360": 7.63213,
+ "361": 7.55298,
+ "362": 7.76875,
+ "363": 7.59465,
+ "364": 7.57928,
+ "365": 7.62839,
+ "366": 7.31096,
+ "367": 7.55919,
+ "368": 7.44577,
+ "369": 7.3551,
+ "370": 7.46985,
+ "371": 7.46609,
+ "372": 7.65475,
+ "373": 7.52989,
+ "374": 7.44843,
+ "375": 7.53627,
+ "376": 7.35288,
+ "377": 7.24313,
+ "378": 7.54312,
+ "379": 7.4994,
+ "380": 7.38859,
+ "381": 7.47577,
+ "382": 7.29951,
+ "383": 7.28478,
+ "384": 7.4126,
+ "385": 7.39829,
+ "386": 7.23652,
+ "387": 7.42535,
+ "388": 7.28487,
+ "389": 7.44425,
+ "390": 7.24578,
+ "391": 7.6482,
+ "392": 7.34245,
+ "393": 7.42463,
+ "394": 7.48248,
+ "395": 7.44483,
+ "396": 7.29231,
+ "397": 7.23386,
+ "398": 7.42507,
+ "399": 7.16173,
+ "400": 7.30149,
+ "401": 7.3585,
+ "402": 7.39832,
+ "403": 7.28806,
+ "404": 7.30832,
+ "405": 7.27202,
+ "406": 7.22485,
+ "407": 7.36688,
+ "408": 7.18877,
+ "409": 7.17334,
+ "410": 7.31999,
+ "411": 7.2223,
+ "412": 7.20595,
+ "413": 7.24047,
+ "414": 6.9176,
+ "415": 7.3341,
+ "416": 7.43139,
+ "417": 7.0298,
+ "418": 7.28201,
+ "419": 7.04286,
+ "420": 7.41864,
+ "421": 7.18456,
+ "422": 7.24003,
+ "423": 7.09785,
+ "424": 7.24581,
+ "425": 7.32182,
+ "426": 7.29342,
+ "427": 7.1359,
+ "428": 7.09617,
+ "429": 6.87976,
+ "430": 7.20691,
+ "431": 7.00662,
+ "432": 7.23762,
+ "433": 6.97996,
+ "434": 6.96131,
+ "435": 7.02219,
+ "436": 7.01484,
+ "437": 6.9921,
+ "438": 7.00514,
+ "439": 6.94235,
+ "440": 7.06367,
+ "441": 7.04936,
+ "442": 7.10187,
+ "443": 7.0941,
+ "444": 6.71175,
+ "445": 6.99825,
+ "446": 7.14631,
+ "447": 7.12745,
+ "448": 6.98621,
+ "449": 7.0508,
+ "450": 7.01761,
+ "451": 6.83255,
+ "452": 6.9157,
+ "453": 7.02056,
+ "454": 6.97019,
+ "455": 7.03145,
+ "456": 6.99451,
+ "457": 6.97283,
+ "458": 6.9066,
+ "459": 6.69482,
+ "460": 7.06773,
+ "461": 7.09857,
+ "462": 6.87116,
+ "463": 7.05522,
+ "464": 6.64922,
+ "465": 7.02852,
+ "466": 7.00594,
+ "467": 6.99935,
+ "468": 6.95215,
+ "469": 6.8291,
+ "470": 7.04615,
+ "471": 6.88316,
+ "472": 6.96104,
+ "473": 6.82398,
+ "474": 6.97228,
+ "475": 7.16917,
+ "476": 6.76379,
+ "477": 6.89771,
+ "478": 6.91142,
+ "479": 6.70396,
+ "480": 7.03025,
+ "481": 6.99763,
+ "482": 6.73608,
+ "483": 6.78502,
+ "484": 6.75413,
+ "485": 6.93205,
+ "486": 7.06796,
+ "487": 6.63653,
+ "488": 6.88737,
+ "489": 6.77108,
+ "490": 6.82685,
+ "491": 6.71122,
+ "492": 6.69849,
+ "493": 6.77155,
+ "494": 6.67651,
+ "495": 6.63733,
+ "496": 6.59006,
+ "497": 6.84564,
+ "498": 6.65256,
+ "499": 6.85952,
+ "500": 6.65795,
+ "501": 6.73562,
+ "502": 6.84527,
+ "503": 6.71173,
+ "504": 6.62075,
+ "505": 6.62291,
+ "506": 6.75234,
+ "507": 6.86844,
+ "508": 6.86157,
+ "509": 6.6555,
+ "510": 6.82834,
+ "511": 6.74132,
+ "512": 6.74051,
+ "513": 6.66032,
+ "514": 6.71273,
+ "515": 6.45045,
+ "516": 6.74436,
+ "517": 6.71073,
+ "518": 6.53817,
+ "519": 6.63527,
+ "520": 6.85868,
+ "521": 6.66571,
+ "522": 6.70871,
+ "523": 6.74553,
+ "524": 6.73396,
+ "525": 6.6762,
+ "526": 6.4139,
+ "527": 6.79901,
+ "528": 6.66011,
+ "529": 6.63182,
+ "530": 6.62611,
+ "531": 6.64289,
+ "532": 6.63292,
+ "533": 6.76391,
+ "534": 6.61301,
+ "535": 6.74754,
+ "536": 6.62605,
+ "537": 6.63867,
+ "538": 6.53166,
+ "539": 6.5542,
+ "540": 6.5862,
+ "541": 6.45207,
+ "542": 6.66957,
+ "543": 6.68064,
+ "544": 6.67601,
+ "545": 6.81307,
+ "546": 6.63333,
+ "547": 6.41838,
+ "548": 6.72367,
+ "549": 6.69982,
+ "550": 6.52974,
+ "551": 6.7478,
+ "552": 6.63991,
+ "553": 6.48451,
+ "554": 6.63407,
+ "555": 6.4629,
+ "556": 6.61792,
+ "557": 6.63496,
+ "558": 6.3874,
+ "559": 6.37379,
+ "560": 6.58293,
+ "561": 6.73352,
+ "562": 6.6356,
+ "563": 6.7444,
+ "564": 6.35291,
+ "565": 6.51482,
+ "566": 6.70247,
+ "567": 6.56973,
+ "568": 6.51145,
+ "569": 6.45578,
+ "570": 6.36768,
+ "571": 6.63597,
+ "572": 6.31359,
+ "573": 6.58668,
+ "574": 6.47613,
+ "575": 6.64961,
+ "576": 6.5168,
+ "577": 6.53078,
+ "578": 6.4847,
+ "579": 6.46709,
+ "580": 6.56793,
+ "581": 6.60857,
+ "582": 6.48362,
+ "583": 6.51541,
+ "584": 6.52831,
+ "585": 6.42713,
+ "586": 6.4178,
+ "587": 6.46113,
+ "588": 6.56878,
+ "589": 6.62653,
+ "590": 6.29114,
+ "591": 6.67541,
+ "592": 6.26902,
+ "593": 6.4773,
+ "594": 6.38719,
+ "595": 6.3632,
+ "596": 6.26099,
+ "597": 6.18986,
+ "598": 6.45726,
+ "599": 6.3998,
+ "600": 6.45709,
+ "601": 6.26132,
+ "602": 6.5338,
+ "603": 6.52288,
+ "604": 6.38993,
+ "605": 6.49993,
+ "606": 6.31475,
+ "607": 6.53507,
+ "608": 6.67525,
+ "609": 6.17714,
+ "610": 6.57295,
+ "611": 6.40188,
+ "612": 6.57929,
+ "613": 6.42667,
+ "614": 6.20672,
+ "615": 6.40081,
+ "616": 6.36019,
+ "617": 6.37969,
+ "618": 6.4512,
+ "619": 6.14244,
+ "620": 6.41233,
+ "621": 6.46338,
+ "622": 6.40096,
+ "623": 6.58352,
+ "624": 6.36078,
+ "625": 6.28553,
+ "626": 6.30525,
+ "627": 6.44574,
+ "628": 6.2557,
+ "629": 6.58813,
+ "630": 6.36641,
+ "631": 6.3498,
+ "632": 6.30972,
+ "633": 6.25733,
+ "634": 6.30887,
+ "635": 6.54592,
+ "636": 6.24834,
+ "637": 6.63634,
+ "638": 6.02046,
+ "639": 6.2798,
+ "640": 6.29548,
+ "641": 6.20953,
+ "642": 6.28471,
+ "643": 6.461,
+ "644": 6.25863,
+ "645": 6.25115,
+ "646": 6.40601,
+ "647": 6.33707,
+ "648": 6.35671,
+ "649": 6.3488,
+ "650": 6.48415,
+ "651": 6.33395,
+ "652": 6.25233,
+ "653": 6.3826,
+ "654": 6.45063,
+ "655": 6.52494,
+ "656": 6.32781,
+ "657": 6.43503,
+ "658": 6.24353,
+ "659": 6.1554,
+ "660": 6.39397,
+ "661": 6.17184,
+ "662": 6.27494,
+ "663": 6.37237,
+ "664": 6.33376,
+ "665": 6.40442,
+ "666": 6.16399,
+ "667": 6.1965,
+ "668": 6.2366,
+ "669": 6.21813,
+ "670": 6.24601,
+ "671": 6.24468,
+ "672": 6.49032,
+ "673": 6.34071,
+ "674": 6.2969,
+ "675": 6.38396,
+ "676": 6.39021,
+ "677": 6.30588,
+ "678": 6.27751,
+ "679": 6.23892,
+ "680": 6.2942,
+ "681": 6.20621,
+ "682": 6.08719,
+ "683": 6.27464,
+ "684": 6.32896,
+ "685": 6.30248,
+ "686": 6.15397,
+ "687": 6.2862,
+ "688": 6.20754,
+ "689": 6.6215,
+ "690": 6.17931,
+ "691": 6.18188,
+ "692": 6.2745,
+ "693": 6.14405,
+ "694": 6.23487,
+ "695": 6.32617,
+ "696": 6.11842,
+ "697": 6.15483,
+ "698": 6.23128,
+ "699": 6.46051,
+ "700": 6.0454,
+ "701": 6.06467,
+ "702": 6.25219,
+ "703": 6.18603,
+ "704": 6.21704,
+ "705": 6.13155,
+ "706": 6.07593,
+ "707": 6.25376,
+ "708": 6.31553,
+ "709": 6.01087,
+ "710": 6.16305,
+ "711": 6.26062,
+ "712": 6.18307,
+ "713": 5.89806,
+ "714": 6.10759,
+ "715": 6.11617,
+ "716": 6.41405,
+ "717": 6.19202,
+ "718": 6.2345,
+ "719": 6.27471,
+ "720": 6.26372,
+ "721": 6.26277,
+ "722": 6.23442,
+ "723": 6.0814,
+ "724": 6.22797,
+ "725": 6.04057,
+ "726": 6.30046,
+ "727": 6.01682,
+ "728": 6.04617,
+ "729": 6.09111,
+ "730": 6.18359,
+ "731": 6.10398,
+ "732": 6.08898,
+ "733": 6.12312,
+ "734": 6.38423,
+ "735": 6.27849,
+ "736": 6.18184,
+ "737": 6.36645,
+ "738": 6.13411,
+ "739": 6.14591,
+ "740": 5.87975,
+ "741": 6.00667,
+ "742": 5.98459,
+ "743": 6.17495,
+ "744": 6.02962,
+ "745": 6.15497,
+ "746": 6.03272,
+ "747": 6.09789,
+ "748": 6.23436,
+ "749": 5.94191,
+ "750": 6.16819,
+ "751": 5.9596,
+ "752": 6.01941,
+ "753": 6.02989,
+ "754": 6.28798,
+ "755": 6.13521,
+ "756": 6.25357,
+ "757": 6.02098,
+ "758": 6.20422,
+ "759": 6.23062,
+ "760": 6.02316,
+ "761": 6.19655,
+ "762": 6.22713,
+ "763": 6.03754,
+ "764": 5.9636,
+ "765": 5.93413,
+ "766": 5.97155,
+ "767": 5.81277,
+ "768": 6.18725,
+ "769": 6.27646,
+ "770": 6.29561,
+ "771": 5.78767,
+ "772": 6.03281,
+ "773": 6.18558,
+ "774": 5.88583,
+ "775": 6.03167,
+ "776": 6.13086,
+ "777": 5.88612,
+ "778": 6.05891,
+ "779": 5.87414,
+ "780": 6.14047,
+ "781": 5.85641,
+ "782": 6.04961,
+ "783": 5.95687,
+ "784": 5.91852,
+ "785": 6.09816,
+ "786": 6.10929,
+ "787": 5.66006,
+ "788": 5.99915,
+ "789": 6.21789,
+ "790": 6.26737,
+ "791": 5.79122,
+ "792": 5.99828,
+ "793": 6.18387,
+ "794": 6.02746,
+ "795": 6.0051,
+ "796": 6.17065,
+ "797": 6.05376,
+ "798": 6.06076,
+ "799": 6.11682,
+ "800": 6.02167,
+ "801": 6.15011,
+ "802": 5.98473,
+ "803": 6.15363,
+ "804": 6.00859,
+ "805": 5.83055,
+ "806": 6.08757,
+ "807": 6.04997,
+ "808": 5.92717,
+ "809": 5.77802,
+ "810": 6.01973,
+ "811": 5.93299,
+ "812": 5.91169,
+ "813": 5.96567,
+ "814": 6.0369,
+ "815": 5.8146,
+ "816": 6.12034,
+ "817": 5.94337,
+ "818": 6.0674,
+ "819": 6.01476,
+ "820": 5.7319,
+ "821": 5.95027,
+ "822": 6.20452,
+ "823": 5.83139,
+ "824": 5.98275,
+ "825": 6.18795,
+ "826": 6.20019,
+ "827": 6.05802,
+ "828": 6.06976,
+ "829": 5.89149,
+ "830": 5.94221,
+ "831": 5.89773,
+ "832": 5.97341,
+ "833": 6.06501,
+ "834": 5.99675,
+ "835": 6.00654,
+ "836": 5.79277,
+ "837": 6.11496,
+ "838": 5.86966,
+ "839": 5.83554,
+ "840": 6.18614,
+ "841": 5.78491,
+ "842": 5.89169,
+ "843": 5.95102,
+ "844": 6.00954,
+ "845": 6.09153,
+ "846": 5.68733,
+ "847": 5.75715,
+ "848": 5.96838,
+ "849": 6.09512,
+ "850": 5.84886,
+ "851": 6.01693,
+ "852": 5.75188,
+ "853": 5.99355,
+ "854": 6.01844,
+ "855": 5.81656,
+ "856": 5.99593,
+ "857": 6.00207,
+ "858": 6.05507,
+ "859": 5.95295,
+ "860": 6.09632,
+ "861": 6.07189,
+ "862": 6.00434,
+ "863": 5.83757,
+ "864": 5.84474,
+ "865": 5.93791,
+ "866": 5.89404,
+ "867": 5.87803,
+ "868": 6.06515,
+ "869": 6.08564,
+ "870": 5.97153,
+ "871": 6.04317,
+ "872": 5.89525,
+ "873": 5.84383,
+ "874": 6.02742,
+ "875": 5.9144,
+ "876": 5.96905,
+ "877": 5.92979,
+ "878": 6.09819,
+ "879": 5.76783,
+ "880": 6.01501,
+ "881": 5.99647,
+ "882": 5.9097,
+ "883": 5.67626,
+ "884": 5.96521,
+ "885": 5.74544,
+ "886": 5.99268,
+ "887": 5.90979,
+ "888": 5.83897,
+ "889": 6.01033,
+ "890": 6.02378,
+ "891": 5.95247,
+ "892": 5.70829,
+ "893": 6.0922,
+ "894": 5.73134,
+ "895": 5.84057,
+ "896": 5.84075,
+ "897": 5.8564,
+ "898": 5.9238,
+ "899": 5.93486,
+ "900": 5.89946,
+ "901": 5.95293,
+ "902": 5.83295,
+ "903": 6.05665,
+ "904": 5.93153,
+ "905": 5.90441,
+ "906": 5.6172,
+ "907": 5.91178,
+ "908": 5.73853,
+ "909": 5.99118,
+ "910": 5.86603,
+ "911": 5.70397,
+ "912": 5.70712,
+ "913": 5.76497,
+ "914": 5.83944,
+ "915": 5.80032,
+ "916": 5.8904,
+ "917": 5.86913,
+ "918": 5.82415,
+ "919": 5.81575,
+ "920": 5.89552,
+ "921": 5.84163,
+ "922": 5.62427,
+ "923": 6.03657,
+ "924": 5.60536,
+ "925": 5.62335,
+ "926": 5.86148,
+ "927": 5.96071,
+ "928": 5.84005,
+ "929": 5.82702,
+ "930": 5.95816,
+ "931": 5.765,
+ "932": 5.59211,
+ "933": 5.6351,
+ "934": 5.80541,
+ "935": 5.63715,
+ "936": 5.83772,
+ "937": 5.96629,
+ "938": 5.59109,
+ "939": 5.7899,
+ "940": 5.96726,
+ "941": 5.7264,
+ "942": 5.83547,
+ "943": 5.86622,
+ "944": 5.95478,
+ "945": 5.70263,
+ "946": 5.55832,
+ "947": 5.74831,
+ "948": 5.79312,
+ "949": 5.8268,
+ "950": 5.84353,
+ "951": 5.72242,
+ "952": 5.69295,
+ "953": 5.67852,
+ "954": 5.72473,
+ "955": 5.53107,
+ "956": 5.62074,
+ "957": 5.84076,
+ "958": 5.79676,
+ "959": 5.57317,
+ "960": 5.80125,
+ "961": 5.82952,
+ "962": 5.76695,
+ "963": 5.76461,
+ "964": 5.70677,
+ "965": 5.64012,
+ "966": 5.59617,
+ "967": 5.72434,
+ "968": 5.74036,
+ "969": 5.82392,
+ "970": 5.64422,
+ "971": 5.7065,
+ "972": 5.85308,
+ "973": 5.66884,
+ "974": 5.71841,
+ "975": 5.86273,
+ "976": 5.70493,
+ "977": 5.77104,
+ "978": 5.6858,
+ "979": 5.58655,
+ "980": 5.75924,
+ "981": 5.8969,
+ "982": 5.47038,
+ "983": 5.61817,
+ "984": 5.54504,
+ "985": 5.59032,
+ "986": 5.64132,
+ "987": 5.56966,
+ "988": 5.70939,
+ "989": 5.69379,
+ "990": 5.62195,
+ "991": 5.84899,
+ "992": 5.77877,
+ "993": 5.87022,
+ "994": 5.69735,
+ "995": 5.73242,
+ "996": 5.73704,
+ "997": 5.81329,
+ "998": 5.83634,
+ "999": 5.83399,
+ "1000": 5.68342,
+ "1001": 5.86668,
+ "1002": 5.76052,
+ "1003": 5.64259,
+ "1004": 5.79811,
+ "1005": 5.53617,
+ "1006": 5.326,
+ "1007": 5.76701,
+ "1008": 5.79136,
+ "1009": 5.65046,
+ "1010": 5.77942,
+ "1011": 5.89493,
+ "1012": 5.62303,
+ "1013": 5.61569,
+ "1014": 5.68111,
+ "1015": 5.55747,
+ "1016": 5.87327,
+ "1017": 5.83312,
+ "1018": 5.61865,
+ "1019": 5.73414,
+ "1020": 5.61755,
+ "1021": 5.848,
+ "1022": 5.50045,
+ "1023": 5.65182,
+ "1024": 5.74493,
+ "1025": 5.5692,
+ "1026": 5.41415,
+ "1027": 5.60696,
+ "1028": 5.6928,
+ "1029": 5.68764,
+ "1030": 5.68746,
+ "1031": 5.40696,
+ "1032": 5.78748,
+ "1033": 5.58136,
+ "1034": 5.61937,
+ "1035": 5.71368,
+ "1036": 5.62818,
+ "1037": 5.3679,
+ "1038": 5.66452,
+ "1039": 5.64347,
+ "1040": 5.57004,
+ "1041": 5.59722,
+ "1042": 5.81329,
+ "1043": 5.566,
+ "1044": 5.46906,
+ "1045": 5.9659,
+ "1046": 5.4866,
+ "1047": 5.38954,
+ "1048": 5.50027,
+ "1049": 5.67182,
+ "1050": 5.6991,
+ "1051": 5.57928,
+ "1052": 5.68227,
+ "1053": 5.62737,
+ "1054": 5.45766,
+ "1055": 5.60313,
+ "1056": 5.67386,
+ "1057": 5.75895,
+ "1058": 5.56782,
+ "1059": 5.74888,
+ "1060": 5.82022,
+ "1061": 5.47624,
+ "1062": 5.64897,
+ "1063": 5.50121,
+ "1064": 5.59136,
+ "1065": 5.55347,
+ "1066": 5.74367,
+ "1067": 5.67235,
+ "1068": 5.44068,
+ "1069": 5.60636,
+ "1070": 5.81264,
+ "1071": 5.51129,
+ "1072": 5.61871,
+ "1073": 5.62147,
+ "1074": 5.524,
+ "1075": 5.70529,
+ "1076": 5.5934,
+ "1077": 5.71153,
+ "1078": 5.56524,
+ "1079": 5.61728,
+ "1080": 5.64251,
+ "1081": 5.62319,
+ "1082": 5.49648,
+ "1083": 5.64086,
+ "1084": 5.55389,
+ "1085": 5.40631,
+ "1086": 5.62008,
+ "1087": 5.44148,
+ "1088": 5.51218,
+ "1089": 5.7676,
+ "1090": 5.53165,
+ "1091": 5.51388,
+ "1092": 5.41011,
+ "1093": 5.70025,
+ "1094": 5.57364,
+ "1095": 5.57735,
+ "1096": 5.61585,
+ "1097": 5.64586,
+ "1098": 5.64877,
+ "1099": 5.51631,
+ "1100": 5.63778,
+ "1101": 5.67335,
+ "1102": 5.54037,
+ "1103": 5.54969,
+ "1104": 5.53882,
+ "1105": 5.54754,
+ "1106": 5.68315,
+ "1107": 5.68556,
+ "1108": 5.78611,
+ "1109": 5.53666,
+ "1110": 5.66598,
+ "1111": 5.58973,
+ "1112": 5.58039,
+ "1113": 5.62611,
+ "1114": 5.61279,
+ "1115": 5.59718,
+ "1116": 5.65925,
+ "1117": 5.64676,
+ "1118": 5.65036,
+ "1119": 5.70919,
+ "1120": 5.62738,
+ "1121": 5.37352,
+ "1122": 5.22976,
+ "1123": 5.47237,
+ "1124": 5.64939,
+ "1125": 5.67974,
+ "1126": 5.679,
+ "1127": 5.56811,
+ "1128": 5.61992,
+ "1129": 5.29637,
+ "1130": 5.54359,
+ "1131": 5.63153,
+ "1132": 5.72427,
+ "1133": 5.51914,
+ "1134": 5.56063,
+ "1135": 5.52056,
+ "1136": 5.42646,
+ "1137": 5.45971,
+ "1138": 5.56927,
+ "1139": 5.41452,
+ "1140": 5.2656,
+ "1141": 5.58265,
+ "1142": 5.64152,
+ "1143": 5.38298,
+ "1144": 5.38584,
+ "1145": 5.36231,
+ "1146": 5.63508,
+ "1147": 5.49183,
+ "1148": 5.50524,
+ "1149": 5.52352,
+ "1150": 5.39801,
+ "1151": 5.5563,
+ "1152": 5.41525,
+ "1153": 5.44791,
+ "1154": 5.49757,
+ "1155": 5.43833,
+ "1156": 5.3488,
+ "1157": 5.66444,
+ "1158": 5.39487,
+ "1159": 5.33455,
+ "1160": 5.79503,
+ "1161": 5.53955,
+ "1162": 5.45818,
+ "1163": 5.52563,
+ "1164": 5.3837,
+ "1165": 5.52861,
+ "1166": 5.48753,
+ "1167": 5.36312,
+ "1168": 5.49491,
+ "1169": 5.39842,
+ "1170": 5.59202,
+ "1171": 5.48502,
+ "1172": 5.64238,
+ "1173": 5.62295,
+ "1174": 5.50843,
+ "1175": 5.34639,
+ "1176": 5.38504,
+ "1177": 5.55461,
+ "1178": 5.46852,
+ "1179": 5.49505,
+ "1180": 5.46014,
+ "1181": 5.56031,
+ "1182": 5.59593,
+ "1183": 5.77155,
+ "1184": 5.54926,
+ "1185": 5.29008,
+ "1186": 5.60451,
+ "1187": 5.55363,
+ "1188": 5.51655,
+ "1189": 5.39133,
+ "1190": 5.40482,
+ "1191": 5.39266,
+ "1192": 5.50142,
+ "1193": 5.46347,
+ "1194": 5.45607,
+ "1195": 5.32751,
+ "1196": 5.52219,
+ "1197": 5.4809,
+ "1198": 5.52789,
+ "1199": 5.3874,
+ "1200": 5.33059,
+ "1201": 5.48969,
+ "1202": 5.43584,
+ "1203": 5.49537,
+ "1204": 5.40861,
+ "1205": 5.48971,
+ "1206": 5.3371,
+ "1207": 5.58625,
+ "1208": 5.4312,
+ "1209": 5.29323,
+ "1210": 5.50765,
+ "1211": 5.51506,
+ "1212": 5.59777,
+ "1213": 5.42123,
+ "1214": 5.51018,
+ "1215": 5.23832,
+ "1216": 5.40989,
+ "1217": 5.38537,
+ "1218": 5.45232,
+ "1219": 5.48221,
+ "1220": 5.38594,
+ "1221": 5.44848,
+ "1222": 5.31032,
+ "1223": 5.47835,
+ "1224": 5.42017,
+ "1225": 5.43499,
+ "1226": 5.3238,
+ "1227": 5.47632,
+ "1228": 5.72418,
+ "1229": 5.32629,
+ "1230": 5.40556,
+ "1231": 5.06972,
+ "1232": 5.78794,
+ "1233": 5.28923,
+ "1234": 5.24535,
+ "1235": 5.37092,
+ "1236": 5.48471,
+ "1237": 5.20864,
+ "1238": 5.41643,
+ "1239": 5.40751,
+ "1240": 5.46767,
+ "1241": 5.57266,
+ "1242": 5.4536,
+ "1243": 5.43063,
+ "1244": 5.51812,
+ "1245": 5.19115,
+ "1246": 5.72042,
+ "1247": 5.43187,
+ "1248": 5.30004,
+ "1249": 5.40113,
+ "1250": 5.33798,
+ "1251": 5.42034,
+ "1252": 5.57217,
+ "1253": 5.48773,
+ "1254": 5.30628,
+ "1255": 5.51443,
+ "1256": 5.60755,
+ "1257": 5.4214,
+ "1258": 5.56457,
+ "1259": 5.48027,
+ "1260": 5.51461,
+ "1261": 5.63883,
+ "1262": 5.39531,
+ "1263": 5.32916,
+ "1264": 5.50671,
+ "1265": 5.30632,
+ "1266": 5.23819,
+ "1267": 5.37206,
+ "1268": 5.39267,
+ "1269": 5.15366,
+ "1270": 5.40418,
+ "1271": 5.27732,
+ "1272": 5.5252,
+ "1273": 5.30228,
+ "1274": 5.3516,
+ "1275": 5.38466,
+ "1276": 5.39786,
+ "1277": 5.46218,
+ "1278": 5.34689,
+ "1279": 5.44274,
+ "1280": 5.45919,
+ "1281": 5.40638,
+ "1282": 5.3824,
+ "1283": 5.42204,
+ "1284": 5.34841,
+ "1285": 5.50133,
+ "1286": 5.33557,
+ "1287": 5.58795,
+ "1288": 5.26493,
+ "1289": 5.429,
+ "1290": 5.50282,
+ "1291": 5.50335,
+ "1292": 5.44662,
+ "1293": 5.41955,
+ "1294": 5.49953,
+ "1295": 5.34675,
+ "1296": 5.19062,
+ "1297": 5.17238,
+ "1298": 5.11916,
+ "1299": 5.30339,
+ "1300": 5.21032,
+ "1301": 5.30157,
+ "1302": 5.27472,
+ "1303": 5.36107,
+ "1304": 5.43231,
+ "1305": 5.36999,
+ "1306": 5.25347,
+ "1307": 5.18829,
+ "1308": 5.27033,
+ "1309": 5.40736,
+ "1310": 5.26399,
+ "1311": 5.38109,
+ "1312": 5.35438,
+ "1313": 5.30056,
+ "1314": 5.2953,
+ "1315": 5.42245,
+ "1316": 5.26148,
+ "1317": 5.28065,
+ "1318": 5.2198,
+ "1319": 5.34619,
+ "1320": 5.42093,
+ "1321": 5.44976,
+ "1322": 5.46399,
+ "1323": 5.37327,
+ "1324": 5.25463,
+ "1325": 5.40657,
+ "1326": 5.54082,
+ "1327": 5.39378,
+ "1328": 5.21893,
+ "1329": 5.41851,
+ "1330": 5.40079,
+ "1331": 5.31685,
+ "1332": 5.31253,
+ "1333": 5.37243,
+ "1334": 5.44685,
+ "1335": 5.37136,
+ "1336": 5.43779,
+ "1337": 5.47852,
+ "1338": 5.30292,
+ "1339": 5.14181,
+ "1340": 5.41486,
+ "1341": 5.3443,
+ "1342": 5.36197,
+ "1343": 5.47816,
+ "1344": 5.37832,
+ "1345": 5.34294,
+ "1346": 5.08195,
+ "1347": 5.38558,
+ "1348": 5.4918,
+ "1349": 5.40832,
+ "1350": 5.02622,
+ "1351": 5.3151,
+ "1352": 5.1591,
+ "1353": 5.34674,
+ "1354": 5.35963,
+ "1355": 5.11092,
+ "1356": 5.2587,
+ "1357": 5.29209,
+ "1358": 5.15773,
+ "1359": 5.11035,
+ "1360": 5.17288,
+ "1361": 5.30521,
+ "1362": 5.06318,
+ "1363": 5.2947,
+ "1364": 5.40031,
+ "1365": 5.02241,
+ "1366": 5.11779,
+ "1367": 5.33051,
+ "1368": 5.18648,
+ "1369": 5.22984,
+ "1370": 5.19906,
+ "1371": 5.2839,
+ "1372": 5.26155,
+ "1373": 5.28402,
+ "1374": 5.28112,
+ "1375": 5.46052,
+ "1376": 5.2713,
+ "1377": 5.26467,
+ "1378": 5.31344,
+ "1379": 5.22741,
+ "1380": 5.26107,
+ "1381": 5.47871,
+ "1382": 5.08923,
+ "1383": 5.375,
+ "1384": 5.35914,
+ "1385": 5.38983,
+ "1386": 5.16417,
+ "1387": 5.16094,
+ "1388": 5.28017,
+ "1389": 5.30376,
+ "1390": 5.25514,
+ "1391": 5.26911,
+ "1392": 5.37008,
+ "1393": 5.38307,
+ "1394": 5.40394,
+ "1395": 5.32492,
+ "1396": 5.21356,
+ "1397": 5.28,
+ "1398": 5.37051,
+ "1399": 5.35873,
+ "1400": 5.26512,
+ "1401": 5.35924,
+ "1402": 5.42148,
+ "1403": 5.20238,
+ "1404": 5.28629,
+ "1405": 5.11984,
+ "1406": 4.99128,
+ "1407": 5.40442,
+ "1408": 5.19825,
+ "1409": 5.3964,
+ "1410": 5.37519,
+ "1411": 4.91758,
+ "1412": 5.35561,
+ "1413": 5.41314,
+ "1414": 5.21823,
+ "1415": 5.44159,
+ "1416": 5.32905,
+ "1417": 5.38859,
+ "1418": 5.29946,
+ "1419": 5.31787,
+ "1420": 5.43974,
+ "1421": 5.39414,
+ "1422": 5.41749,
+ "1423": 5.005,
+ "1424": 5.32995,
+ "1425": 5.58618,
+ "1426": 5.23059,
+ "1427": 5.31804,
+ "1428": 5.33277,
+ "1429": 5.07552,
+ "1430": 5.33075,
+ "1431": 5.32688,
+ "1432": 5.33826,
+ "1433": 5.19107,
+ "1434": 5.16341,
+ "1435": 5.19905,
+ "1436": 5.10851,
+ "1437": 5.229,
+ "1438": 5.31867,
+ "1439": 5.34731,
+ "1440": 5.34991,
+ "1441": 5.16484,
+ "1442": 5.22015,
+ "1443": 5.20933,
+ "1444": 5.13701,
+ "1445": 5.07414,
+ "1446": 5.26836,
+ "1447": 5.25895,
+ "1448": 5.2904,
+ "1449": 5.2498,
+ "1450": 5.34281,
+ "1451": 5.07084,
+ "1452": 5.27052,
+ "1453": 5.1668,
+ "1454": 5.01539,
+ "1455": 5.12292,
+ "1456": 5.2717,
+ "1457": 5.18713,
+ "1458": 5.00608,
+ "1459": 5.22304,
+ "1460": 5.23389,
+ "1461": 5.07142,
+ "1462": 4.96923,
+ "1463": 5.14383,
+ "1464": 5.21128,
+ "1465": 5.26911,
+ "1466": 5.34961,
+ "1467": 5.33438,
+ "1468": 5.22205,
+ "1469": 5.04373,
+ "1470": 5.11715,
+ "1471": 5.25199,
+ "1472": 5.12294,
+ "1473": 5.10395,
+ "1474": 5.21775,
+ "1475": 5.18567,
+ "1476": 5.15287,
+ "1477": 5.26203,
+ "1478": 5.30399,
+ "1479": 5.01175,
+ "1480": 5.1809,
+ "1481": 5.24516,
+ "1482": 5.34866,
+ "1483": 5.26395,
+ "1484": 4.92397,
+ "1485": 5.29179,
+ "1486": 5.04178,
+ "1487": 4.88296,
+ "1488": 5.18145,
+ "1489": 5.10246,
+ "1490": 5.04399,
+ "1491": 5.31709,
+ "1492": 5.22469,
+ "1493": 4.94051,
+ "1494": 5.10929,
+ "1495": 5.13424,
+ "1496": 5.05862,
+ "1497": 5.36633,
+ "1498": 5.30967,
+ "1499": 5.13834,
+ "1500": 5.09851,
+ "1501": 5.03466,
+ "1502": 5.15527,
+ "1503": 5.43143,
+ "1504": 5.31968,
+ "1505": 5.00114,
+ "1506": 5.14444,
+ "1507": 5.16068,
+ "1508": 5.16575,
+ "1509": 5.31451,
+ "1510": 5.0185,
+ "1511": 5.11697,
+ "1512": 4.98287,
+ "1513": 5.16993,
+ "1514": 5.33962,
+ "1515": 5.36563,
+ "1516": 5.27715,
+ "1517": 5.22687,
+ "1518": 5.02626,
+ "1519": 5.29861,
+ "1520": 5.1417,
+ "1521": 5.15866,
+ "1522": 5.32824,
+ "1523": 5.24625,
+ "1524": 5.06725,
+ "1525": 5.20424,
+ "1526": 5.27994,
+ "1527": 5.25677,
+ "1528": 5.23589,
+ "1529": 5.18688,
+ "1530": 5.24365,
+ "1531": 5.09964,
+ "1532": 5.15141,
+ "1533": 5.05087,
+ "1534": 5.21589,
+ "1535": 5.1635,
+ "1536": 5.09678,
+ "1537": 5.02713,
+ "1538": 4.91184,
+ "1539": 5.23801,
+ "1540": 5.11515,
+ "1541": 5.25246,
+ "1542": 5.23484,
+ "1543": 5.05152,
+ "1544": 5.07544,
+ "1545": 5.1161,
+ "1546": 5.33085,
+ "1547": 5.11115,
+ "1548": 5.23527,
+ "1549": 5.23735,
+ "1550": 4.97596,
+ "1551": 5.2566,
+ "1552": 5.02944,
+ "1553": 5.14849,
+ "1554": 5.11205,
+ "1555": 5.10901,
+ "1556": 5.19824,
+ "1557": 5.08883,
+ "1558": 5.23067,
+ "1559": 5.00402,
+ "1560": 5.11835,
+ "1561": 5.14529,
+ "1562": 5.17996,
+ "1563": 5.24454,
+ "1564": 5.26389,
+ "1565": 5.08902,
+ "1566": 5.29474,
+ "1567": 5.04166,
+ "1568": 5.09256,
+ "1569": 5.20014,
+ "1570": 5.17348,
+ "1571": 4.95353,
+ "1572": 5.04005,
+ "1573": 5.02897,
+ "1574": 4.99751,
+ "1575": 5.2314,
+ "1576": 5.21263,
+ "1577": 5.12799,
+ "1578": 5.36241,
+ "1579": 4.94367,
+ "1580": 5.12197,
+ "1581": 5.09638,
+ "1582": 5.28497,
+ "1583": 5.04918,
+ "1584": 5.05482,
+ "1585": 5.11977,
+ "1586": 5.30243,
+ "1587": 5.13447,
+ "1588": 5.2184,
+ "1589": 4.83833,
+ "1590": 5.09497,
+ "1591": 5.17411,
+ "1592": 5.13721,
+ "1593": 5.23457,
+ "1594": 5.11805,
+ "1595": 5.10775,
+ "1596": 5.18964,
+ "1597": 5.11486,
+ "1598": 5.15917,
+ "1599": 5.19102,
+ "1600": 4.86871,
+ "1601": 5.11732,
+ "1602": 5.23185,
+ "1603": 5.19543,
+ "1604": 5.05128,
+ "1605": 5.02692,
+ "1606": 4.98659,
+ "1607": 5.07391,
+ "1608": 4.97985,
+ "1609": 5.07337,
+ "1610": 5.04745,
+ "1611": 4.99848,
+ "1612": 4.75205,
+ "1613": 5.03316,
+ "1614": 4.88034,
+ "1615": 5.07442,
+ "1616": 5.23082,
+ "1617": 5.06132,
+ "1618": 4.98704,
+ "1619": 5.18333,
+ "1620": 5.14491,
+ "1621": 5.31452,
+ "1622": 5.05677,
+ "1623": 5.14346,
+ "1624": 5.1355,
+ "1625": 5.12006,
+ "1626": 5.10245,
+ "1627": 5.10987,
+ "1628": 5.06581,
+ "1629": 4.92971,
+ "1630": 5.06799,
+ "1631": 5.06088,
+ "1632": 5.10428,
+ "1633": 4.97515,
+ "1634": 4.9235,
+ "1635": 5.05833,
+ "1636": 4.92289,
+ "1637": 5.24051,
+ "1638": 5.15574,
+ "1639": 4.977,
+ "1640": 5.00918,
+ "1641": 5.12718,
+ "1642": 5.08305,
+ "1643": 5.04894,
+ "1644": 5.1181,
+ "1645": 4.96677,
+ "1646": 5.11931,
+ "1647": 5.03295,
+ "1648": 5.19969,
+ "1649": 4.92396,
+ "1650": 5.05963,
+ "1651": 4.92965,
+ "1652": 5.21121,
+ "1653": 5.15959,
+ "1654": 5.12828,
+ "1655": 5.16263,
+ "1656": 5.34595,
+ "1657": 5.20677,
+ "1658": 5.04112,
+ "1659": 4.9258,
+ "1660": 4.80954,
+ "1661": 5.03086,
+ "1662": 5.14123,
+ "1663": 5.15449,
+ "1664": 4.981,
+ "1665": 5.11714,
+ "1666": 5.10575,
+ "1667": 4.84897,
+ "1668": 5.11513,
+ "1669": 5.06995,
+ "1670": 5.11266,
+ "1671": 5.17201,
+ "1672": 4.77569,
+ "1673": 5.03851,
+ "1674": 4.91569,
+ "1675": 5.05176,
+ "1676": 5.00402,
+ "1677": 4.79944,
+ "1678": 5.02487,
+ "1679": 4.89421,
+ "1680": 5.03847,
+ "1681": 5.06815,
+ "1682": 5.03274,
+ "1683": 4.90688,
+ "1684": 5.06515,
+ "1685": 5.13579,
+ "1686": 5.0732,
+ "1687": 4.97656,
+ "1688": 5.16537,
+ "1689": 5.14707,
+ "1690": 4.99688,
+ "1691": 5.00011,
+ "1692": 4.91822,
+ "1693": 5.01472,
+ "1694": 4.94657,
+ "1695": 4.91341,
+ "1696": 5.08209,
+ "1697": 5.04294,
+ "1698": 4.9511,
+ "1699": 5.00187,
+ "1700": 4.95393,
+ "1701": 5.16563,
+ "1702": 5.07666,
+ "1703": 5.17125,
+ "1704": 5.14332,
+ "1705": 4.96247,
+ "1706": 4.98333,
+ "1707": 4.79005,
+ "1708": 5.03831,
+ "1709": 5.23334,
+ "1710": 5.02934,
+ "1711": 5.19037,
+ "1712": 5.1958,
+ "1713": 5.03582,
+ "1714": 5.04603,
+ "1715": 4.91495,
+ "1716": 4.9332,
+ "1717": 4.86109,
+ "1718": 5.0273,
+ "1719": 5.12334,
+ "1720": 5.02189,
+ "1721": 4.92752,
+ "1722": 5.05412,
+ "1723": 4.93537,
+ "1724": 5.0407,
+ "1725": 5.1914,
+ "1726": 5.06447,
+ "1727": 4.90742,
+ "1728": 5.02116,
+ "1729": 5.04574,
+ "1730": 4.90343,
+ "1731": 4.99945,
+ "1732": 4.92083,
+ "1733": 5.1311,
+ "1734": 4.82837,
+ "1735": 5.20905,
+ "1736": 4.91585,
+ "1737": 4.85859,
+ "1738": 4.97909,
+ "1739": 5.16688,
+ "1740": 4.83514,
+ "1741": 4.77896,
+ "1742": 4.90909,
+ "1743": 5.08523,
+ "1744": 4.9784,
+ "1745": 4.82327,
+ "1746": 4.94833,
+ "1747": 4.87022,
+ "1748": 5.06379,
+ "1749": 4.8705,
+ "1750": 5.01347,
+ "1751": 5.12189,
+ "1752": 4.90364,
+ "1753": 5.09398,
+ "1754": 5.05918,
+ "1755": 4.89649,
+ "1756": 5.02243,
+ "1757": 5.14389,
+ "1758": 4.8716,
+ "1759": 4.94237,
+ "1760": 4.83366,
+ "1761": 5.02233,
+ "1762": 4.81292,
+ "1763": 4.77382,
+ "1764": 4.93787,
+ "1765": 5.14977,
+ "1766": 5.33847,
+ "1767": 5.22339,
+ "1768": 4.95072,
+ "1769": 5.00607,
+ "1770": 4.98077,
+ "1771": 4.96436,
+ "1772": 4.98395,
+ "1773": 4.97312,
+ "1774": 4.86859,
+ "1775": 4.95207,
+ "1776": 4.99761,
+ "1777": 4.94332,
+ "1778": 4.99268,
+ "1779": 5.08376,
+ "1780": 4.83276,
+ "1781": 5.05321,
+ "1782": 4.9968,
+ "1783": 5.01268,
+ "1784": 4.93195,
+ "1785": 5.16736,
+ "1786": 4.81265,
+ "1787": 4.97081,
+ "1788": 4.82725,
+ "1789": 4.88846,
+ "1790": 4.79821,
+ "1791": 4.73741,
+ "1792": 4.87626,
+ "1793": 5.10356,
+ "1794": 4.98084,
+ "1795": 4.96551,
+ "1796": 4.99704,
+ "1797": 4.7903,
+ "1798": 4.76702,
+ "1799": 5.01884,
+ "1800": 4.91364,
+ "1801": 5.04679,
+ "1802": 4.82665,
+ "1803": 4.95171,
+ "1804": 4.88594,
+ "1805": 4.90346,
+ "1806": 4.87351,
+ "1807": 4.92406,
+ "1808": 4.92697,
+ "1809": 5.1451,
+ "1810": 5.09976,
+ "1811": 4.95906,
+ "1812": 4.80139,
+ "1813": 5.09748,
+ "1814": 4.77766,
+ "1815": 4.86134,
+ "1816": 5.05005,
+ "1817": 4.79012,
+ "1818": 4.80376,
+ "1819": 5.02382,
+ "1820": 4.68652,
+ "1821": 5.02661,
+ "1822": 4.66251,
+ "1823": 4.8659,
+ "1824": 4.78635,
+ "1825": 5.06537,
+ "1826": 4.81944,
+ "1827": 4.7895,
+ "1828": 4.94677,
+ "1829": 5.11262,
+ "1830": 4.91236,
+ "1831": 4.89818,
+ "1832": 4.83359,
+ "1833": 4.78363,
+ "1834": 4.9482,
+ "1835": 4.95795,
+ "1836": 4.90747,
+ "1837": 4.67243,
+ "1838": 4.80953,
+ "1839": 4.89546,
+ "1840": 4.90488,
+ "1841": 4.8292,
+ "1842": 4.94678,
+ "1843": 4.70293,
+ "1844": 4.61431,
+ "1845": 5.00086,
+ "1846": 4.74657,
+ "1847": 4.8645,
+ "1848": 4.89695,
+ "1849": 4.85358,
+ "1850": 4.8676,
+ "1851": 5.02236,
+ "1852": 4.97647,
+ "1853": 4.83325,
+ "1854": 4.86791,
+ "1855": 4.8219,
+ "1856": 4.75614,
+ "1857": 4.9619,
+ "1858": 4.96856,
+ "1859": 4.75323,
+ "1860": 4.86592,
+ "1861": 5.20685,
+ "1862": 4.61669,
+ "1863": 4.83385,
+ "1864": 4.7505,
+ "1865": 4.86441,
+ "1866": 4.79455,
+ "1867": 4.99688,
+ "1868": 4.71331,
+ "1869": 4.75634,
+ "1870": 4.93203,
+ "1871": 4.99184,
+ "1872": 4.68332,
+ "1873": 4.69823,
+ "1874": 4.85174,
+ "1875": 4.85999,
+ "1876": 4.7392,
+ "1877": 4.80362,
+ "1878": 4.81239,
+ "1879": 4.82084,
+ "1880": 4.89314,
+ "1881": 4.79389,
+ "1882": 4.79419,
+ "1883": 4.78157,
+ "1884": 4.97086,
+ "1885": 4.91799,
+ "1886": 4.82203,
+ "1887": 4.81334,
+ "1888": 4.97395,
+ "1889": 4.95922,
+ "1890": 4.70676,
+ "1891": 4.65282,
+ "1892": 4.84393,
+ "1893": 4.64594,
+ "1894": 4.90265,
+ "1895": 4.7886,
+ "1896": 4.66112,
+ "1897": 4.78966,
+ "1898": 4.9139,
+ "1899": 4.77532,
+ "1900": 4.91571,
+ "1901": 4.84525,
+ "1902": 4.78411,
+ "1903": 4.75997,
+ "1904": 4.65339,
+ "1905": 4.54188,
+ "1906": 4.81097,
+ "1907": 4.90225,
+ "1908": 5.03012,
+ "1909": 4.88434,
+ "1910": 4.78852,
+ "1911": 4.80477,
+ "1912": 4.64685,
+ "1913": 4.94065,
+ "1914": 4.87965,
+ "1915": 4.85906,
+ "1916": 4.92227,
+ "1917": 4.85425,
+ "1918": 4.87001,
+ "1919": 4.99304,
+ "1920": 4.76319,
+ "1921": 4.88494,
+ "1922": 4.81295,
+ "1923": 4.7592,
+ "1924": 4.82501,
+ "1925": 5.05793,
+ "1926": 4.92996,
+ "1927": 4.92587,
+ "1928": 4.92702,
+ "1929": 4.92705,
+ "1930": 4.91019,
+ "1931": 4.77616,
+ "1932": 4.85963,
+ "1933": 4.83545,
+ "1934": 4.84013,
+ "1935": 5.10729,
+ "1936": 4.88314,
+ "1937": 4.87654,
+ "1938": 4.79463,
+ "1939": 4.71148,
+ "1940": 4.82418,
+ "1941": 4.73372,
+ "1942": 4.87249,
+ "1943": 4.7353,
+ "1944": 4.74198,
+ "1945": 4.6818,
+ "1946": 4.91539,
+ "1947": 4.86756,
+ "1948": 4.59887,
+ "1949": 4.90387,
+ "1950": 4.78785,
+ "1951": 4.95942,
+ "1952": 4.73677,
+ "1953": 4.79496,
+ "1954": 4.73264,
+ "1955": 4.84308,
+ "1956": 4.88233,
+ "1957": 4.73496,
+ "1958": 4.70018,
+ "1959": 4.75966,
+ "1960": 4.76849,
+ "1961": 4.7146,
+ "1962": 4.83392,
+ "1963": 4.82321,
+ "1964": 4.84664,
+ "1965": 4.87523,
+ "1966": 4.78753,
+ "1967": 4.59211,
+ "1968": 4.82724,
+ "1969": 4.59184,
+ "1970": 4.56633,
+ "1971": 4.9072,
+ "1972": 4.90064,
+ "1973": 4.54642,
+ "1974": 4.82423,
+ "1975": 4.82778,
+ "1976": 4.71327,
+ "1977": 4.57967,
+ "1978": 5.0045,
+ "1979": 4.66094,
+ "1980": 4.74256,
+ "1981": 4.86301,
+ "1982": 4.72234,
+ "1983": 4.8786,
+ "1984": 4.64152,
+ "1985": 4.78,
+ "1986": 4.70167,
+ "1987": 4.81036,
+ "1988": 4.8871,
+ "1989": 4.63185,
+ "1990": 4.79636,
+ "1991": 4.69424,
+ "1992": 4.79439,
+ "1993": 4.74063,
+ "1994": 4.84977,
+ "1995": 4.5596,
+ "1996": 4.65161,
+ "1997": 4.80342,
+ "1998": 4.67403,
+ "1999": 4.72284,
+ "2000": 4.61765
+ }
+ },
+ "num-zeros": {
+ "start_step": 1,
+ "end_step": 2000,
+ "step_interval": 1,
+ "values": {
+ "1": 80.0,
+ "2": 70.0,
+ "3": 78.0,
+ "4": 80.0,
+ "5": 75.0,
+ "6": 87.0,
+ "7": 63.0,
+ "8": 77.0,
+ "9": 62.0,
+ "10": 90.0,
+ "11": 74.0,
+ "12": 79.0,
+ "13": 77.0,
+ "14": 83.0,
+ "15": 78.0,
+ "16": 69.0,
+ "17": 64.0,
+ "18": 63.0,
+ "19": 87.0,
+ "20": 90.0,
+ "21": 75.0,
+ "22": 84.0,
+ "23": 81.0,
+ "24": 78.0,
+ "25": 87.0,
+ "26": 69.0,
+ "27": 86.0,
+ "28": 91.0,
+ "29": 94.0,
+ "30": 115.0,
+ "31": 99.0,
+ "32": 109.0,
+ "33": 92.0,
+ "34": 103.0,
+ "35": 118.0,
+ "36": 117.0,
+ "37": 105.0,
+ "38": 129.0,
+ "39": 89.0,
+ "40": 129.0,
+ "41": 114.0,
+ "42": 121.0,
+ "43": 135.0,
+ "44": 128.0,
+ "45": 126.0,
+ "46": 129.0,
+ "47": 133.0,
+ "48": 139.0,
+ "49": 135.0,
+ "50": 157.0,
+ "51": 122.0,
+ "52": 150.0,
+ "53": 108.0,
+ "54": 140.0,
+ "55": 133.0,
+ "56": 156.0,
+ "57": 150.0,
+ "58": 153.0,
+ "59": 135.0,
+ "60": 135.0,
+ "61": 165.0,
+ "62": 145.0,
+ "63": 199.0,
+ "64": 161.0,
+ "65": 162.0,
+ "66": 162.0,
+ "67": 195.0,
+ "68": 140.0,
+ "69": 158.0,
+ "70": 169.0,
+ "71": 188.0,
+ "72": 160.0,
+ "73": 151.0,
+ "74": 154.0,
+ "75": 172.0,
+ "76": 169.0,
+ "77": 165.0,
+ "78": 193.0,
+ "79": 144.0,
+ "80": 173.0,
+ "81": 150.0,
+ "82": 141.0,
+ "83": 186.0,
+ "84": 169.0,
+ "85": 183.0,
+ "86": 196.0,
+ "87": 197.0,
+ "88": 184.0,
+ "89": 169.0,
+ "90": 182.0,
+ "91": 200.0,
+ "92": 179.0,
+ "93": 165.0,
+ "94": 153.0,
+ "95": 176.0,
+ "96": 191.0,
+ "97": 183.0,
+ "98": 199.0,
+ "99": 163.0,
+ "100": 157.0,
+ "101": 144.0,
+ "102": 184.0,
+ "103": 206.0,
+ "104": 171.0,
+ "105": 215.0,
+ "106": 176.0,
+ "107": 172.0,
+ "108": 172.0,
+ "109": 172.0,
+ "110": 216.0,
+ "111": 182.0,
+ "112": 172.0,
+ "113": 167.0,
+ "114": 192.0,
+ "115": 175.0,
+ "116": 181.0,
+ "117": 177.0,
+ "118": 142.0,
+ "119": 212.0,
+ "120": 164.0,
+ "121": 193.0,
+ "122": 160.0,
+ "123": 169.0,
+ "124": 191.0,
+ "125": 214.0,
+ "126": 160.0,
+ "127": 192.0,
+ "128": 160.0,
+ "129": 180.0,
+ "130": 214.0,
+ "131": 219.0,
+ "132": 173.0,
+ "133": 166.0,
+ "134": 171.0,
+ "135": 182.0,
+ "136": 172.0,
+ "137": 176.0,
+ "138": 174.0,
+ "139": 161.0,
+ "140": 178.0,
+ "141": 164.0,
+ "142": 159.0,
+ "143": 192.0,
+ "144": 157.0,
+ "145": 144.0,
+ "146": 149.0,
+ "147": 148.0,
+ "148": 169.0,
+ "149": 143.0,
+ "150": 111.0,
+ "151": 159.0,
+ "152": 115.0,
+ "153": 147.0,
+ "154": 162.0,
+ "155": 185.0,
+ "156": 144.0,
+ "157": 147.0,
+ "158": 130.0,
+ "159": 165.0,
+ "160": 190.0,
+ "161": 141.0,
+ "162": 155.0,
+ "163": 140.0,
+ "164": 174.0,
+ "165": 168.0,
+ "166": 179.0,
+ "167": 147.0,
+ "168": 138.0,
+ "169": 161.0,
+ "170": 159.0,
+ "171": 125.0,
+ "172": 193.0,
+ "173": 172.0,
+ "174": 190.0,
+ "175": 192.0,
+ "176": 146.0,
+ "177": 168.0,
+ "178": 172.0,
+ "179": 177.0,
+ "180": 148.0,
+ "181": 161.0,
+ "182": 213.0,
+ "183": 215.0,
+ "184": 201.0,
+ "185": 154.0,
+ "186": 207.0,
+ "187": 175.0,
+ "188": 183.0,
+ "189": 169.0,
+ "190": 167.0,
+ "191": 163.0,
+ "192": 193.0,
+ "193": 169.0,
+ "194": 161.0,
+ "195": 141.0,
+ "196": 174.0,
+ "197": 188.0,
+ "198": 168.0,
+ "199": 150.0,
+ "200": 187.0,
+ "201": 173.0,
+ "202": 183.0,
+ "203": 142.0,
+ "204": 177.0,
+ "205": 153.0,
+ "206": 198.0,
+ "207": 168.0,
+ "208": 140.0,
+ "209": 179.0,
+ "210": 175.0,
+ "211": 167.0,
+ "212": 194.0,
+ "213": 192.0,
+ "214": 174.0,
+ "215": 188.0,
+ "216": 164.0,
+ "217": 170.0,
+ "218": 171.0,
+ "219": 211.0,
+ "220": 195.0,
+ "221": 181.0,
+ "222": 154.0,
+ "223": 176.0,
+ "224": 173.0,
+ "225": 166.0,
+ "226": 174.0,
+ "227": 211.0,
+ "228": 146.0,
+ "229": 193.0,
+ "230": 149.0,
+ "231": 177.0,
+ "232": 169.0,
+ "233": 193.0,
+ "234": 183.0,
+ "235": 215.0,
+ "236": 200.0,
+ "237": 218.0,
+ "238": 179.0,
+ "239": 139.0,
+ "240": 217.0,
+ "241": 174.0,
+ "242": 193.0,
+ "243": 192.0,
+ "244": 181.0,
+ "245": 206.0,
+ "246": 221.0,
+ "247": 219.0,
+ "248": 175.0,
+ "249": 189.0,
+ "250": 156.0,
+ "251": 205.0,
+ "252": 164.0,
+ "253": 172.0,
+ "254": 184.0,
+ "255": 218.0,
+ "256": 171.0,
+ "257": 208.0,
+ "258": 210.0,
+ "259": 174.0,
+ "260": 199.0,
+ "261": 178.0,
+ "262": 185.0,
+ "263": 181.0,
+ "264": 200.0,
+ "265": 171.0,
+ "266": 149.0,
+ "267": 141.0,
+ "268": 186.0,
+ "269": 198.0,
+ "270": 170.0,
+ "271": 168.0,
+ "272": 210.0,
+ "273": 151.0,
+ "274": 212.0,
+ "275": 182.0,
+ "276": 172.0,
+ "277": 159.0,
+ "278": 169.0,
+ "279": 185.0,
+ "280": 174.0,
+ "281": 160.0,
+ "282": 171.0,
+ "283": 174.0,
+ "284": 183.0,
+ "285": 169.0,
+ "286": 173.0,
+ "287": 203.0,
+ "288": 168.0,
+ "289": 202.0,
+ "290": 157.0,
+ "291": 241.0,
+ "292": 172.0,
+ "293": 209.0,
+ "294": 194.0,
+ "295": 207.0,
+ "296": 217.0,
+ "297": 160.0,
+ "298": 126.0,
+ "299": 170.0,
+ "300": 177.0,
+ "301": 189.0,
+ "302": 209.0,
+ "303": 170.0,
+ "304": 177.0,
+ "305": 148.0,
+ "306": 172.0,
+ "307": 213.0,
+ "308": 184.0,
+ "309": 193.0,
+ "310": 218.0,
+ "311": 159.0,
+ "312": 178.0,
+ "313": 177.0,
+ "314": 199.0,
+ "315": 165.0,
+ "316": 168.0,
+ "317": 185.0,
+ "318": 261.0,
+ "319": 181.0,
+ "320": 196.0,
+ "321": 200.0,
+ "322": 217.0,
+ "323": 198.0,
+ "324": 200.0,
+ "325": 184.0,
+ "326": 283.0,
+ "327": 211.0,
+ "328": 231.0,
+ "329": 189.0,
+ "330": 248.0,
+ "331": 205.0,
+ "332": 208.0,
+ "333": 199.0,
+ "334": 182.0,
+ "335": 202.0,
+ "336": 207.0,
+ "337": 216.0,
+ "338": 231.0,
+ "339": 213.0,
+ "340": 240.0,
+ "341": 207.0,
+ "342": 153.0,
+ "343": 264.0,
+ "344": 214.0,
+ "345": 202.0,
+ "346": 183.0,
+ "347": 194.0,
+ "348": 216.0,
+ "349": 206.0,
+ "350": 218.0,
+ "351": 218.0,
+ "352": 207.0,
+ "353": 225.0,
+ "354": 213.0,
+ "355": 201.0,
+ "356": 227.0,
+ "357": 217.0,
+ "358": 206.0,
+ "359": 186.0,
+ "360": 217.0,
+ "361": 187.0,
+ "362": 256.0,
+ "363": 226.0,
+ "364": 203.0,
+ "365": 200.0,
+ "366": 241.0,
+ "367": 205.0,
+ "368": 192.0,
+ "369": 160.0,
+ "370": 221.0,
+ "371": 212.0,
+ "372": 193.0,
+ "373": 218.0,
+ "374": 164.0,
+ "375": 249.0,
+ "376": 195.0,
+ "377": 197.0,
+ "378": 222.0,
+ "379": 254.0,
+ "380": 210.0,
+ "381": 199.0,
+ "382": 217.0,
+ "383": 208.0,
+ "384": 238.0,
+ "385": 183.0,
+ "386": 221.0,
+ "387": 185.0,
+ "388": 205.0,
+ "389": 185.0,
+ "390": 217.0,
+ "391": 241.0,
+ "392": 212.0,
+ "393": 247.0,
+ "394": 242.0,
+ "395": 247.0,
+ "396": 197.0,
+ "397": 202.0,
+ "398": 191.0,
+ "399": 231.0,
+ "400": 211.0,
+ "401": 200.0,
+ "402": 210.0,
+ "403": 261.0,
+ "404": 211.0,
+ "405": 171.0,
+ "406": 209.0,
+ "407": 200.0,
+ "408": 226.0,
+ "409": 200.0,
+ "410": 220.0,
+ "411": 196.0,
+ "412": 194.0,
+ "413": 168.0,
+ "414": 223.0,
+ "415": 204.0,
+ "416": 225.0,
+ "417": 213.0,
+ "418": 196.0,
+ "419": 203.0,
+ "420": 203.0,
+ "421": 217.0,
+ "422": 200.0,
+ "423": 213.0,
+ "424": 237.0,
+ "425": 239.0,
+ "426": 178.0,
+ "427": 213.0,
+ "428": 196.0,
+ "429": 174.0,
+ "430": 243.0,
+ "431": 169.0,
+ "432": 203.0,
+ "433": 211.0,
+ "434": 194.0,
+ "435": 188.0,
+ "436": 208.0,
+ "437": 170.0,
+ "438": 194.0,
+ "439": 156.0,
+ "440": 199.0,
+ "441": 190.0,
+ "442": 232.0,
+ "443": 225.0,
+ "444": 172.0,
+ "445": 194.0,
+ "446": 221.0,
+ "447": 209.0,
+ "448": 233.0,
+ "449": 257.0,
+ "450": 207.0,
+ "451": 199.0,
+ "452": 177.0,
+ "453": 200.0,
+ "454": 227.0,
+ "455": 263.0,
+ "456": 196.0,
+ "457": 204.0,
+ "458": 169.0,
+ "459": 131.0,
+ "460": 216.0,
+ "461": 223.0,
+ "462": 210.0,
+ "463": 203.0,
+ "464": 208.0,
+ "465": 187.0,
+ "466": 190.0,
+ "467": 192.0,
+ "468": 194.0,
+ "469": 188.0,
+ "470": 193.0,
+ "471": 221.0,
+ "472": 166.0,
+ "473": 191.0,
+ "474": 193.0,
+ "475": 196.0,
+ "476": 192.0,
+ "477": 168.0,
+ "478": 180.0,
+ "479": 176.0,
+ "480": 145.0,
+ "481": 197.0,
+ "482": 167.0,
+ "483": 198.0,
+ "484": 172.0,
+ "485": 175.0,
+ "486": 192.0,
+ "487": 143.0,
+ "488": 182.0,
+ "489": 172.0,
+ "490": 178.0,
+ "491": 175.0,
+ "492": 194.0,
+ "493": 211.0,
+ "494": 159.0,
+ "495": 165.0,
+ "496": 153.0,
+ "497": 145.0,
+ "498": 196.0,
+ "499": 195.0,
+ "500": 165.0,
+ "501": 183.0,
+ "502": 167.0,
+ "503": 175.0,
+ "504": 182.0,
+ "505": 212.0,
+ "506": 177.0,
+ "507": 159.0,
+ "508": 135.0,
+ "509": 195.0,
+ "510": 156.0,
+ "511": 186.0,
+ "512": 177.0,
+ "513": 186.0,
+ "514": 173.0,
+ "515": 190.0,
+ "516": 175.0,
+ "517": 143.0,
+ "518": 169.0,
+ "519": 186.0,
+ "520": 156.0,
+ "521": 146.0,
+ "522": 173.0,
+ "523": 175.0,
+ "524": 172.0,
+ "525": 202.0,
+ "526": 168.0,
+ "527": 178.0,
+ "528": 173.0,
+ "529": 183.0,
+ "530": 168.0,
+ "531": 161.0,
+ "532": 185.0,
+ "533": 172.0,
+ "534": 166.0,
+ "535": 140.0,
+ "536": 164.0,
+ "537": 150.0,
+ "538": 155.0,
+ "539": 125.0,
+ "540": 151.0,
+ "541": 130.0,
+ "542": 153.0,
+ "543": 149.0,
+ "544": 185.0,
+ "545": 132.0,
+ "546": 184.0,
+ "547": 150.0,
+ "548": 155.0,
+ "549": 162.0,
+ "550": 170.0,
+ "551": 144.0,
+ "552": 147.0,
+ "553": 213.0,
+ "554": 182.0,
+ "555": 150.0,
+ "556": 162.0,
+ "557": 154.0,
+ "558": 181.0,
+ "559": 144.0,
+ "560": 194.0,
+ "561": 174.0,
+ "562": 147.0,
+ "563": 125.0,
+ "564": 169.0,
+ "565": 143.0,
+ "566": 136.0,
+ "567": 144.0,
+ "568": 153.0,
+ "569": 167.0,
+ "570": 153.0,
+ "571": 131.0,
+ "572": 143.0,
+ "573": 128.0,
+ "574": 162.0,
+ "575": 133.0,
+ "576": 143.0,
+ "577": 171.0,
+ "578": 167.0,
+ "579": 140.0,
+ "580": 165.0,
+ "581": 164.0,
+ "582": 145.0,
+ "583": 151.0,
+ "584": 146.0,
+ "585": 148.0,
+ "586": 102.0,
+ "587": 147.0,
+ "588": 146.0,
+ "589": 123.0,
+ "590": 146.0,
+ "591": 149.0,
+ "592": 115.0,
+ "593": 166.0,
+ "594": 159.0,
+ "595": 127.0,
+ "596": 113.0,
+ "597": 135.0,
+ "598": 139.0,
+ "599": 157.0,
+ "600": 129.0,
+ "601": 144.0,
+ "602": 129.0,
+ "603": 125.0,
+ "604": 125.0,
+ "605": 139.0,
+ "606": 135.0,
+ "607": 144.0,
+ "608": 149.0,
+ "609": 139.0,
+ "610": 135.0,
+ "611": 148.0,
+ "612": 148.0,
+ "613": 115.0,
+ "614": 150.0,
+ "615": 132.0,
+ "616": 156.0,
+ "617": 120.0,
+ "618": 145.0,
+ "619": 136.0,
+ "620": 170.0,
+ "621": 147.0,
+ "622": 150.0,
+ "623": 119.0,
+ "624": 128.0,
+ "625": 141.0,
+ "626": 122.0,
+ "627": 121.0,
+ "628": 157.0,
+ "629": 126.0,
+ "630": 134.0,
+ "631": 147.0,
+ "632": 146.0,
+ "633": 131.0,
+ "634": 145.0,
+ "635": 174.0,
+ "636": 151.0,
+ "637": 169.0,
+ "638": 128.0,
+ "639": 164.0,
+ "640": 145.0,
+ "641": 136.0,
+ "642": 132.0,
+ "643": 134.0,
+ "644": 124.0,
+ "645": 145.0,
+ "646": 106.0,
+ "647": 123.0,
+ "648": 121.0,
+ "649": 134.0,
+ "650": 153.0,
+ "651": 117.0,
+ "652": 163.0,
+ "653": 155.0,
+ "654": 140.0,
+ "655": 154.0,
+ "656": 124.0,
+ "657": 116.0,
+ "658": 130.0,
+ "659": 114.0,
+ "660": 145.0,
+ "661": 121.0,
+ "662": 143.0,
+ "663": 124.0,
+ "664": 139.0,
+ "665": 138.0,
+ "666": 111.0,
+ "667": 127.0,
+ "668": 144.0,
+ "669": 116.0,
+ "670": 139.0,
+ "671": 132.0,
+ "672": 136.0,
+ "673": 139.0,
+ "674": 119.0,
+ "675": 165.0,
+ "676": 123.0,
+ "677": 127.0,
+ "678": 135.0,
+ "679": 83.0,
+ "680": 139.0,
+ "681": 120.0,
+ "682": 111.0,
+ "683": 119.0,
+ "684": 121.0,
+ "685": 145.0,
+ "686": 127.0,
+ "687": 145.0,
+ "688": 117.0,
+ "689": 119.0,
+ "690": 119.0,
+ "691": 124.0,
+ "692": 118.0,
+ "693": 112.0,
+ "694": 156.0,
+ "695": 114.0,
+ "696": 141.0,
+ "697": 123.0,
+ "698": 130.0,
+ "699": 147.0,
+ "700": 119.0,
+ "701": 139.0,
+ "702": 111.0,
+ "703": 113.0,
+ "704": 118.0,
+ "705": 115.0,
+ "706": 102.0,
+ "707": 121.0,
+ "708": 115.0,
+ "709": 116.0,
+ "710": 95.0,
+ "711": 101.0,
+ "712": 98.0,
+ "713": 117.0,
+ "714": 127.0,
+ "715": 135.0,
+ "716": 124.0,
+ "717": 88.0,
+ "718": 143.0,
+ "719": 114.0,
+ "720": 120.0,
+ "721": 106.0,
+ "722": 117.0,
+ "723": 101.0,
+ "724": 97.0,
+ "725": 106.0,
+ "726": 103.0,
+ "727": 95.0,
+ "728": 123.0,
+ "729": 104.0,
+ "730": 124.0,
+ "731": 111.0,
+ "732": 78.0,
+ "733": 96.0,
+ "734": 129.0,
+ "735": 142.0,
+ "736": 110.0,
+ "737": 132.0,
+ "738": 110.0,
+ "739": 136.0,
+ "740": 106.0,
+ "741": 102.0,
+ "742": 123.0,
+ "743": 133.0,
+ "744": 130.0,
+ "745": 109.0,
+ "746": 122.0,
+ "747": 125.0,
+ "748": 133.0,
+ "749": 114.0,
+ "750": 121.0,
+ "751": 113.0,
+ "752": 111.0,
+ "753": 96.0,
+ "754": 118.0,
+ "755": 87.0,
+ "756": 113.0,
+ "757": 91.0,
+ "758": 105.0,
+ "759": 99.0,
+ "760": 125.0,
+ "761": 106.0,
+ "762": 105.0,
+ "763": 101.0,
+ "764": 109.0,
+ "765": 118.0,
+ "766": 95.0,
+ "767": 133.0,
+ "768": 115.0,
+ "769": 122.0,
+ "770": 106.0,
+ "771": 123.0,
+ "772": 106.0,
+ "773": 136.0,
+ "774": 128.0,
+ "775": 116.0,
+ "776": 112.0,
+ "777": 95.0,
+ "778": 113.0,
+ "779": 119.0,
+ "780": 99.0,
+ "781": 107.0,
+ "782": 80.0,
+ "783": 108.0,
+ "784": 122.0,
+ "785": 111.0,
+ "786": 111.0,
+ "787": 115.0,
+ "788": 116.0,
+ "789": 108.0,
+ "790": 127.0,
+ "791": 83.0,
+ "792": 117.0,
+ "793": 102.0,
+ "794": 106.0,
+ "795": 123.0,
+ "796": 121.0,
+ "797": 124.0,
+ "798": 112.0,
+ "799": 136.0,
+ "800": 99.0,
+ "801": 117.0,
+ "802": 93.0,
+ "803": 166.0,
+ "804": 127.0,
+ "805": 124.0,
+ "806": 97.0,
+ "807": 134.0,
+ "808": 108.0,
+ "809": 121.0,
+ "810": 126.0,
+ "811": 107.0,
+ "812": 116.0,
+ "813": 126.0,
+ "814": 105.0,
+ "815": 98.0,
+ "816": 99.0,
+ "817": 97.0,
+ "818": 97.0,
+ "819": 109.0,
+ "820": 106.0,
+ "821": 88.0,
+ "822": 109.0,
+ "823": 108.0,
+ "824": 127.0,
+ "825": 108.0,
+ "826": 128.0,
+ "827": 134.0,
+ "828": 100.0,
+ "829": 125.0,
+ "830": 113.0,
+ "831": 114.0,
+ "832": 107.0,
+ "833": 113.0,
+ "834": 100.0,
+ "835": 98.0,
+ "836": 123.0,
+ "837": 95.0,
+ "838": 118.0,
+ "839": 96.0,
+ "840": 109.0,
+ "841": 98.0,
+ "842": 114.0,
+ "843": 113.0,
+ "844": 123.0,
+ "845": 108.0,
+ "846": 124.0,
+ "847": 112.0,
+ "848": 115.0,
+ "849": 118.0,
+ "850": 92.0,
+ "851": 145.0,
+ "852": 89.0,
+ "853": 106.0,
+ "854": 101.0,
+ "855": 113.0,
+ "856": 125.0,
+ "857": 105.0,
+ "858": 129.0,
+ "859": 107.0,
+ "860": 118.0,
+ "861": 85.0,
+ "862": 106.0,
+ "863": 95.0,
+ "864": 81.0,
+ "865": 104.0,
+ "866": 105.0,
+ "867": 104.0,
+ "868": 106.0,
+ "869": 109.0,
+ "870": 105.0,
+ "871": 122.0,
+ "872": 114.0,
+ "873": 100.0,
+ "874": 113.0,
+ "875": 108.0,
+ "876": 93.0,
+ "877": 130.0,
+ "878": 110.0,
+ "879": 122.0,
+ "880": 106.0,
+ "881": 103.0,
+ "882": 80.0,
+ "883": 107.0,
+ "884": 115.0,
+ "885": 113.0,
+ "886": 116.0,
+ "887": 131.0,
+ "888": 89.0,
+ "889": 120.0,
+ "890": 110.0,
+ "891": 103.0,
+ "892": 102.0,
+ "893": 106.0,
+ "894": 91.0,
+ "895": 118.0,
+ "896": 110.0,
+ "897": 103.0,
+ "898": 115.0,
+ "899": 119.0,
+ "900": 120.0,
+ "901": 99.0,
+ "902": 100.0,
+ "903": 102.0,
+ "904": 127.0,
+ "905": 105.0,
+ "906": 124.0,
+ "907": 104.0,
+ "908": 117.0,
+ "909": 124.0,
+ "910": 108.0,
+ "911": 102.0,
+ "912": 117.0,
+ "913": 122.0,
+ "914": 130.0,
+ "915": 98.0,
+ "916": 120.0,
+ "917": 113.0,
+ "918": 112.0,
+ "919": 85.0,
+ "920": 110.0,
+ "921": 108.0,
+ "922": 111.0,
+ "923": 116.0,
+ "924": 119.0,
+ "925": 105.0,
+ "926": 128.0,
+ "927": 120.0,
+ "928": 106.0,
+ "929": 94.0,
+ "930": 116.0,
+ "931": 102.0,
+ "932": 123.0,
+ "933": 114.0,
+ "934": 133.0,
+ "935": 86.0,
+ "936": 114.0,
+ "937": 96.0,
+ "938": 118.0,
+ "939": 111.0,
+ "940": 110.0,
+ "941": 102.0,
+ "942": 98.0,
+ "943": 119.0,
+ "944": 107.0,
+ "945": 106.0,
+ "946": 112.0,
+ "947": 93.0,
+ "948": 119.0,
+ "949": 116.0,
+ "950": 124.0,
+ "951": 112.0,
+ "952": 106.0,
+ "953": 97.0,
+ "954": 111.0,
+ "955": 112.0,
+ "956": 87.0,
+ "957": 117.0,
+ "958": 97.0,
+ "959": 91.0,
+ "960": 103.0,
+ "961": 102.0,
+ "962": 103.0,
+ "963": 127.0,
+ "964": 113.0,
+ "965": 120.0,
+ "966": 106.0,
+ "967": 104.0,
+ "968": 119.0,
+ "969": 89.0,
+ "970": 121.0,
+ "971": 115.0,
+ "972": 96.0,
+ "973": 90.0,
+ "974": 113.0,
+ "975": 109.0,
+ "976": 113.0,
+ "977": 85.0,
+ "978": 104.0,
+ "979": 109.0,
+ "980": 100.0,
+ "981": 94.0,
+ "982": 105.0,
+ "983": 84.0,
+ "984": 112.0,
+ "985": 108.0,
+ "986": 92.0,
+ "987": 88.0,
+ "988": 123.0,
+ "989": 106.0,
+ "990": 103.0,
+ "991": 128.0,
+ "992": 104.0,
+ "993": 109.0,
+ "994": 98.0,
+ "995": 104.0,
+ "996": 93.0,
+ "997": 128.0,
+ "998": 121.0,
+ "999": 89.0,
+ "1000": 118.0,
+ "1001": 104.0,
+ "1002": 96.0,
+ "1003": 107.0,
+ "1004": 88.0,
+ "1005": 103.0,
+ "1006": 105.0,
+ "1007": 102.0,
+ "1008": 83.0,
+ "1009": 117.0,
+ "1010": 104.0,
+ "1011": 127.0,
+ "1012": 117.0,
+ "1013": 106.0,
+ "1014": 111.0,
+ "1015": 110.0,
+ "1016": 91.0,
+ "1017": 76.0,
+ "1018": 115.0,
+ "1019": 123.0,
+ "1020": 111.0,
+ "1021": 106.0,
+ "1022": 108.0,
+ "1023": 137.0,
+ "1024": 122.0,
+ "1025": 104.0,
+ "1026": 109.0,
+ "1027": 92.0,
+ "1028": 96.0,
+ "1029": 116.0,
+ "1030": 96.0,
+ "1031": 122.0,
+ "1032": 103.0,
+ "1033": 108.0,
+ "1034": 111.0,
+ "1035": 86.0,
+ "1036": 74.0,
+ "1037": 123.0,
+ "1038": 85.0,
+ "1039": 128.0,
+ "1040": 95.0,
+ "1041": 116.0,
+ "1042": 107.0,
+ "1043": 96.0,
+ "1044": 116.0,
+ "1045": 115.0,
+ "1046": 92.0,
+ "1047": 106.0,
+ "1048": 88.0,
+ "1049": 121.0,
+ "1050": 117.0,
+ "1051": 105.0,
+ "1052": 96.0,
+ "1053": 98.0,
+ "1054": 85.0,
+ "1055": 110.0,
+ "1056": 91.0,
+ "1057": 109.0,
+ "1058": 95.0,
+ "1059": 106.0,
+ "1060": 109.0,
+ "1061": 97.0,
+ "1062": 105.0,
+ "1063": 91.0,
+ "1064": 103.0,
+ "1065": 108.0,
+ "1066": 112.0,
+ "1067": 108.0,
+ "1068": 108.0,
+ "1069": 123.0,
+ "1070": 100.0,
+ "1071": 95.0,
+ "1072": 111.0,
+ "1073": 118.0,
+ "1074": 101.0,
+ "1075": 95.0,
+ "1076": 111.0,
+ "1077": 89.0,
+ "1078": 94.0,
+ "1079": 113.0,
+ "1080": 82.0,
+ "1081": 114.0,
+ "1082": 87.0,
+ "1083": 116.0,
+ "1084": 105.0,
+ "1085": 97.0,
+ "1086": 119.0,
+ "1087": 86.0,
+ "1088": 93.0,
+ "1089": 114.0,
+ "1090": 87.0,
+ "1091": 109.0,
+ "1092": 90.0,
+ "1093": 109.0,
+ "1094": 101.0,
+ "1095": 90.0,
+ "1096": 106.0,
+ "1097": 100.0,
+ "1098": 105.0,
+ "1099": 96.0,
+ "1100": 92.0,
+ "1101": 108.0,
+ "1102": 94.0,
+ "1103": 86.0,
+ "1104": 103.0,
+ "1105": 109.0,
+ "1106": 87.0,
+ "1107": 87.0,
+ "1108": 96.0,
+ "1109": 102.0,
+ "1110": 89.0,
+ "1111": 76.0,
+ "1112": 110.0,
+ "1113": 104.0,
+ "1114": 89.0,
+ "1115": 114.0,
+ "1116": 97.0,
+ "1117": 108.0,
+ "1118": 107.0,
+ "1119": 118.0,
+ "1120": 112.0,
+ "1121": 96.0,
+ "1122": 103.0,
+ "1123": 112.0,
+ "1124": 98.0,
+ "1125": 97.0,
+ "1126": 121.0,
+ "1127": 80.0,
+ "1128": 91.0,
+ "1129": 106.0,
+ "1130": 96.0,
+ "1131": 82.0,
+ "1132": 103.0,
+ "1133": 86.0,
+ "1134": 92.0,
+ "1135": 98.0,
+ "1136": 90.0,
+ "1137": 120.0,
+ "1138": 102.0,
+ "1139": 109.0,
+ "1140": 88.0,
+ "1141": 90.0,
+ "1142": 95.0,
+ "1143": 88.0,
+ "1144": 77.0,
+ "1145": 92.0,
+ "1146": 85.0,
+ "1147": 108.0,
+ "1148": 77.0,
+ "1149": 93.0,
+ "1150": 101.0,
+ "1151": 116.0,
+ "1152": 72.0,
+ "1153": 90.0,
+ "1154": 103.0,
+ "1155": 106.0,
+ "1156": 91.0,
+ "1157": 100.0,
+ "1158": 101.0,
+ "1159": 111.0,
+ "1160": 114.0,
+ "1161": 90.0,
+ "1162": 92.0,
+ "1163": 90.0,
+ "1164": 96.0,
+ "1165": 100.0,
+ "1166": 114.0,
+ "1167": 82.0,
+ "1168": 96.0,
+ "1169": 77.0,
+ "1170": 91.0,
+ "1171": 94.0,
+ "1172": 99.0,
+ "1173": 124.0,
+ "1174": 106.0,
+ "1175": 97.0,
+ "1176": 102.0,
+ "1177": 78.0,
+ "1178": 108.0,
+ "1179": 103.0,
+ "1180": 84.0,
+ "1181": 76.0,
+ "1182": 115.0,
+ "1183": 104.0,
+ "1184": 122.0,
+ "1185": 104.0,
+ "1186": 104.0,
+ "1187": 91.0,
+ "1188": 112.0,
+ "1189": 101.0,
+ "1190": 106.0,
+ "1191": 97.0,
+ "1192": 90.0,
+ "1193": 105.0,
+ "1194": 99.0,
+ "1195": 118.0,
+ "1196": 120.0,
+ "1197": 93.0,
+ "1198": 101.0,
+ "1199": 103.0,
+ "1200": 90.0,
+ "1201": 108.0,
+ "1202": 120.0,
+ "1203": 90.0,
+ "1204": 98.0,
+ "1205": 113.0,
+ "1206": 102.0,
+ "1207": 116.0,
+ "1208": 104.0,
+ "1209": 85.0,
+ "1210": 101.0,
+ "1211": 87.0,
+ "1212": 100.0,
+ "1213": 109.0,
+ "1214": 92.0,
+ "1215": 103.0,
+ "1216": 117.0,
+ "1217": 102.0,
+ "1218": 135.0,
+ "1219": 95.0,
+ "1220": 122.0,
+ "1221": 121.0,
+ "1222": 109.0,
+ "1223": 103.0,
+ "1224": 93.0,
+ "1225": 107.0,
+ "1226": 82.0,
+ "1227": 108.0,
+ "1228": 106.0,
+ "1229": 87.0,
+ "1230": 97.0,
+ "1231": 109.0,
+ "1232": 95.0,
+ "1233": 99.0,
+ "1234": 107.0,
+ "1235": 105.0,
+ "1236": 101.0,
+ "1237": 110.0,
+ "1238": 102.0,
+ "1239": 118.0,
+ "1240": 114.0,
+ "1241": 119.0,
+ "1242": 90.0,
+ "1243": 104.0,
+ "1244": 102.0,
+ "1245": 105.0,
+ "1246": 104.0,
+ "1247": 121.0,
+ "1248": 104.0,
+ "1249": 129.0,
+ "1250": 111.0,
+ "1251": 91.0,
+ "1252": 120.0,
+ "1253": 121.0,
+ "1254": 110.0,
+ "1255": 113.0,
+ "1256": 97.0,
+ "1257": 114.0,
+ "1258": 110.0,
+ "1259": 106.0,
+ "1260": 93.0,
+ "1261": 104.0,
+ "1262": 109.0,
+ "1263": 104.0,
+ "1264": 101.0,
+ "1265": 85.0,
+ "1266": 106.0,
+ "1267": 104.0,
+ "1268": 90.0,
+ "1269": 102.0,
+ "1270": 106.0,
+ "1271": 107.0,
+ "1272": 79.0,
+ "1273": 85.0,
+ "1274": 99.0,
+ "1275": 127.0,
+ "1276": 89.0,
+ "1277": 144.0,
+ "1278": 109.0,
+ "1279": 110.0,
+ "1280": 123.0,
+ "1281": 98.0,
+ "1282": 94.0,
+ "1283": 110.0,
+ "1284": 88.0,
+ "1285": 112.0,
+ "1286": 106.0,
+ "1287": 86.0,
+ "1288": 100.0,
+ "1289": 118.0,
+ "1290": 109.0,
+ "1291": 82.0,
+ "1292": 106.0,
+ "1293": 97.0,
+ "1294": 96.0,
+ "1295": 91.0,
+ "1296": 110.0,
+ "1297": 120.0,
+ "1298": 105.0,
+ "1299": 114.0,
+ "1300": 113.0,
+ "1301": 106.0,
+ "1302": 112.0,
+ "1303": 102.0,
+ "1304": 94.0,
+ "1305": 109.0,
+ "1306": 83.0,
+ "1307": 97.0,
+ "1308": 120.0,
+ "1309": 126.0,
+ "1310": 103.0,
+ "1311": 126.0,
+ "1312": 100.0,
+ "1313": 101.0,
+ "1314": 107.0,
+ "1315": 117.0,
+ "1316": 101.0,
+ "1317": 107.0,
+ "1318": 103.0,
+ "1319": 98.0,
+ "1320": 103.0,
+ "1321": 112.0,
+ "1322": 86.0,
+ "1323": 117.0,
+ "1324": 94.0,
+ "1325": 94.0,
+ "1326": 139.0,
+ "1327": 82.0,
+ "1328": 124.0,
+ "1329": 103.0,
+ "1330": 91.0,
+ "1331": 94.0,
+ "1332": 106.0,
+ "1333": 86.0,
+ "1334": 86.0,
+ "1335": 96.0,
+ "1336": 113.0,
+ "1337": 114.0,
+ "1338": 126.0,
+ "1339": 104.0,
+ "1340": 101.0,
+ "1341": 83.0,
+ "1342": 106.0,
+ "1343": 122.0,
+ "1344": 99.0,
+ "1345": 93.0,
+ "1346": 110.0,
+ "1347": 105.0,
+ "1348": 104.0,
+ "1349": 103.0,
+ "1350": 111.0,
+ "1351": 121.0,
+ "1352": 106.0,
+ "1353": 108.0,
+ "1354": 108.0,
+ "1355": 92.0,
+ "1356": 89.0,
+ "1357": 103.0,
+ "1358": 120.0,
+ "1359": 110.0,
+ "1360": 125.0,
+ "1361": 116.0,
+ "1362": 133.0,
+ "1363": 103.0,
+ "1364": 109.0,
+ "1365": 101.0,
+ "1366": 100.0,
+ "1367": 93.0,
+ "1368": 108.0,
+ "1369": 127.0,
+ "1370": 99.0,
+ "1371": 121.0,
+ "1372": 116.0,
+ "1373": 110.0,
+ "1374": 94.0,
+ "1375": 107.0,
+ "1376": 104.0,
+ "1377": 115.0,
+ "1378": 100.0,
+ "1379": 106.0,
+ "1380": 88.0,
+ "1381": 103.0,
+ "1382": 101.0,
+ "1383": 118.0,
+ "1384": 120.0,
+ "1385": 117.0,
+ "1386": 123.0,
+ "1387": 93.0,
+ "1388": 86.0,
+ "1389": 119.0,
+ "1390": 116.0,
+ "1391": 103.0,
+ "1392": 84.0,
+ "1393": 100.0,
+ "1394": 112.0,
+ "1395": 77.0,
+ "1396": 101.0,
+ "1397": 124.0,
+ "1398": 104.0,
+ "1399": 120.0,
+ "1400": 103.0,
+ "1401": 100.0,
+ "1402": 105.0,
+ "1403": 82.0,
+ "1404": 104.0,
+ "1405": 93.0,
+ "1406": 102.0,
+ "1407": 118.0,
+ "1408": 100.0,
+ "1409": 114.0,
+ "1410": 85.0,
+ "1411": 101.0,
+ "1412": 99.0,
+ "1413": 117.0,
+ "1414": 116.0,
+ "1415": 115.0,
+ "1416": 90.0,
+ "1417": 99.0,
+ "1418": 97.0,
+ "1419": 96.0,
+ "1420": 119.0,
+ "1421": 108.0,
+ "1422": 113.0,
+ "1423": 91.0,
+ "1424": 123.0,
+ "1425": 101.0,
+ "1426": 110.0,
+ "1427": 107.0,
+ "1428": 116.0,
+ "1429": 128.0,
+ "1430": 87.0,
+ "1431": 96.0,
+ "1432": 113.0,
+ "1433": 92.0,
+ "1434": 101.0,
+ "1435": 101.0,
+ "1436": 111.0,
+ "1437": 122.0,
+ "1438": 105.0,
+ "1439": 99.0,
+ "1440": 101.0,
+ "1441": 104.0,
+ "1442": 89.0,
+ "1443": 109.0,
+ "1444": 86.0,
+ "1445": 100.0,
+ "1446": 87.0,
+ "1447": 105.0,
+ "1448": 102.0,
+ "1449": 88.0,
+ "1450": 100.0,
+ "1451": 94.0,
+ "1452": 95.0,
+ "1453": 116.0,
+ "1454": 98.0,
+ "1455": 92.0,
+ "1456": 91.0,
+ "1457": 132.0,
+ "1458": 121.0,
+ "1459": 109.0,
+ "1460": 111.0,
+ "1461": 111.0,
+ "1462": 89.0,
+ "1463": 99.0,
+ "1464": 108.0,
+ "1465": 97.0,
+ "1466": 87.0,
+ "1467": 99.0,
+ "1468": 127.0,
+ "1469": 88.0,
+ "1470": 103.0,
+ "1471": 101.0,
+ "1472": 106.0,
+ "1473": 120.0,
+ "1474": 96.0,
+ "1475": 123.0,
+ "1476": 85.0,
+ "1477": 122.0,
+ "1478": 107.0,
+ "1479": 113.0,
+ "1480": 109.0,
+ "1481": 107.0,
+ "1482": 118.0,
+ "1483": 86.0,
+ "1484": 98.0,
+ "1485": 91.0,
+ "1486": 96.0,
+ "1487": 119.0,
+ "1488": 106.0,
+ "1489": 93.0,
+ "1490": 113.0,
+ "1491": 107.0,
+ "1492": 100.0,
+ "1493": 123.0,
+ "1494": 105.0,
+ "1495": 121.0,
+ "1496": 105.0,
+ "1497": 99.0,
+ "1498": 112.0,
+ "1499": 106.0,
+ "1500": 104.0,
+ "1501": 129.0,
+ "1502": 109.0,
+ "1503": 91.0,
+ "1504": 111.0,
+ "1505": 97.0,
+ "1506": 116.0,
+ "1507": 122.0,
+ "1508": 103.0,
+ "1509": 141.0,
+ "1510": 86.0,
+ "1511": 120.0,
+ "1512": 120.0,
+ "1513": 128.0,
+ "1514": 100.0,
+ "1515": 108.0,
+ "1516": 99.0,
+ "1517": 109.0,
+ "1518": 106.0,
+ "1519": 88.0,
+ "1520": 89.0,
+ "1521": 101.0,
+ "1522": 112.0,
+ "1523": 88.0,
+ "1524": 113.0,
+ "1525": 94.0,
+ "1526": 110.0,
+ "1527": 112.0,
+ "1528": 84.0,
+ "1529": 91.0,
+ "1530": 114.0,
+ "1531": 113.0,
+ "1532": 119.0,
+ "1533": 95.0,
+ "1534": 112.0,
+ "1535": 112.0,
+ "1536": 109.0,
+ "1537": 97.0,
+ "1538": 111.0,
+ "1539": 115.0,
+ "1540": 114.0,
+ "1541": 88.0,
+ "1542": 126.0,
+ "1543": 97.0,
+ "1544": 84.0,
+ "1545": 105.0,
+ "1546": 82.0,
+ "1547": 93.0,
+ "1548": 90.0,
+ "1549": 99.0,
+ "1550": 93.0,
+ "1551": 98.0,
+ "1552": 86.0,
+ "1553": 120.0,
+ "1554": 109.0,
+ "1555": 111.0,
+ "1556": 98.0,
+ "1557": 90.0,
+ "1558": 120.0,
+ "1559": 84.0,
+ "1560": 107.0,
+ "1561": 103.0,
+ "1562": 121.0,
+ "1563": 116.0,
+ "1564": 113.0,
+ "1565": 114.0,
+ "1566": 113.0,
+ "1567": 102.0,
+ "1568": 91.0,
+ "1569": 122.0,
+ "1570": 95.0,
+ "1571": 115.0,
+ "1572": 102.0,
+ "1573": 100.0,
+ "1574": 121.0,
+ "1575": 108.0,
+ "1576": 88.0,
+ "1577": 116.0,
+ "1578": 101.0,
+ "1579": 98.0,
+ "1580": 114.0,
+ "1581": 102.0,
+ "1582": 108.0,
+ "1583": 115.0,
+ "1584": 70.0,
+ "1585": 112.0,
+ "1586": 120.0,
+ "1587": 101.0,
+ "1588": 118.0,
+ "1589": 99.0,
+ "1590": 103.0,
+ "1591": 108.0,
+ "1592": 106.0,
+ "1593": 121.0,
+ "1594": 110.0,
+ "1595": 103.0,
+ "1596": 117.0,
+ "1597": 115.0,
+ "1598": 105.0,
+ "1599": 76.0,
+ "1600": 90.0,
+ "1601": 108.0,
+ "1602": 105.0,
+ "1603": 122.0,
+ "1604": 113.0,
+ "1605": 122.0,
+ "1606": 117.0,
+ "1607": 92.0,
+ "1608": 118.0,
+ "1609": 115.0,
+ "1610": 103.0,
+ "1611": 117.0,
+ "1612": 106.0,
+ "1613": 106.0,
+ "1614": 104.0,
+ "1615": 114.0,
+ "1616": 88.0,
+ "1617": 97.0,
+ "1618": 111.0,
+ "1619": 107.0,
+ "1620": 112.0,
+ "1621": 91.0,
+ "1622": 130.0,
+ "1623": 109.0,
+ "1624": 102.0,
+ "1625": 121.0,
+ "1626": 100.0,
+ "1627": 119.0,
+ "1628": 99.0,
+ "1629": 119.0,
+ "1630": 117.0,
+ "1631": 105.0,
+ "1632": 116.0,
+ "1633": 112.0,
+ "1634": 120.0,
+ "1635": 99.0,
+ "1636": 105.0,
+ "1637": 94.0,
+ "1638": 107.0,
+ "1639": 97.0,
+ "1640": 106.0,
+ "1641": 120.0,
+ "1642": 101.0,
+ "1643": 135.0,
+ "1644": 117.0,
+ "1645": 110.0,
+ "1646": 106.0,
+ "1647": 127.0,
+ "1648": 82.0,
+ "1649": 114.0,
+ "1650": 121.0,
+ "1651": 107.0,
+ "1652": 100.0,
+ "1653": 108.0,
+ "1654": 114.0,
+ "1655": 92.0,
+ "1656": 80.0,
+ "1657": 110.0,
+ "1658": 114.0,
+ "1659": 105.0,
+ "1660": 104.0,
+ "1661": 102.0,
+ "1662": 124.0,
+ "1663": 96.0,
+ "1664": 127.0,
+ "1665": 89.0,
+ "1666": 115.0,
+ "1667": 114.0,
+ "1668": 122.0,
+ "1669": 94.0,
+ "1670": 114.0,
+ "1671": 102.0,
+ "1672": 99.0,
+ "1673": 109.0,
+ "1674": 117.0,
+ "1675": 105.0,
+ "1676": 116.0,
+ "1677": 101.0,
+ "1678": 110.0,
+ "1679": 112.0,
+ "1680": 96.0,
+ "1681": 93.0,
+ "1682": 97.0,
+ "1683": 106.0,
+ "1684": 103.0,
+ "1685": 101.0,
+ "1686": 109.0,
+ "1687": 104.0,
+ "1688": 127.0,
+ "1689": 88.0,
+ "1690": 98.0,
+ "1691": 90.0,
+ "1692": 107.0,
+ "1693": 111.0,
+ "1694": 125.0,
+ "1695": 129.0,
+ "1696": 112.0,
+ "1697": 126.0,
+ "1698": 104.0,
+ "1699": 124.0,
+ "1700": 112.0,
+ "1701": 120.0,
+ "1702": 89.0,
+ "1703": 103.0,
+ "1704": 103.0,
+ "1705": 111.0,
+ "1706": 124.0,
+ "1707": 93.0,
+ "1708": 96.0,
+ "1709": 116.0,
+ "1710": 133.0,
+ "1711": 107.0,
+ "1712": 100.0,
+ "1713": 91.0,
+ "1714": 122.0,
+ "1715": 108.0,
+ "1716": 110.0,
+ "1717": 121.0,
+ "1718": 101.0,
+ "1719": 110.0,
+ "1720": 121.0,
+ "1721": 109.0,
+ "1722": 96.0,
+ "1723": 125.0,
+ "1724": 118.0,
+ "1725": 122.0,
+ "1726": 113.0,
+ "1727": 99.0,
+ "1728": 98.0,
+ "1729": 115.0,
+ "1730": 106.0,
+ "1731": 96.0,
+ "1732": 95.0,
+ "1733": 115.0,
+ "1734": 106.0,
+ "1735": 102.0,
+ "1736": 104.0,
+ "1737": 122.0,
+ "1738": 94.0,
+ "1739": 92.0,
+ "1740": 105.0,
+ "1741": 113.0,
+ "1742": 129.0,
+ "1743": 113.0,
+ "1744": 110.0,
+ "1745": 113.0,
+ "1746": 127.0,
+ "1747": 108.0,
+ "1748": 120.0,
+ "1749": 115.0,
+ "1750": 104.0,
+ "1751": 114.0,
+ "1752": 122.0,
+ "1753": 113.0,
+ "1754": 123.0,
+ "1755": 114.0,
+ "1756": 115.0,
+ "1757": 126.0,
+ "1758": 105.0,
+ "1759": 109.0,
+ "1760": 136.0,
+ "1761": 111.0,
+ "1762": 104.0,
+ "1763": 104.0,
+ "1764": 105.0,
+ "1765": 133.0,
+ "1766": 118.0,
+ "1767": 108.0,
+ "1768": 114.0,
+ "1769": 105.0,
+ "1770": 98.0,
+ "1771": 112.0,
+ "1772": 92.0,
+ "1773": 77.0,
+ "1774": 130.0,
+ "1775": 104.0,
+ "1776": 85.0,
+ "1777": 106.0,
+ "1778": 84.0,
+ "1779": 111.0,
+ "1780": 109.0,
+ "1781": 124.0,
+ "1782": 109.0,
+ "1783": 128.0,
+ "1784": 117.0,
+ "1785": 118.0,
+ "1786": 111.0,
+ "1787": 112.0,
+ "1788": 104.0,
+ "1789": 135.0,
+ "1790": 105.0,
+ "1791": 115.0,
+ "1792": 130.0,
+ "1793": 119.0,
+ "1794": 128.0,
+ "1795": 110.0,
+ "1796": 130.0,
+ "1797": 97.0,
+ "1798": 139.0,
+ "1799": 104.0,
+ "1800": 103.0,
+ "1801": 94.0,
+ "1802": 134.0,
+ "1803": 117.0,
+ "1804": 139.0,
+ "1805": 124.0,
+ "1806": 127.0,
+ "1807": 128.0,
+ "1808": 99.0,
+ "1809": 92.0,
+ "1810": 116.0,
+ "1811": 104.0,
+ "1812": 103.0,
+ "1813": 122.0,
+ "1814": 129.0,
+ "1815": 94.0,
+ "1816": 104.0,
+ "1817": 98.0,
+ "1818": 128.0,
+ "1819": 112.0,
+ "1820": 99.0,
+ "1821": 126.0,
+ "1822": 83.0,
+ "1823": 117.0,
+ "1824": 96.0,
+ "1825": 95.0,
+ "1826": 127.0,
+ "1827": 124.0,
+ "1828": 120.0,
+ "1829": 110.0,
+ "1830": 123.0,
+ "1831": 110.0,
+ "1832": 92.0,
+ "1833": 100.0,
+ "1834": 113.0,
+ "1835": 120.0,
+ "1836": 113.0,
+ "1837": 114.0,
+ "1838": 99.0,
+ "1839": 123.0,
+ "1840": 109.0,
+ "1841": 95.0,
+ "1842": 101.0,
+ "1843": 122.0,
+ "1844": 113.0,
+ "1845": 127.0,
+ "1846": 100.0,
+ "1847": 117.0,
+ "1848": 133.0,
+ "1849": 87.0,
+ "1850": 103.0,
+ "1851": 89.0,
+ "1852": 99.0,
+ "1853": 93.0,
+ "1854": 99.0,
+ "1855": 107.0,
+ "1856": 111.0,
+ "1857": 121.0,
+ "1858": 92.0,
+ "1859": 105.0,
+ "1860": 115.0,
+ "1861": 92.0,
+ "1862": 91.0,
+ "1863": 112.0,
+ "1864": 109.0,
+ "1865": 125.0,
+ "1866": 124.0,
+ "1867": 110.0,
+ "1868": 113.0,
+ "1869": 119.0,
+ "1870": 137.0,
+ "1871": 126.0,
+ "1872": 95.0,
+ "1873": 119.0,
+ "1874": 105.0,
+ "1875": 128.0,
+ "1876": 104.0,
+ "1877": 120.0,
+ "1878": 95.0,
+ "1879": 99.0,
+ "1880": 123.0,
+ "1881": 99.0,
+ "1882": 97.0,
+ "1883": 101.0,
+ "1884": 115.0,
+ "1885": 106.0,
+ "1886": 123.0,
+ "1887": 121.0,
+ "1888": 121.0,
+ "1889": 114.0,
+ "1890": 100.0,
+ "1891": 110.0,
+ "1892": 107.0,
+ "1893": 113.0,
+ "1894": 134.0,
+ "1895": 114.0,
+ "1896": 111.0,
+ "1897": 122.0,
+ "1898": 108.0,
+ "1899": 94.0,
+ "1900": 123.0,
+ "1901": 125.0,
+ "1902": 115.0,
+ "1903": 112.0,
+ "1904": 113.0,
+ "1905": 109.0,
+ "1906": 115.0,
+ "1907": 95.0,
+ "1908": 113.0,
+ "1909": 79.0,
+ "1910": 97.0,
+ "1911": 135.0,
+ "1912": 122.0,
+ "1913": 105.0,
+ "1914": 112.0,
+ "1915": 129.0,
+ "1916": 117.0,
+ "1917": 115.0,
+ "1918": 113.0,
+ "1919": 117.0,
+ "1920": 122.0,
+ "1921": 105.0,
+ "1922": 86.0,
+ "1923": 113.0,
+ "1924": 111.0,
+ "1925": 110.0,
+ "1926": 112.0,
+ "1927": 103.0,
+ "1928": 108.0,
+ "1929": 113.0,
+ "1930": 121.0,
+ "1931": 111.0,
+ "1932": 106.0,
+ "1933": 114.0,
+ "1934": 117.0,
+ "1935": 93.0,
+ "1936": 109.0,
+ "1937": 121.0,
+ "1938": 108.0,
+ "1939": 132.0,
+ "1940": 127.0,
+ "1941": 126.0,
+ "1942": 101.0,
+ "1943": 120.0,
+ "1944": 87.0,
+ "1945": 114.0,
+ "1946": 105.0,
+ "1947": 109.0,
+ "1948": 109.0,
+ "1949": 106.0,
+ "1950": 111.0,
+ "1951": 120.0,
+ "1952": 104.0,
+ "1953": 113.0,
+ "1954": 116.0,
+ "1955": 131.0,
+ "1956": 91.0,
+ "1957": 118.0,
+ "1958": 139.0,
+ "1959": 114.0,
+ "1960": 96.0,
+ "1961": 109.0,
+ "1962": 113.0,
+ "1963": 125.0,
+ "1964": 112.0,
+ "1965": 108.0,
+ "1966": 130.0,
+ "1967": 120.0,
+ "1968": 110.0,
+ "1969": 96.0,
+ "1970": 110.0,
+ "1971": 121.0,
+ "1972": 104.0,
+ "1973": 103.0,
+ "1974": 110.0,
+ "1975": 101.0,
+ "1976": 144.0,
+ "1977": 122.0,
+ "1978": 118.0,
+ "1979": 121.0,
+ "1980": 115.0,
+ "1981": 114.0,
+ "1982": 136.0,
+ "1983": 123.0,
+ "1984": 112.0,
+ "1985": 116.0,
+ "1986": 104.0,
+ "1987": 133.0,
+ "1988": 107.0,
+ "1989": 100.0,
+ "1990": 112.0,
+ "1991": 119.0,
+ "1992": 103.0,
+ "1993": 133.0,
+ "1994": 123.0,
+ "1995": 118.0,
+ "1996": 109.0,
+ "1997": 119.0,
+ "1998": 107.0,
+ "1999": 119.0,
+ "2000": 134.0
+ }
+ },
+ "mem-allocated-bytes": {
+ "start_step": 1,
+ "end_step": 2000,
+ "step_interval": 1,
+ "values": {
+ "1": 442918400.0,
+ "2": 442918400.0,
+ "3": 442918400.0,
+ "4": 442918400.0,
+ "5": 442918400.0,
+ "6": 442918400.0,
+ "7": 442918400.0,
+ "8": 442918400.0,
+ "9": 442918400.0,
+ "10": 442918400.0,
+ "11": 442918400.0,
+ "12": 442918400.0,
+ "13": 442918400.0,
+ "14": 442918400.0,
+ "15": 442918400.0,
+ "16": 442918400.0,
+ "17": 442918400.0,
+ "18": 442918400.0,
+ "19": 442918400.0,
+ "20": 442918400.0,
+ "21": 442918400.0,
+ "22": 442918400.0,
+ "23": 442918400.0,
+ "24": 442918400.0,
+ "25": 442918400.0,
+ "26": 442918400.0,
+ "27": 442918400.0,
+ "28": 442918400.0,
+ "29": 442918400.0,
+ "30": 442918400.0,
+ "31": 442918400.0,
+ "32": 442918400.0,
+ "33": 442918400.0,
+ "34": 442918400.0,
+ "35": 442918400.0,
+ "36": 442918400.0,
+ "37": 442918400.0,
+ "38": 442918400.0,
+ "39": 442918400.0,
+ "40": 442918400.0,
+ "41": 442918400.0,
+ "42": 442918400.0,
+ "43": 442918400.0,
+ "44": 442918400.0,
+ "45": 442918400.0,
+ "46": 442918400.0,
+ "47": 442918400.0,
+ "48": 442918400.0,
+ "49": 442918400.0,
+ "50": 442918400.0,
+ "51": 442918400.0,
+ "52": 442918400.0,
+ "53": 442918400.0,
+ "54": 442918400.0,
+ "55": 442918400.0,
+ "56": 442918400.0,
+ "57": 442918400.0,
+ "58": 442918400.0,
+ "59": 442918400.0,
+ "60": 442918400.0,
+ "61": 442918400.0,
+ "62": 442918400.0,
+ "63": 442918400.0,
+ "64": 442918400.0,
+ "65": 442918400.0,
+ "66": 442918400.0,
+ "67": 442918400.0,
+ "68": 442918400.0,
+ "69": 442918400.0,
+ "70": 442918400.0,
+ "71": 442918400.0,
+ "72": 442918400.0,
+ "73": 442918400.0,
+ "74": 442918400.0,
+ "75": 442918400.0,
+ "76": 442918400.0,
+ "77": 442918400.0,
+ "78": 442918400.0,
+ "79": 442918400.0,
+ "80": 442918400.0,
+ "81": 442918400.0,
+ "82": 442918400.0,
+ "83": 442918400.0,
+ "84": 442918400.0,
+ "85": 442918400.0,
+ "86": 442918400.0,
+ "87": 442918400.0,
+ "88": 442918400.0,
+ "89": 442918400.0,
+ "90": 442918400.0,
+ "91": 442918400.0,
+ "92": 442918400.0,
+ "93": 442918400.0,
+ "94": 442918400.0,
+ "95": 442918400.0,
+ "96": 442918400.0,
+ "97": 442918400.0,
+ "98": 442918400.0,
+ "99": 442918400.0,
+ "100": 442918400.0,
+ "101": 442918400.0,
+ "102": 442918400.0,
+ "103": 442918400.0,
+ "104": 442918400.0,
+ "105": 442918400.0,
+ "106": 442918400.0,
+ "107": 442918400.0,
+ "108": 442918400.0,
+ "109": 442918400.0,
+ "110": 442918400.0,
+ "111": 442918400.0,
+ "112": 442918400.0,
+ "113": 442918400.0,
+ "114": 442918400.0,
+ "115": 442918400.0,
+ "116": 442918400.0,
+ "117": 442918400.0,
+ "118": 442918400.0,
+ "119": 442918400.0,
+ "120": 442918400.0,
+ "121": 442918400.0,
+ "122": 442918400.0,
+ "123": 442918400.0,
+ "124": 442918400.0,
+ "125": 442918400.0,
+ "126": 442918400.0,
+ "127": 442918400.0,
+ "128": 442918400.0,
+ "129": 442918400.0,
+ "130": 442918400.0,
+ "131": 442918400.0,
+ "132": 442918400.0,
+ "133": 442918400.0,
+ "134": 442918400.0,
+ "135": 442918400.0,
+ "136": 442918400.0,
+ "137": 442918400.0,
+ "138": 442918400.0,
+ "139": 442918400.0,
+ "140": 442918400.0,
+ "141": 442918400.0,
+ "142": 442918400.0,
+ "143": 442918400.0,
+ "144": 442918400.0,
+ "145": 442918400.0,
+ "146": 442918400.0,
+ "147": 442918400.0,
+ "148": 442918400.0,
+ "149": 442918400.0,
+ "150": 442918400.0,
+ "151": 442918400.0,
+ "152": 442918400.0,
+ "153": 442918400.0,
+ "154": 442918400.0,
+ "155": 442918400.0,
+ "156": 442918400.0,
+ "157": 442918400.0,
+ "158": 442918400.0,
+ "159": 442918400.0,
+ "160": 442918400.0,
+ "161": 442918400.0,
+ "162": 442918400.0,
+ "163": 442918400.0,
+ "164": 442918400.0,
+ "165": 442918400.0,
+ "166": 442918400.0,
+ "167": 442918400.0,
+ "168": 442918400.0,
+ "169": 442918400.0,
+ "170": 442918400.0,
+ "171": 442918400.0,
+ "172": 442918400.0,
+ "173": 442918400.0,
+ "174": 442918400.0,
+ "175": 442918400.0,
+ "176": 442918400.0,
+ "177": 442918400.0,
+ "178": 442918400.0,
+ "179": 442918400.0,
+ "180": 442918400.0,
+ "181": 442918400.0,
+ "182": 442918400.0,
+ "183": 442918400.0,
+ "184": 442918400.0,
+ "185": 442918400.0,
+ "186": 442918400.0,
+ "187": 442918400.0,
+ "188": 442918400.0,
+ "189": 442918400.0,
+ "190": 442918400.0,
+ "191": 442918400.0,
+ "192": 442918400.0,
+ "193": 442918400.0,
+ "194": 442918400.0,
+ "195": 442918400.0,
+ "196": 442918400.0,
+ "197": 442918400.0,
+ "198": 442918400.0,
+ "199": 442918400.0,
+ "200": 442918400.0,
+ "201": 442918400.0,
+ "202": 442918400.0,
+ "203": 442918400.0,
+ "204": 442918400.0,
+ "205": 442918400.0,
+ "206": 442918400.0,
+ "207": 442918400.0,
+ "208": 442918400.0,
+ "209": 442918400.0,
+ "210": 442918400.0,
+ "211": 442918400.0,
+ "212": 442918400.0,
+ "213": 442918400.0,
+ "214": 442918400.0,
+ "215": 442918400.0,
+ "216": 442918400.0,
+ "217": 442918400.0,
+ "218": 442918400.0,
+ "219": 442918400.0,
+ "220": 442918400.0,
+ "221": 442918400.0,
+ "222": 442918400.0,
+ "223": 442918400.0,
+ "224": 442918400.0,
+ "225": 442918400.0,
+ "226": 442918400.0,
+ "227": 442918400.0,
+ "228": 442918400.0,
+ "229": 442918400.0,
+ "230": 442918400.0,
+ "231": 442918400.0,
+ "232": 442918400.0,
+ "233": 442918400.0,
+ "234": 442918400.0,
+ "235": 442918400.0,
+ "236": 442918400.0,
+ "237": 442918400.0,
+ "238": 442918400.0,
+ "239": 442918400.0,
+ "240": 442918400.0,
+ "241": 442918400.0,
+ "242": 442918400.0,
+ "243": 442918400.0,
+ "244": 442918400.0,
+ "245": 442918400.0,
+ "246": 442918400.0,
+ "247": 442918400.0,
+ "248": 442918400.0,
+ "249": 442918400.0,
+ "250": 442918400.0,
+ "251": 442918400.0,
+ "252": 442918400.0,
+ "253": 442918400.0,
+ "254": 442918400.0,
+ "255": 442918400.0,
+ "256": 442918400.0,
+ "257": 442918400.0,
+ "258": 442918400.0,
+ "259": 442918400.0,
+ "260": 442918400.0,
+ "261": 442918400.0,
+ "262": 442918400.0,
+ "263": 442918400.0,
+ "264": 442918400.0,
+ "265": 442918400.0,
+ "266": 442918400.0,
+ "267": 442918400.0,
+ "268": 442918400.0,
+ "269": 442918400.0,
+ "270": 442918400.0,
+ "271": 442918400.0,
+ "272": 442918400.0,
+ "273": 442918400.0,
+ "274": 442918400.0,
+ "275": 442918400.0,
+ "276": 442918400.0,
+ "277": 442918400.0,
+ "278": 442918400.0,
+ "279": 442918400.0,
+ "280": 442918400.0,
+ "281": 442918400.0,
+ "282": 442918400.0,
+ "283": 442918400.0,
+ "284": 442918400.0,
+ "285": 442918400.0,
+ "286": 442918400.0,
+ "287": 442918400.0,
+ "288": 442918400.0,
+ "289": 442918400.0,
+ "290": 442918400.0,
+ "291": 442918400.0,
+ "292": 442918400.0,
+ "293": 442918400.0,
+ "294": 442918400.0,
+ "295": 442918400.0,
+ "296": 442918400.0,
+ "297": 442918400.0,
+ "298": 442918400.0,
+ "299": 442918400.0,
+ "300": 442918400.0,
+ "301": 442918400.0,
+ "302": 442918400.0,
+ "303": 442918400.0,
+ "304": 442918400.0,
+ "305": 442918400.0,
+ "306": 442918400.0,
+ "307": 442918400.0,
+ "308": 442918400.0,
+ "309": 442918400.0,
+ "310": 442918400.0,
+ "311": 442918400.0,
+ "312": 442918400.0,
+ "313": 442918400.0,
+ "314": 442918400.0,
+ "315": 442918400.0,
+ "316": 442918400.0,
+ "317": 442918400.0,
+ "318": 442918400.0,
+ "319": 442918400.0,
+ "320": 442918400.0,
+ "321": 442918400.0,
+ "322": 442918400.0,
+ "323": 442918400.0,
+ "324": 442918400.0,
+ "325": 442918400.0,
+ "326": 442918400.0,
+ "327": 442918400.0,
+ "328": 442918400.0,
+ "329": 442918400.0,
+ "330": 442918400.0,
+ "331": 442918400.0,
+ "332": 442918400.0,
+ "333": 442918400.0,
+ "334": 442918400.0,
+ "335": 442918400.0,
+ "336": 442918400.0,
+ "337": 442918400.0,
+ "338": 442918400.0,
+ "339": 442918400.0,
+ "340": 442918400.0,
+ "341": 442918400.0,
+ "342": 442918400.0,
+ "343": 442918400.0,
+ "344": 442918400.0,
+ "345": 442918400.0,
+ "346": 442918400.0,
+ "347": 442918400.0,
+ "348": 442918400.0,
+ "349": 442918400.0,
+ "350": 442918400.0,
+ "351": 442918400.0,
+ "352": 442918400.0,
+ "353": 442918400.0,
+ "354": 442918400.0,
+ "355": 442918400.0,
+ "356": 442918400.0,
+ "357": 442918400.0,
+ "358": 442918400.0,
+ "359": 442918400.0,
+ "360": 442918400.0,
+ "361": 442918400.0,
+ "362": 442918400.0,
+ "363": 442918400.0,
+ "364": 442918400.0,
+ "365": 442918400.0,
+ "366": 442918400.0,
+ "367": 442918400.0,
+ "368": 442918400.0,
+ "369": 442918400.0,
+ "370": 442918400.0,
+ "371": 442918400.0,
+ "372": 442918400.0,
+ "373": 442918400.0,
+ "374": 442918400.0,
+ "375": 442918400.0,
+ "376": 442918400.0,
+ "377": 442918400.0,
+ "378": 442918400.0,
+ "379": 442918400.0,
+ "380": 442918400.0,
+ "381": 442918400.0,
+ "382": 442918400.0,
+ "383": 442918400.0,
+ "384": 442918400.0,
+ "385": 442918400.0,
+ "386": 442918400.0,
+ "387": 442918400.0,
+ "388": 442918400.0,
+ "389": 442918400.0,
+ "390": 442918400.0,
+ "391": 442918400.0,
+ "392": 442918400.0,
+ "393": 442918400.0,
+ "394": 442918400.0,
+ "395": 442918400.0,
+ "396": 442918400.0,
+ "397": 442918400.0,
+ "398": 442918400.0,
+ "399": 442918400.0,
+ "400": 442918400.0,
+ "401": 442918400.0,
+ "402": 442918400.0,
+ "403": 442918400.0,
+ "404": 442918400.0,
+ "405": 442918400.0,
+ "406": 442918400.0,
+ "407": 442918400.0,
+ "408": 442918400.0,
+ "409": 442918400.0,
+ "410": 442918400.0,
+ "411": 442918400.0,
+ "412": 442918400.0,
+ "413": 442918400.0,
+ "414": 442918400.0,
+ "415": 442918400.0,
+ "416": 442918400.0,
+ "417": 442918400.0,
+ "418": 442918400.0,
+ "419": 442918400.0,
+ "420": 442918400.0,
+ "421": 442918400.0,
+ "422": 442918400.0,
+ "423": 442918400.0,
+ "424": 442918400.0,
+ "425": 442918400.0,
+ "426": 442918400.0,
+ "427": 442918400.0,
+ "428": 442918400.0,
+ "429": 442918400.0,
+ "430": 442918400.0,
+ "431": 442918400.0,
+ "432": 442918400.0,
+ "433": 442918400.0,
+ "434": 442918400.0,
+ "435": 442918400.0,
+ "436": 442918400.0,
+ "437": 442918400.0,
+ "438": 442918400.0,
+ "439": 442918400.0,
+ "440": 442918400.0,
+ "441": 442918400.0,
+ "442": 442918400.0,
+ "443": 442918400.0,
+ "444": 442918400.0,
+ "445": 442918400.0,
+ "446": 442918400.0,
+ "447": 442918400.0,
+ "448": 442918400.0,
+ "449": 442918400.0,
+ "450": 442918400.0,
+ "451": 442918400.0,
+ "452": 442918400.0,
+ "453": 442918400.0,
+ "454": 442918400.0,
+ "455": 442918400.0,
+ "456": 442918400.0,
+ "457": 442918400.0,
+ "458": 442918400.0,
+ "459": 442918400.0,
+ "460": 442918400.0,
+ "461": 442918400.0,
+ "462": 442918400.0,
+ "463": 442918400.0,
+ "464": 442918400.0,
+ "465": 442918400.0,
+ "466": 442918400.0,
+ "467": 442918400.0,
+ "468": 442918400.0,
+ "469": 442918400.0,
+ "470": 442918400.0,
+ "471": 442918400.0,
+ "472": 442918400.0,
+ "473": 442918400.0,
+ "474": 442918400.0,
+ "475": 442918400.0,
+ "476": 442918400.0,
+ "477": 442918400.0,
+ "478": 442918400.0,
+ "479": 442918400.0,
+ "480": 442918400.0,
+ "481": 442918400.0,
+ "482": 442918400.0,
+ "483": 442918400.0,
+ "484": 442918400.0,
+ "485": 442918400.0,
+ "486": 442918400.0,
+ "487": 442918400.0,
+ "488": 442918400.0,
+ "489": 442918400.0,
+ "490": 442918400.0,
+ "491": 442918400.0,
+ "492": 442918400.0,
+ "493": 442918400.0,
+ "494": 442918400.0,
+ "495": 442918400.0,
+ "496": 442918400.0,
+ "497": 442918400.0,
+ "498": 442918400.0,
+ "499": 442918400.0,
+ "500": 442918400.0,
+ "501": 442918400.0,
+ "502": 442918400.0,
+ "503": 442918400.0,
+ "504": 442918400.0,
+ "505": 442918400.0,
+ "506": 442918400.0,
+ "507": 442918400.0,
+ "508": 442918400.0,
+ "509": 442918400.0,
+ "510": 442918400.0,
+ "511": 442918400.0,
+ "512": 442918400.0,
+ "513": 442918400.0,
+ "514": 442918400.0,
+ "515": 442918400.0,
+ "516": 442918400.0,
+ "517": 442918400.0,
+ "518": 442918400.0,
+ "519": 442918400.0,
+ "520": 442918400.0,
+ "521": 442918400.0,
+ "522": 442918400.0,
+ "523": 442918400.0,
+ "524": 442918400.0,
+ "525": 442918400.0,
+ "526": 442918400.0,
+ "527": 442918400.0,
+ "528": 442918400.0,
+ "529": 442918400.0,
+ "530": 442918400.0,
+ "531": 442918400.0,
+ "532": 442918400.0,
+ "533": 442918400.0,
+ "534": 442918400.0,
+ "535": 442918400.0,
+ "536": 442918400.0,
+ "537": 442918400.0,
+ "538": 442918400.0,
+ "539": 442918400.0,
+ "540": 442918400.0,
+ "541": 442918400.0,
+ "542": 442918400.0,
+ "543": 442918400.0,
+ "544": 442918400.0,
+ "545": 442918400.0,
+ "546": 442918400.0,
+ "547": 442918400.0,
+ "548": 442918400.0,
+ "549": 442918400.0,
+ "550": 442918400.0,
+ "551": 442918400.0,
+ "552": 442918400.0,
+ "553": 442918400.0,
+ "554": 442918400.0,
+ "555": 442918400.0,
+ "556": 442918400.0,
+ "557": 442918400.0,
+ "558": 442918400.0,
+ "559": 442918400.0,
+ "560": 442918400.0,
+ "561": 442918400.0,
+ "562": 442918400.0,
+ "563": 442918400.0,
+ "564": 442918400.0,
+ "565": 442918400.0,
+ "566": 442918400.0,
+ "567": 442918400.0,
+ "568": 442918400.0,
+ "569": 442918400.0,
+ "570": 442918400.0,
+ "571": 442918400.0,
+ "572": 442918400.0,
+ "573": 442918400.0,
+ "574": 442918400.0,
+ "575": 442918400.0,
+ "576": 442918400.0,
+ "577": 442918400.0,
+ "578": 442918400.0,
+ "579": 442918400.0,
+ "580": 442918400.0,
+ "581": 442918400.0,
+ "582": 442918400.0,
+ "583": 442918400.0,
+ "584": 442918400.0,
+ "585": 442918400.0,
+ "586": 442918400.0,
+ "587": 442918400.0,
+ "588": 442918400.0,
+ "589": 442918400.0,
+ "590": 442918400.0,
+ "591": 442918400.0,
+ "592": 442918400.0,
+ "593": 442918400.0,
+ "594": 442918400.0,
+ "595": 442918400.0,
+ "596": 442918400.0,
+ "597": 442918400.0,
+ "598": 442918400.0,
+ "599": 442918400.0,
+ "600": 442918400.0,
+ "601": 442918400.0,
+ "602": 442918400.0,
+ "603": 442918400.0,
+ "604": 442918400.0,
+ "605": 442918400.0,
+ "606": 442918400.0,
+ "607": 442918400.0,
+ "608": 442918400.0,
+ "609": 442918400.0,
+ "610": 442918400.0,
+ "611": 442918400.0,
+ "612": 442918400.0,
+ "613": 442918400.0,
+ "614": 442918400.0,
+ "615": 442918400.0,
+ "616": 442918400.0,
+ "617": 442918400.0,
+ "618": 442918400.0,
+ "619": 442918400.0,
+ "620": 442918400.0,
+ "621": 442918400.0,
+ "622": 442918400.0,
+ "623": 442918400.0,
+ "624": 442918400.0,
+ "625": 442918400.0,
+ "626": 442918400.0,
+ "627": 442918400.0,
+ "628": 442918400.0,
+ "629": 442918400.0,
+ "630": 442918400.0,
+ "631": 442918400.0,
+ "632": 442918400.0,
+ "633": 442918400.0,
+ "634": 442918400.0,
+ "635": 442918400.0,
+ "636": 442918400.0,
+ "637": 442918400.0,
+ "638": 442918400.0,
+ "639": 442918400.0,
+ "640": 442918400.0,
+ "641": 442918400.0,
+ "642": 442918400.0,
+ "643": 442918400.0,
+ "644": 442918400.0,
+ "645": 442918400.0,
+ "646": 442918400.0,
+ "647": 442918400.0,
+ "648": 442918400.0,
+ "649": 442918400.0,
+ "650": 442918400.0,
+ "651": 442918400.0,
+ "652": 442918400.0,
+ "653": 442918400.0,
+ "654": 442918400.0,
+ "655": 442918400.0,
+ "656": 442918400.0,
+ "657": 442918400.0,
+ "658": 442918400.0,
+ "659": 442918400.0,
+ "660": 442918400.0,
+ "661": 442918400.0,
+ "662": 442918400.0,
+ "663": 442918400.0,
+ "664": 442918400.0,
+ "665": 442918400.0,
+ "666": 442918400.0,
+ "667": 442918400.0,
+ "668": 442918400.0,
+ "669": 442918400.0,
+ "670": 442918400.0,
+ "671": 442918400.0,
+ "672": 442918400.0,
+ "673": 442918400.0,
+ "674": 442918400.0,
+ "675": 442918400.0,
+ "676": 442918400.0,
+ "677": 442918400.0,
+ "678": 442918400.0,
+ "679": 442918400.0,
+ "680": 442918400.0,
+ "681": 442918400.0,
+ "682": 442918400.0,
+ "683": 442918400.0,
+ "684": 442918400.0,
+ "685": 442918400.0,
+ "686": 442918400.0,
+ "687": 442918400.0,
+ "688": 442918400.0,
+ "689": 442918400.0,
+ "690": 442918400.0,
+ "691": 442918400.0,
+ "692": 442918400.0,
+ "693": 442918400.0,
+ "694": 442918400.0,
+ "695": 442918400.0,
+ "696": 442918400.0,
+ "697": 442918400.0,
+ "698": 442918400.0,
+ "699": 442918400.0,
+ "700": 442918400.0,
+ "701": 442918400.0,
+ "702": 442918400.0,
+ "703": 442918400.0,
+ "704": 442918400.0,
+ "705": 442918400.0,
+ "706": 442918400.0,
+ "707": 442918400.0,
+ "708": 442918400.0,
+ "709": 442918400.0,
+ "710": 442918400.0,
+ "711": 442918400.0,
+ "712": 442918400.0,
+ "713": 442918400.0,
+ "714": 442918400.0,
+ "715": 442918400.0,
+ "716": 442918400.0,
+ "717": 442918400.0,
+ "718": 442918400.0,
+ "719": 442918400.0,
+ "720": 442918400.0,
+ "721": 442918400.0,
+ "722": 442918400.0,
+ "723": 442918400.0,
+ "724": 442918400.0,
+ "725": 442918400.0,
+ "726": 442918400.0,
+ "727": 442918400.0,
+ "728": 442918400.0,
+ "729": 442918400.0,
+ "730": 442918400.0,
+ "731": 442918400.0,
+ "732": 442918400.0,
+ "733": 442918400.0,
+ "734": 442918400.0,
+ "735": 442918400.0,
+ "736": 442918400.0,
+ "737": 442918400.0,
+ "738": 442918400.0,
+ "739": 442918400.0,
+ "740": 442918400.0,
+ "741": 442918400.0,
+ "742": 442918400.0,
+ "743": 442918400.0,
+ "744": 442918400.0,
+ "745": 442918400.0,
+ "746": 442918400.0,
+ "747": 442918400.0,
+ "748": 442918400.0,
+ "749": 442918400.0,
+ "750": 442918400.0,
+ "751": 442918400.0,
+ "752": 442918400.0,
+ "753": 442918400.0,
+ "754": 442918400.0,
+ "755": 442918400.0,
+ "756": 442918400.0,
+ "757": 442918400.0,
+ "758": 442918400.0,
+ "759": 442918400.0,
+ "760": 442918400.0,
+ "761": 442918400.0,
+ "762": 442918400.0,
+ "763": 442918400.0,
+ "764": 442918400.0,
+ "765": 442918400.0,
+ "766": 442918400.0,
+ "767": 442918400.0,
+ "768": 442918400.0,
+ "769": 442918400.0,
+ "770": 442918400.0,
+ "771": 442918400.0,
+ "772": 442918400.0,
+ "773": 442918400.0,
+ "774": 442918400.0,
+ "775": 442918400.0,
+ "776": 442918400.0,
+ "777": 442918400.0,
+ "778": 442918400.0,
+ "779": 442918400.0,
+ "780": 442918400.0,
+ "781": 442918400.0,
+ "782": 442918400.0,
+ "783": 442918400.0,
+ "784": 442918400.0,
+ "785": 442918400.0,
+ "786": 442918400.0,
+ "787": 442918400.0,
+ "788": 442918400.0,
+ "789": 442918400.0,
+ "790": 442918400.0,
+ "791": 442918400.0,
+ "792": 442918400.0,
+ "793": 442918400.0,
+ "794": 442918400.0,
+ "795": 442918400.0,
+ "796": 442918400.0,
+ "797": 442918400.0,
+ "798": 442918400.0,
+ "799": 442918400.0,
+ "800": 442918400.0,
+ "801": 442918400.0,
+ "802": 442918400.0,
+ "803": 442918400.0,
+ "804": 442918400.0,
+ "805": 442918400.0,
+ "806": 442918400.0,
+ "807": 442918400.0,
+ "808": 442918400.0,
+ "809": 442918400.0,
+ "810": 442918400.0,
+ "811": 442918400.0,
+ "812": 442918400.0,
+ "813": 442918400.0,
+ "814": 442918400.0,
+ "815": 442918400.0,
+ "816": 442918400.0,
+ "817": 442918400.0,
+ "818": 442918400.0,
+ "819": 442918400.0,
+ "820": 442918400.0,
+ "821": 442918400.0,
+ "822": 442918400.0,
+ "823": 442918400.0,
+ "824": 442918400.0,
+ "825": 442918400.0,
+ "826": 442918400.0,
+ "827": 442918400.0,
+ "828": 442918400.0,
+ "829": 442918400.0,
+ "830": 442918400.0,
+ "831": 442918400.0,
+ "832": 442918400.0,
+ "833": 442918400.0,
+ "834": 442918400.0,
+ "835": 442918400.0,
+ "836": 442918400.0,
+ "837": 442918400.0,
+ "838": 442918400.0,
+ "839": 442918400.0,
+ "840": 442918400.0,
+ "841": 442918400.0,
+ "842": 442918400.0,
+ "843": 442918400.0,
+ "844": 442918400.0,
+ "845": 442918400.0,
+ "846": 442918400.0,
+ "847": 442918400.0,
+ "848": 442918400.0,
+ "849": 442918400.0,
+ "850": 442918400.0,
+ "851": 442918400.0,
+ "852": 442918400.0,
+ "853": 442918400.0,
+ "854": 442918400.0,
+ "855": 442918400.0,
+ "856": 442918400.0,
+ "857": 442918400.0,
+ "858": 442918400.0,
+ "859": 442918400.0,
+ "860": 442918400.0,
+ "861": 442918400.0,
+ "862": 442918400.0,
+ "863": 442918400.0,
+ "864": 442918400.0,
+ "865": 442918400.0,
+ "866": 442918400.0,
+ "867": 442918400.0,
+ "868": 442918400.0,
+ "869": 442918400.0,
+ "870": 442918400.0,
+ "871": 442918400.0,
+ "872": 442918400.0,
+ "873": 442918400.0,
+ "874": 442918400.0,
+ "875": 442918400.0,
+ "876": 442918400.0,
+ "877": 442918400.0,
+ "878": 442918400.0,
+ "879": 442918400.0,
+ "880": 442918400.0,
+ "881": 442918400.0,
+ "882": 442918400.0,
+ "883": 442918400.0,
+ "884": 442918400.0,
+ "885": 442918400.0,
+ "886": 442918400.0,
+ "887": 442918400.0,
+ "888": 442918400.0,
+ "889": 442918400.0,
+ "890": 442918400.0,
+ "891": 442918400.0,
+ "892": 442918400.0,
+ "893": 442918400.0,
+ "894": 442918400.0,
+ "895": 442918400.0,
+ "896": 442918400.0,
+ "897": 442918400.0,
+ "898": 442918400.0,
+ "899": 442918400.0,
+ "900": 442918400.0,
+ "901": 442918400.0,
+ "902": 442918400.0,
+ "903": 442918400.0,
+ "904": 442918400.0,
+ "905": 442918400.0,
+ "906": 442918400.0,
+ "907": 442918400.0,
+ "908": 442918400.0,
+ "909": 442918400.0,
+ "910": 442918400.0,
+ "911": 442918400.0,
+ "912": 442918400.0,
+ "913": 442918400.0,
+ "914": 442918400.0,
+ "915": 442918400.0,
+ "916": 442918400.0,
+ "917": 442918400.0,
+ "918": 442918400.0,
+ "919": 442918400.0,
+ "920": 442918400.0,
+ "921": 442918400.0,
+ "922": 442918400.0,
+ "923": 442918400.0,
+ "924": 442918400.0,
+ "925": 442918400.0,
+ "926": 442918400.0,
+ "927": 442918400.0,
+ "928": 442918400.0,
+ "929": 442918400.0,
+ "930": 442918400.0,
+ "931": 442918400.0,
+ "932": 442918400.0,
+ "933": 442918400.0,
+ "934": 442918400.0,
+ "935": 442918400.0,
+ "936": 442918400.0,
+ "937": 442918400.0,
+ "938": 442918400.0,
+ "939": 442918400.0,
+ "940": 442918400.0,
+ "941": 442918400.0,
+ "942": 442918400.0,
+ "943": 442918400.0,
+ "944": 442918400.0,
+ "945": 442918400.0,
+ "946": 442918400.0,
+ "947": 442918400.0,
+ "948": 442918400.0,
+ "949": 442918400.0,
+ "950": 442918400.0,
+ "951": 442918400.0,
+ "952": 442918400.0,
+ "953": 442918400.0,
+ "954": 442918400.0,
+ "955": 442918400.0,
+ "956": 442918400.0,
+ "957": 442918400.0,
+ "958": 442918400.0,
+ "959": 442918400.0,
+ "960": 442918400.0,
+ "961": 442918400.0,
+ "962": 442918400.0,
+ "963": 442918400.0,
+ "964": 442918400.0,
+ "965": 442918400.0,
+ "966": 442918400.0,
+ "967": 442918400.0,
+ "968": 442918400.0,
+ "969": 442918400.0,
+ "970": 442918400.0,
+ "971": 442918400.0,
+ "972": 442918400.0,
+ "973": 442918400.0,
+ "974": 442918400.0,
+ "975": 442918400.0,
+ "976": 442918400.0,
+ "977": 442918400.0,
+ "978": 442918400.0,
+ "979": 442918400.0,
+ "980": 442918400.0,
+ "981": 442918400.0,
+ "982": 442918400.0,
+ "983": 442918400.0,
+ "984": 442918400.0,
+ "985": 442918400.0,
+ "986": 442918400.0,
+ "987": 442918400.0,
+ "988": 442918400.0,
+ "989": 442918400.0,
+ "990": 442918400.0,
+ "991": 442918400.0,
+ "992": 442918400.0,
+ "993": 442918400.0,
+ "994": 442918400.0,
+ "995": 442918400.0,
+ "996": 442918400.0,
+ "997": 442918400.0,
+ "998": 442918400.0,
+ "999": 442918400.0,
+ "1000": 442918400.0,
+ "1001": 442918400.0,
+ "1002": 442918400.0,
+ "1003": 442918400.0,
+ "1004": 442918400.0,
+ "1005": 442918400.0,
+ "1006": 442918400.0,
+ "1007": 442918400.0,
+ "1008": 442918400.0,
+ "1009": 442918400.0,
+ "1010": 442918400.0,
+ "1011": 442918400.0,
+ "1012": 442918400.0,
+ "1013": 442918400.0,
+ "1014": 442918400.0,
+ "1015": 442918400.0,
+ "1016": 442918400.0,
+ "1017": 442918400.0,
+ "1018": 442918400.0,
+ "1019": 442918400.0,
+ "1020": 442918400.0,
+ "1021": 442918400.0,
+ "1022": 442918400.0,
+ "1023": 442918400.0,
+ "1024": 442918400.0,
+ "1025": 442918400.0,
+ "1026": 442918400.0,
+ "1027": 442918400.0,
+ "1028": 442918400.0,
+ "1029": 442918400.0,
+ "1030": 442918400.0,
+ "1031": 442918400.0,
+ "1032": 442918400.0,
+ "1033": 442918400.0,
+ "1034": 442918400.0,
+ "1035": 442918400.0,
+ "1036": 442918400.0,
+ "1037": 442918400.0,
+ "1038": 442918400.0,
+ "1039": 442918400.0,
+ "1040": 442918400.0,
+ "1041": 442918400.0,
+ "1042": 442918400.0,
+ "1043": 442918400.0,
+ "1044": 442918400.0,
+ "1045": 442918400.0,
+ "1046": 442918400.0,
+ "1047": 442918400.0,
+ "1048": 442918400.0,
+ "1049": 442918400.0,
+ "1050": 442918400.0,
+ "1051": 442918400.0,
+ "1052": 442918400.0,
+ "1053": 442918400.0,
+ "1054": 442918400.0,
+ "1055": 442918400.0,
+ "1056": 442918400.0,
+ "1057": 442918400.0,
+ "1058": 442918400.0,
+ "1059": 442918400.0,
+ "1060": 442918400.0,
+ "1061": 442918400.0,
+ "1062": 442918400.0,
+ "1063": 442918400.0,
+ "1064": 442918400.0,
+ "1065": 442918400.0,
+ "1066": 442918400.0,
+ "1067": 442918400.0,
+ "1068": 442918400.0,
+ "1069": 442918400.0,
+ "1070": 442918400.0,
+ "1071": 442918400.0,
+ "1072": 442918400.0,
+ "1073": 442918400.0,
+ "1074": 442918400.0,
+ "1075": 442918400.0,
+ "1076": 442918400.0,
+ "1077": 442918400.0,
+ "1078": 442918400.0,
+ "1079": 442918400.0,
+ "1080": 442918400.0,
+ "1081": 442918400.0,
+ "1082": 442918400.0,
+ "1083": 442918400.0,
+ "1084": 442918400.0,
+ "1085": 442918400.0,
+ "1086": 442918400.0,
+ "1087": 442918400.0,
+ "1088": 442918400.0,
+ "1089": 442918400.0,
+ "1090": 442918400.0,
+ "1091": 442918400.0,
+ "1092": 442918400.0,
+ "1093": 442918400.0,
+ "1094": 442918400.0,
+ "1095": 442918400.0,
+ "1096": 442918400.0,
+ "1097": 442918400.0,
+ "1098": 442918400.0,
+ "1099": 442918400.0,
+ "1100": 442918400.0,
+ "1101": 442918400.0,
+ "1102": 442918400.0,
+ "1103": 442918400.0,
+ "1104": 442918400.0,
+ "1105": 442918400.0,
+ "1106": 442918400.0,
+ "1107": 442918400.0,
+ "1108": 442918400.0,
+ "1109": 442918400.0,
+ "1110": 442918400.0,
+ "1111": 442918400.0,
+ "1112": 442918400.0,
+ "1113": 442918400.0,
+ "1114": 442918400.0,
+ "1115": 442918400.0,
+ "1116": 442918400.0,
+ "1117": 442918400.0,
+ "1118": 442918400.0,
+ "1119": 442918400.0,
+ "1120": 442918400.0,
+ "1121": 442918400.0,
+ "1122": 442918400.0,
+ "1123": 442918400.0,
+ "1124": 442918400.0,
+ "1125": 442918400.0,
+ "1126": 442918400.0,
+ "1127": 442918400.0,
+ "1128": 442918400.0,
+ "1129": 442918400.0,
+ "1130": 442918400.0,
+ "1131": 442918400.0,
+ "1132": 442918400.0,
+ "1133": 442918400.0,
+ "1134": 442918400.0,
+ "1135": 442918400.0,
+ "1136": 442918400.0,
+ "1137": 442918400.0,
+ "1138": 442918400.0,
+ "1139": 442918400.0,
+ "1140": 442918400.0,
+ "1141": 442918400.0,
+ "1142": 442918400.0,
+ "1143": 442918400.0,
+ "1144": 442918400.0,
+ "1145": 442918400.0,
+ "1146": 442918400.0,
+ "1147": 442918400.0,
+ "1148": 442918400.0,
+ "1149": 442918400.0,
+ "1150": 442918400.0,
+ "1151": 442918400.0,
+ "1152": 442918400.0,
+ "1153": 442918400.0,
+ "1154": 442918400.0,
+ "1155": 442918400.0,
+ "1156": 442918400.0,
+ "1157": 442918400.0,
+ "1158": 442918400.0,
+ "1159": 442918400.0,
+ "1160": 442918400.0,
+ "1161": 442918400.0,
+ "1162": 442918400.0,
+ "1163": 442918400.0,
+ "1164": 442918400.0,
+ "1165": 442918400.0,
+ "1166": 442918400.0,
+ "1167": 442918400.0,
+ "1168": 442918400.0,
+ "1169": 442918400.0,
+ "1170": 442918400.0,
+ "1171": 442918400.0,
+ "1172": 442918400.0,
+ "1173": 442918400.0,
+ "1174": 442918400.0,
+ "1175": 442918400.0,
+ "1176": 442918400.0,
+ "1177": 442918400.0,
+ "1178": 442918400.0,
+ "1179": 442918400.0,
+ "1180": 442918400.0,
+ "1181": 442918400.0,
+ "1182": 442918400.0,
+ "1183": 442918400.0,
+ "1184": 442918400.0,
+ "1185": 442918400.0,
+ "1186": 442918400.0,
+ "1187": 442918400.0,
+ "1188": 442918400.0,
+ "1189": 442918400.0,
+ "1190": 442918400.0,
+ "1191": 442918400.0,
+ "1192": 442918400.0,
+ "1193": 442918400.0,
+ "1194": 442918400.0,
+ "1195": 442918400.0,
+ "1196": 442918400.0,
+ "1197": 442918400.0,
+ "1198": 442918400.0,
+ "1199": 442918400.0,
+ "1200": 442918400.0,
+ "1201": 442918400.0,
+ "1202": 442918400.0,
+ "1203": 442918400.0,
+ "1204": 442918400.0,
+ "1205": 442918400.0,
+ "1206": 442918400.0,
+ "1207": 442918400.0,
+ "1208": 442918400.0,
+ "1209": 442918400.0,
+ "1210": 442918400.0,
+ "1211": 442918400.0,
+ "1212": 442918400.0,
+ "1213": 442918400.0,
+ "1214": 442918400.0,
+ "1215": 442918400.0,
+ "1216": 442918400.0,
+ "1217": 442918400.0,
+ "1218": 442918400.0,
+ "1219": 442918400.0,
+ "1220": 442918400.0,
+ "1221": 442918400.0,
+ "1222": 442918400.0,
+ "1223": 442918400.0,
+ "1224": 442918400.0,
+ "1225": 442918400.0,
+ "1226": 442918400.0,
+ "1227": 442918400.0,
+ "1228": 442918400.0,
+ "1229": 442918400.0,
+ "1230": 442918400.0,
+ "1231": 442918400.0,
+ "1232": 442918400.0,
+ "1233": 442918400.0,
+ "1234": 442918400.0,
+ "1235": 442918400.0,
+ "1236": 442918400.0,
+ "1237": 442918400.0,
+ "1238": 442918400.0,
+ "1239": 442918400.0,
+ "1240": 442918400.0,
+ "1241": 442918400.0,
+ "1242": 442918400.0,
+ "1243": 442918400.0,
+ "1244": 442918400.0,
+ "1245": 442918400.0,
+ "1246": 442918400.0,
+ "1247": 442918400.0,
+ "1248": 442918400.0,
+ "1249": 442918400.0,
+ "1250": 442918400.0,
+ "1251": 442918400.0,
+ "1252": 442918400.0,
+ "1253": 442918400.0,
+ "1254": 442918400.0,
+ "1255": 442918400.0,
+ "1256": 442918400.0,
+ "1257": 442918400.0,
+ "1258": 442918400.0,
+ "1259": 442918400.0,
+ "1260": 442918400.0,
+ "1261": 442918400.0,
+ "1262": 442918400.0,
+ "1263": 442918400.0,
+ "1264": 442918400.0,
+ "1265": 442918400.0,
+ "1266": 442918400.0,
+ "1267": 442918400.0,
+ "1268": 442918400.0,
+ "1269": 442918400.0,
+ "1270": 442918400.0,
+ "1271": 442918400.0,
+ "1272": 442918400.0,
+ "1273": 442918400.0,
+ "1274": 442918400.0,
+ "1275": 442918400.0,
+ "1276": 442918400.0,
+ "1277": 442918400.0,
+ "1278": 442918400.0,
+ "1279": 442918400.0,
+ "1280": 442918400.0,
+ "1281": 442918400.0,
+ "1282": 442918400.0,
+ "1283": 442918400.0,
+ "1284": 442918400.0,
+ "1285": 442918400.0,
+ "1286": 442918400.0,
+ "1287": 442918400.0,
+ "1288": 442918400.0,
+ "1289": 442918400.0,
+ "1290": 442918400.0,
+ "1291": 442918400.0,
+ "1292": 442918400.0,
+ "1293": 442918400.0,
+ "1294": 442918400.0,
+ "1295": 442918400.0,
+ "1296": 442918400.0,
+ "1297": 442918400.0,
+ "1298": 442918400.0,
+ "1299": 442918400.0,
+ "1300": 442918400.0,
+ "1301": 442918400.0,
+ "1302": 442918400.0,
+ "1303": 442918400.0,
+ "1304": 442918400.0,
+ "1305": 442918400.0,
+ "1306": 442918400.0,
+ "1307": 442918400.0,
+ "1308": 442918400.0,
+ "1309": 442918400.0,
+ "1310": 442918400.0,
+ "1311": 442918400.0,
+ "1312": 442918400.0,
+ "1313": 442918400.0,
+ "1314": 442918400.0,
+ "1315": 442918400.0,
+ "1316": 442918400.0,
+ "1317": 442918400.0,
+ "1318": 442918400.0,
+ "1319": 442918400.0,
+ "1320": 442918400.0,
+ "1321": 442918400.0,
+ "1322": 442918400.0,
+ "1323": 442918400.0,
+ "1324": 442918400.0,
+ "1325": 442918400.0,
+ "1326": 442918400.0,
+ "1327": 442918400.0,
+ "1328": 442918400.0,
+ "1329": 442918400.0,
+ "1330": 442918400.0,
+ "1331": 442918400.0,
+ "1332": 442918400.0,
+ "1333": 442918400.0,
+ "1334": 442918400.0,
+ "1335": 442918400.0,
+ "1336": 442918400.0,
+ "1337": 442918400.0,
+ "1338": 442918400.0,
+ "1339": 442918400.0,
+ "1340": 442918400.0,
+ "1341": 442918400.0,
+ "1342": 442918400.0,
+ "1343": 442918400.0,
+ "1344": 442918400.0,
+ "1345": 442918400.0,
+ "1346": 442918400.0,
+ "1347": 442918400.0,
+ "1348": 442918400.0,
+ "1349": 442918400.0,
+ "1350": 442918400.0,
+ "1351": 442918400.0,
+ "1352": 442918400.0,
+ "1353": 442918400.0,
+ "1354": 442918400.0,
+ "1355": 442918400.0,
+ "1356": 442918400.0,
+ "1357": 442918400.0,
+ "1358": 442918400.0,
+ "1359": 442918400.0,
+ "1360": 442918400.0,
+ "1361": 442918400.0,
+ "1362": 442918400.0,
+ "1363": 442918400.0,
+ "1364": 442918400.0,
+ "1365": 442918400.0,
+ "1366": 442918400.0,
+ "1367": 442918400.0,
+ "1368": 442918400.0,
+ "1369": 442918400.0,
+ "1370": 442918400.0,
+ "1371": 442918400.0,
+ "1372": 442918400.0,
+ "1373": 442918400.0,
+ "1374": 442918400.0,
+ "1375": 442918400.0,
+ "1376": 442918400.0,
+ "1377": 442918400.0,
+ "1378": 442918400.0,
+ "1379": 442918400.0,
+ "1380": 442918400.0,
+ "1381": 442918400.0,
+ "1382": 442918400.0,
+ "1383": 442918400.0,
+ "1384": 442918400.0,
+ "1385": 442918400.0,
+ "1386": 442918400.0,
+ "1387": 442918400.0,
+ "1388": 442918400.0,
+ "1389": 442918400.0,
+ "1390": 442918400.0,
+ "1391": 442918400.0,
+ "1392": 442918400.0,
+ "1393": 442918400.0,
+ "1394": 442918400.0,
+ "1395": 442918400.0,
+ "1396": 442918400.0,
+ "1397": 442918400.0,
+ "1398": 442918400.0,
+ "1399": 442918400.0,
+ "1400": 442918400.0,
+ "1401": 442918400.0,
+ "1402": 442918400.0,
+ "1403": 442918400.0,
+ "1404": 442918400.0,
+ "1405": 442918400.0,
+ "1406": 442918400.0,
+ "1407": 442918400.0,
+ "1408": 442918400.0,
+ "1409": 442918400.0,
+ "1410": 442918400.0,
+ "1411": 442918400.0,
+ "1412": 442918400.0,
+ "1413": 442918400.0,
+ "1414": 442918400.0,
+ "1415": 442918400.0,
+ "1416": 442918400.0,
+ "1417": 442918400.0,
+ "1418": 442918400.0,
+ "1419": 442918400.0,
+ "1420": 442918400.0,
+ "1421": 442918400.0,
+ "1422": 442918400.0,
+ "1423": 442918400.0,
+ "1424": 442918400.0,
+ "1425": 442918400.0,
+ "1426": 442918400.0,
+ "1427": 442918400.0,
+ "1428": 442918400.0,
+ "1429": 442918400.0,
+ "1430": 442918400.0,
+ "1431": 442918400.0,
+ "1432": 442918400.0,
+ "1433": 442918400.0,
+ "1434": 442918400.0,
+ "1435": 442918400.0,
+ "1436": 442918400.0,
+ "1437": 442918400.0,
+ "1438": 442918400.0,
+ "1439": 442918400.0,
+ "1440": 442918400.0,
+ "1441": 442918400.0,
+ "1442": 442918400.0,
+ "1443": 442918400.0,
+ "1444": 442918400.0,
+ "1445": 442918400.0,
+ "1446": 442918400.0,
+ "1447": 442918400.0,
+ "1448": 442918400.0,
+ "1449": 442918400.0,
+ "1450": 442918400.0,
+ "1451": 442918400.0,
+ "1452": 442918400.0,
+ "1453": 442918400.0,
+ "1454": 442918400.0,
+ "1455": 442918400.0,
+ "1456": 442918400.0,
+ "1457": 442918400.0,
+ "1458": 442918400.0,
+ "1459": 442918400.0,
+ "1460": 442918400.0,
+ "1461": 442918400.0,
+ "1462": 442918400.0,
+ "1463": 442918400.0,
+ "1464": 442918400.0,
+ "1465": 442918400.0,
+ "1466": 442918400.0,
+ "1467": 442918400.0,
+ "1468": 442918400.0,
+ "1469": 442918400.0,
+ "1470": 442918400.0,
+ "1471": 442918400.0,
+ "1472": 442918400.0,
+ "1473": 442918400.0,
+ "1474": 442918400.0,
+ "1475": 442918400.0,
+ "1476": 442918400.0,
+ "1477": 442918400.0,
+ "1478": 442918400.0,
+ "1479": 442918400.0,
+ "1480": 442918400.0,
+ "1481": 442918400.0,
+ "1482": 442918400.0,
+ "1483": 442918400.0,
+ "1484": 442918400.0,
+ "1485": 442918400.0,
+ "1486": 442918400.0,
+ "1487": 442918400.0,
+ "1488": 442918400.0,
+ "1489": 442918400.0,
+ "1490": 442918400.0,
+ "1491": 442918400.0,
+ "1492": 442918400.0,
+ "1493": 442918400.0,
+ "1494": 442918400.0,
+ "1495": 442918400.0,
+ "1496": 442918400.0,
+ "1497": 442918400.0,
+ "1498": 442918400.0,
+ "1499": 442918400.0,
+ "1500": 442918400.0,
+ "1501": 442918400.0,
+ "1502": 442918400.0,
+ "1503": 442918400.0,
+ "1504": 442918400.0,
+ "1505": 442918400.0,
+ "1506": 442918400.0,
+ "1507": 442918400.0,
+ "1508": 442918400.0,
+ "1509": 442918400.0,
+ "1510": 442918400.0,
+ "1511": 442918400.0,
+ "1512": 442918400.0,
+ "1513": 442918400.0,
+ "1514": 442918400.0,
+ "1515": 442918400.0,
+ "1516": 442918400.0,
+ "1517": 442918400.0,
+ "1518": 442918400.0,
+ "1519": 442918400.0,
+ "1520": 442918400.0,
+ "1521": 442918400.0,
+ "1522": 442918400.0,
+ "1523": 442918400.0,
+ "1524": 442918400.0,
+ "1525": 442918400.0,
+ "1526": 442918400.0,
+ "1527": 442918400.0,
+ "1528": 442918400.0,
+ "1529": 442918400.0,
+ "1530": 442918400.0,
+ "1531": 442918400.0,
+ "1532": 442918400.0,
+ "1533": 442918400.0,
+ "1534": 442918400.0,
+ "1535": 442918400.0,
+ "1536": 442918400.0,
+ "1537": 442918400.0,
+ "1538": 442918400.0,
+ "1539": 442918400.0,
+ "1540": 442918400.0,
+ "1541": 442918400.0,
+ "1542": 442918400.0,
+ "1543": 442918400.0,
+ "1544": 442918400.0,
+ "1545": 442918400.0,
+ "1546": 442918400.0,
+ "1547": 442918400.0,
+ "1548": 442918400.0,
+ "1549": 442918400.0,
+ "1550": 442918400.0,
+ "1551": 442918400.0,
+ "1552": 442918400.0,
+ "1553": 442918400.0,
+ "1554": 442918400.0,
+ "1555": 442918400.0,
+ "1556": 442918400.0,
+ "1557": 442918400.0,
+ "1558": 442918400.0,
+ "1559": 442918400.0,
+ "1560": 442918400.0,
+ "1561": 442918400.0,
+ "1562": 442918400.0,
+ "1563": 442918400.0,
+ "1564": 442918400.0,
+ "1565": 442918400.0,
+ "1566": 442918400.0,
+ "1567": 442918400.0,
+ "1568": 442918400.0,
+ "1569": 442918400.0,
+ "1570": 442918400.0,
+ "1571": 442918400.0,
+ "1572": 442918400.0,
+ "1573": 442918400.0,
+ "1574": 442918400.0,
+ "1575": 442918400.0,
+ "1576": 442918400.0,
+ "1577": 442918400.0,
+ "1578": 442918400.0,
+ "1579": 442918400.0,
+ "1580": 442918400.0,
+ "1581": 442918400.0,
+ "1582": 442918400.0,
+ "1583": 442918400.0,
+ "1584": 442918400.0,
+ "1585": 442918400.0,
+ "1586": 442918400.0,
+ "1587": 442918400.0,
+ "1588": 442918400.0,
+ "1589": 442918400.0,
+ "1590": 442918400.0,
+ "1591": 442918400.0,
+ "1592": 442918400.0,
+ "1593": 442918400.0,
+ "1594": 442918400.0,
+ "1595": 442918400.0,
+ "1596": 442918400.0,
+ "1597": 442918400.0,
+ "1598": 442918400.0,
+ "1599": 442918400.0,
+ "1600": 442918400.0,
+ "1601": 442918400.0,
+ "1602": 442918400.0,
+ "1603": 442918400.0,
+ "1604": 442918400.0,
+ "1605": 442918400.0,
+ "1606": 442918400.0,
+ "1607": 442918400.0,
+ "1608": 442918400.0,
+ "1609": 442918400.0,
+ "1610": 442918400.0,
+ "1611": 442918400.0,
+ "1612": 442918400.0,
+ "1613": 442918400.0,
+ "1614": 442918400.0,
+ "1615": 442918400.0,
+ "1616": 442918400.0,
+ "1617": 442918400.0,
+ "1618": 442918400.0,
+ "1619": 442918400.0,
+ "1620": 442918400.0,
+ "1621": 442918400.0,
+ "1622": 442918400.0,
+ "1623": 442918400.0,
+ "1624": 442918400.0,
+ "1625": 442918400.0,
+ "1626": 442918400.0,
+ "1627": 442918400.0,
+ "1628": 442918400.0,
+ "1629": 442918400.0,
+ "1630": 442918400.0,
+ "1631": 442918400.0,
+ "1632": 442918400.0,
+ "1633": 442918400.0,
+ "1634": 442918400.0,
+ "1635": 442918400.0,
+ "1636": 442918400.0,
+ "1637": 442918400.0,
+ "1638": 442918400.0,
+ "1639": 442918400.0,
+ "1640": 442918400.0,
+ "1641": 442918400.0,
+ "1642": 442918400.0,
+ "1643": 442918400.0,
+ "1644": 442918400.0,
+ "1645": 442918400.0,
+ "1646": 442918400.0,
+ "1647": 442918400.0,
+ "1648": 442918400.0,
+ "1649": 442918400.0,
+ "1650": 442918400.0,
+ "1651": 442918400.0,
+ "1652": 442918400.0,
+ "1653": 442918400.0,
+ "1654": 442918400.0,
+ "1655": 442918400.0,
+ "1656": 442918400.0,
+ "1657": 442918400.0,
+ "1658": 442918400.0,
+ "1659": 442918400.0,
+ "1660": 442918400.0,
+ "1661": 442918400.0,
+ "1662": 442918400.0,
+ "1663": 442918400.0,
+ "1664": 442918400.0,
+ "1665": 442918400.0,
+ "1666": 442918400.0,
+ "1667": 442918400.0,
+ "1668": 442918400.0,
+ "1669": 442918400.0,
+ "1670": 442918400.0,
+ "1671": 442918400.0,
+ "1672": 442918400.0,
+ "1673": 442918400.0,
+ "1674": 442918400.0,
+ "1675": 442918400.0,
+ "1676": 442918400.0,
+ "1677": 442918400.0,
+ "1678": 442918400.0,
+ "1679": 442918400.0,
+ "1680": 442918400.0,
+ "1681": 442918400.0,
+ "1682": 442918400.0,
+ "1683": 442918400.0,
+ "1684": 442918400.0,
+ "1685": 442918400.0,
+ "1686": 442918400.0,
+ "1687": 442918400.0,
+ "1688": 442918400.0,
+ "1689": 442918400.0,
+ "1690": 442918400.0,
+ "1691": 442918400.0,
+ "1692": 442918400.0,
+ "1693": 442918400.0,
+ "1694": 442918400.0,
+ "1695": 442918400.0,
+ "1696": 442918400.0,
+ "1697": 442918400.0,
+ "1698": 442918400.0,
+ "1699": 442918400.0,
+ "1700": 442918400.0,
+ "1701": 442918400.0,
+ "1702": 442918400.0,
+ "1703": 442918400.0,
+ "1704": 442918400.0,
+ "1705": 442918400.0,
+ "1706": 442918400.0,
+ "1707": 442918400.0,
+ "1708": 442918400.0,
+ "1709": 442918400.0,
+ "1710": 442918400.0,
+ "1711": 442918400.0,
+ "1712": 442918400.0,
+ "1713": 442918400.0,
+ "1714": 442918400.0,
+ "1715": 442918400.0,
+ "1716": 442918400.0,
+ "1717": 442918400.0,
+ "1718": 442918400.0,
+ "1719": 442918400.0,
+ "1720": 442918400.0,
+ "1721": 442918400.0,
+ "1722": 442918400.0,
+ "1723": 442918400.0,
+ "1724": 442918400.0,
+ "1725": 442918400.0,
+ "1726": 442918400.0,
+ "1727": 442918400.0,
+ "1728": 442918400.0,
+ "1729": 442918400.0,
+ "1730": 442918400.0,
+ "1731": 442918400.0,
+ "1732": 442918400.0,
+ "1733": 442918400.0,
+ "1734": 442918400.0,
+ "1735": 442918400.0,
+ "1736": 442918400.0,
+ "1737": 442918400.0,
+ "1738": 442918400.0,
+ "1739": 442918400.0,
+ "1740": 442918400.0,
+ "1741": 442918400.0,
+ "1742": 442918400.0,
+ "1743": 442918400.0,
+ "1744": 442918400.0,
+ "1745": 442918400.0,
+ "1746": 442918400.0,
+ "1747": 442918400.0,
+ "1748": 442918400.0,
+ "1749": 442918400.0,
+ "1750": 442918400.0,
+ "1751": 442918400.0,
+ "1752": 442918400.0,
+ "1753": 442918400.0,
+ "1754": 442918400.0,
+ "1755": 442918400.0,
+ "1756": 442918400.0,
+ "1757": 442918400.0,
+ "1758": 442918400.0,
+ "1759": 442918400.0,
+ "1760": 442918400.0,
+ "1761": 442918400.0,
+ "1762": 442918400.0,
+ "1763": 442918400.0,
+ "1764": 442918400.0,
+ "1765": 442918400.0,
+ "1766": 442918400.0,
+ "1767": 442918400.0,
+ "1768": 442918400.0,
+ "1769": 442918400.0,
+ "1770": 442918400.0,
+ "1771": 442918400.0,
+ "1772": 442918400.0,
+ "1773": 442918400.0,
+ "1774": 442918400.0,
+ "1775": 442918400.0,
+ "1776": 442918400.0,
+ "1777": 442918400.0,
+ "1778": 442918400.0,
+ "1779": 442918400.0,
+ "1780": 442918400.0,
+ "1781": 442918400.0,
+ "1782": 442918400.0,
+ "1783": 442918400.0,
+ "1784": 442918400.0,
+ "1785": 442918400.0,
+ "1786": 442918400.0,
+ "1787": 442918400.0,
+ "1788": 442918400.0,
+ "1789": 442918400.0,
+ "1790": 442918400.0,
+ "1791": 442918400.0,
+ "1792": 442918400.0,
+ "1793": 442918400.0,
+ "1794": 442918400.0,
+ "1795": 442918400.0,
+ "1796": 442918400.0,
+ "1797": 442918400.0,
+ "1798": 442918400.0,
+ "1799": 442918400.0,
+ "1800": 442918400.0,
+ "1801": 442918400.0,
+ "1802": 442918400.0,
+ "1803": 442918400.0,
+ "1804": 442918400.0,
+ "1805": 442918400.0,
+ "1806": 442918400.0,
+ "1807": 442918400.0,
+ "1808": 442918400.0,
+ "1809": 442918400.0,
+ "1810": 442918400.0,
+ "1811": 442918400.0,
+ "1812": 442918400.0,
+ "1813": 442918400.0,
+ "1814": 442918400.0,
+ "1815": 442918400.0,
+ "1816": 442918400.0,
+ "1817": 442918400.0,
+ "1818": 442918400.0,
+ "1819": 442918400.0,
+ "1820": 442918400.0,
+ "1821": 442918400.0,
+ "1822": 442918400.0,
+ "1823": 442918400.0,
+ "1824": 442918400.0,
+ "1825": 442918400.0,
+ "1826": 442918400.0,
+ "1827": 442918400.0,
+ "1828": 442918400.0,
+ "1829": 442918400.0,
+ "1830": 442918400.0,
+ "1831": 442918400.0,
+ "1832": 442918400.0,
+ "1833": 442918400.0,
+ "1834": 442918400.0,
+ "1835": 442918400.0,
+ "1836": 442918400.0,
+ "1837": 442918400.0,
+ "1838": 442918400.0,
+ "1839": 442918400.0,
+ "1840": 442918400.0,
+ "1841": 442918400.0,
+ "1842": 442918400.0,
+ "1843": 442918400.0,
+ "1844": 442918400.0,
+ "1845": 442918400.0,
+ "1846": 442918400.0,
+ "1847": 442918400.0,
+ "1848": 442918400.0,
+ "1849": 442918400.0,
+ "1850": 442918400.0,
+ "1851": 442918400.0,
+ "1852": 442918400.0,
+ "1853": 442918400.0,
+ "1854": 442918400.0,
+ "1855": 442918400.0,
+ "1856": 442918400.0,
+ "1857": 442918400.0,
+ "1858": 442918400.0,
+ "1859": 442918400.0,
+ "1860": 442918400.0,
+ "1861": 442918400.0,
+ "1862": 442918400.0,
+ "1863": 442918400.0,
+ "1864": 442918400.0,
+ "1865": 442918400.0,
+ "1866": 442918400.0,
+ "1867": 442918400.0,
+ "1868": 442918400.0,
+ "1869": 442918400.0,
+ "1870": 442918400.0,
+ "1871": 442918400.0,
+ "1872": 442918400.0,
+ "1873": 442918400.0,
+ "1874": 442918400.0,
+ "1875": 442918400.0,
+ "1876": 442918400.0,
+ "1877": 442918400.0,
+ "1878": 442918400.0,
+ "1879": 442918400.0,
+ "1880": 442918400.0,
+ "1881": 442918400.0,
+ "1882": 442918400.0,
+ "1883": 442918400.0,
+ "1884": 442918400.0,
+ "1885": 442918400.0,
+ "1886": 442918400.0,
+ "1887": 442918400.0,
+ "1888": 442918400.0,
+ "1889": 442918400.0,
+ "1890": 442918400.0,
+ "1891": 442918400.0,
+ "1892": 442918400.0,
+ "1893": 442918400.0,
+ "1894": 442918400.0,
+ "1895": 442918400.0,
+ "1896": 442918400.0,
+ "1897": 442918400.0,
+ "1898": 442918400.0,
+ "1899": 442918400.0,
+ "1900": 442918400.0,
+ "1901": 442918400.0,
+ "1902": 442918400.0,
+ "1903": 442918400.0,
+ "1904": 442918400.0,
+ "1905": 442918400.0,
+ "1906": 442918400.0,
+ "1907": 442918400.0,
+ "1908": 442918400.0,
+ "1909": 442918400.0,
+ "1910": 442918400.0,
+ "1911": 442918400.0,
+ "1912": 442918400.0,
+ "1913": 442918400.0,
+ "1914": 442918400.0,
+ "1915": 442918400.0,
+ "1916": 442918400.0,
+ "1917": 442918400.0,
+ "1918": 442918400.0,
+ "1919": 442918400.0,
+ "1920": 442918400.0,
+ "1921": 442918400.0,
+ "1922": 442918400.0,
+ "1923": 442918400.0,
+ "1924": 442918400.0,
+ "1925": 442918400.0,
+ "1926": 442918400.0,
+ "1927": 442918400.0,
+ "1928": 442918400.0,
+ "1929": 442918400.0,
+ "1930": 442918400.0,
+ "1931": 442918400.0,
+ "1932": 442918400.0,
+ "1933": 442918400.0,
+ "1934": 442918400.0,
+ "1935": 442918400.0,
+ "1936": 442918400.0,
+ "1937": 442918400.0,
+ "1938": 442918400.0,
+ "1939": 442918400.0,
+ "1940": 442918400.0,
+ "1941": 442918400.0,
+ "1942": 442918400.0,
+ "1943": 442918400.0,
+ "1944": 442918400.0,
+ "1945": 442918400.0,
+ "1946": 442918400.0,
+ "1947": 442918400.0,
+ "1948": 442918400.0,
+ "1949": 442918400.0,
+ "1950": 442918400.0,
+ "1951": 442918400.0,
+ "1952": 442918400.0,
+ "1953": 442918400.0,
+ "1954": 442918400.0,
+ "1955": 442918400.0,
+ "1956": 442918400.0,
+ "1957": 442918400.0,
+ "1958": 442918400.0,
+ "1959": 442918400.0,
+ "1960": 442918400.0,
+ "1961": 442918400.0,
+ "1962": 442918400.0,
+ "1963": 442918400.0,
+ "1964": 442918400.0,
+ "1965": 442918400.0,
+ "1966": 442918400.0,
+ "1967": 442918400.0,
+ "1968": 442918400.0,
+ "1969": 442918400.0,
+ "1970": 442918400.0,
+ "1971": 442918400.0,
+ "1972": 442918400.0,
+ "1973": 442918400.0,
+ "1974": 442918400.0,
+ "1975": 442918400.0,
+ "1976": 442918400.0,
+ "1977": 442918400.0,
+ "1978": 442918400.0,
+ "1979": 442918400.0,
+ "1980": 442918400.0,
+ "1981": 442918400.0,
+ "1982": 442918400.0,
+ "1983": 442918400.0,
+ "1984": 442918400.0,
+ "1985": 442918400.0,
+ "1986": 442918400.0,
+ "1987": 442918400.0,
+ "1988": 442918400.0,
+ "1989": 442918400.0,
+ "1990": 442918400.0,
+ "1991": 442918400.0,
+ "1992": 442918400.0,
+ "1993": 442918400.0,
+ "1994": 442918400.0,
+ "1995": 442918400.0,
+ "1996": 442918400.0,
+ "1997": 442918400.0,
+ "1998": 442918400.0,
+ "1999": 442918400.0,
+ "2000": 442918400.0
+ }
+ },
+ "mem-max-allocated-bytes": {
+ "start_step": 1,
+ "end_step": 2000,
+ "step_interval": 1,
+ "values": {
+ "1": 761183744.0,
+ "2": 849621504.0,
+ "3": 849621504.0,
+ "4": 849621504.0,
+ "5": 849621504.0,
+ "6": 849621504.0,
+ "7": 849621504.0,
+ "8": 849621504.0,
+ "9": 849621504.0,
+ "10": 849621504.0,
+ "11": 849621504.0,
+ "12": 849621504.0,
+ "13": 849621504.0,
+ "14": 849621504.0,
+ "15": 849621504.0,
+ "16": 849621504.0,
+ "17": 849621504.0,
+ "18": 849621504.0,
+ "19": 849621504.0,
+ "20": 849621504.0,
+ "21": 849621504.0,
+ "22": 849621504.0,
+ "23": 849621504.0,
+ "24": 849621504.0,
+ "25": 849621504.0,
+ "26": 849621504.0,
+ "27": 849621504.0,
+ "28": 849621504.0,
+ "29": 849621504.0,
+ "30": 849621504.0,
+ "31": 849621504.0,
+ "32": 849621504.0,
+ "33": 849621504.0,
+ "34": 849621504.0,
+ "35": 849621504.0,
+ "36": 849621504.0,
+ "37": 849621504.0,
+ "38": 849621504.0,
+ "39": 849621504.0,
+ "40": 849621504.0,
+ "41": 849621504.0,
+ "42": 849621504.0,
+ "43": 849621504.0,
+ "44": 849621504.0,
+ "45": 849621504.0,
+ "46": 849621504.0,
+ "47": 849621504.0,
+ "48": 849621504.0,
+ "49": 849621504.0,
+ "50": 849621504.0,
+ "51": 849621504.0,
+ "52": 849621504.0,
+ "53": 849621504.0,
+ "54": 849621504.0,
+ "55": 849621504.0,
+ "56": 849621504.0,
+ "57": 849621504.0,
+ "58": 849621504.0,
+ "59": 849621504.0,
+ "60": 849621504.0,
+ "61": 849621504.0,
+ "62": 849621504.0,
+ "63": 849621504.0,
+ "64": 849621504.0,
+ "65": 849621504.0,
+ "66": 849621504.0,
+ "67": 849621504.0,
+ "68": 849621504.0,
+ "69": 849621504.0,
+ "70": 849621504.0,
+ "71": 849621504.0,
+ "72": 849621504.0,
+ "73": 849621504.0,
+ "74": 849621504.0,
+ "75": 849621504.0,
+ "76": 849621504.0,
+ "77": 849621504.0,
+ "78": 849621504.0,
+ "79": 849621504.0,
+ "80": 849621504.0,
+ "81": 849621504.0,
+ "82": 849621504.0,
+ "83": 849621504.0,
+ "84": 849621504.0,
+ "85": 849621504.0,
+ "86": 849621504.0,
+ "87": 849621504.0,
+ "88": 849621504.0,
+ "89": 849621504.0,
+ "90": 849621504.0,
+ "91": 849621504.0,
+ "92": 849621504.0,
+ "93": 849621504.0,
+ "94": 849621504.0,
+ "95": 849621504.0,
+ "96": 849621504.0,
+ "97": 849621504.0,
+ "98": 849621504.0,
+ "99": 849621504.0,
+ "100": 849621504.0,
+ "101": 849621504.0,
+ "102": 849621504.0,
+ "103": 849621504.0,
+ "104": 849621504.0,
+ "105": 849621504.0,
+ "106": 849621504.0,
+ "107": 849621504.0,
+ "108": 849621504.0,
+ "109": 849621504.0,
+ "110": 849621504.0,
+ "111": 849621504.0,
+ "112": 849621504.0,
+ "113": 849621504.0,
+ "114": 849621504.0,
+ "115": 849621504.0,
+ "116": 849621504.0,
+ "117": 849621504.0,
+ "118": 849621504.0,
+ "119": 849621504.0,
+ "120": 849621504.0,
+ "121": 849621504.0,
+ "122": 849621504.0,
+ "123": 849621504.0,
+ "124": 849621504.0,
+ "125": 849621504.0,
+ "126": 849621504.0,
+ "127": 849621504.0,
+ "128": 849621504.0,
+ "129": 849621504.0,
+ "130": 849621504.0,
+ "131": 849621504.0,
+ "132": 849621504.0,
+ "133": 849621504.0,
+ "134": 849621504.0,
+ "135": 849621504.0,
+ "136": 849621504.0,
+ "137": 849621504.0,
+ "138": 849621504.0,
+ "139": 849621504.0,
+ "140": 849621504.0,
+ "141": 849621504.0,
+ "142": 849621504.0,
+ "143": 849621504.0,
+ "144": 849621504.0,
+ "145": 849621504.0,
+ "146": 849621504.0,
+ "147": 849621504.0,
+ "148": 849621504.0,
+ "149": 849621504.0,
+ "150": 849621504.0,
+ "151": 849621504.0,
+ "152": 849621504.0,
+ "153": 849621504.0,
+ "154": 849621504.0,
+ "155": 849621504.0,
+ "156": 849621504.0,
+ "157": 849621504.0,
+ "158": 849621504.0,
+ "159": 849621504.0,
+ "160": 849621504.0,
+ "161": 849621504.0,
+ "162": 849621504.0,
+ "163": 849621504.0,
+ "164": 849621504.0,
+ "165": 849621504.0,
+ "166": 849621504.0,
+ "167": 849621504.0,
+ "168": 849621504.0,
+ "169": 849621504.0,
+ "170": 849621504.0,
+ "171": 849621504.0,
+ "172": 849621504.0,
+ "173": 849621504.0,
+ "174": 849621504.0,
+ "175": 849621504.0,
+ "176": 849621504.0,
+ "177": 849621504.0,
+ "178": 849621504.0,
+ "179": 849621504.0,
+ "180": 849621504.0,
+ "181": 849621504.0,
+ "182": 849621504.0,
+ "183": 849621504.0,
+ "184": 849621504.0,
+ "185": 849621504.0,
+ "186": 849621504.0,
+ "187": 849621504.0,
+ "188": 849621504.0,
+ "189": 849621504.0,
+ "190": 849621504.0,
+ "191": 849621504.0,
+ "192": 849621504.0,
+ "193": 849621504.0,
+ "194": 849621504.0,
+ "195": 849621504.0,
+ "196": 849621504.0,
+ "197": 849621504.0,
+ "198": 849621504.0,
+ "199": 849621504.0,
+ "200": 849621504.0,
+ "201": 849621504.0,
+ "202": 849621504.0,
+ "203": 849621504.0,
+ "204": 849621504.0,
+ "205": 849621504.0,
+ "206": 849621504.0,
+ "207": 849621504.0,
+ "208": 849621504.0,
+ "209": 849621504.0,
+ "210": 849621504.0,
+ "211": 849621504.0,
+ "212": 849621504.0,
+ "213": 849621504.0,
+ "214": 849621504.0,
+ "215": 849621504.0,
+ "216": 849621504.0,
+ "217": 849621504.0,
+ "218": 849621504.0,
+ "219": 849621504.0,
+ "220": 849621504.0,
+ "221": 849621504.0,
+ "222": 849621504.0,
+ "223": 849621504.0,
+ "224": 849621504.0,
+ "225": 849621504.0,
+ "226": 849621504.0,
+ "227": 849621504.0,
+ "228": 849621504.0,
+ "229": 849621504.0,
+ "230": 849621504.0,
+ "231": 849621504.0,
+ "232": 849621504.0,
+ "233": 849621504.0,
+ "234": 849621504.0,
+ "235": 849621504.0,
+ "236": 849621504.0,
+ "237": 849621504.0,
+ "238": 849621504.0,
+ "239": 849621504.0,
+ "240": 849621504.0,
+ "241": 849621504.0,
+ "242": 849621504.0,
+ "243": 849621504.0,
+ "244": 849621504.0,
+ "245": 849621504.0,
+ "246": 849621504.0,
+ "247": 849621504.0,
+ "248": 849621504.0,
+ "249": 849621504.0,
+ "250": 849621504.0,
+ "251": 849621504.0,
+ "252": 849621504.0,
+ "253": 849621504.0,
+ "254": 849621504.0,
+ "255": 849621504.0,
+ "256": 849621504.0,
+ "257": 849621504.0,
+ "258": 849621504.0,
+ "259": 849621504.0,
+ "260": 849621504.0,
+ "261": 849621504.0,
+ "262": 849621504.0,
+ "263": 849621504.0,
+ "264": 849621504.0,
+ "265": 849621504.0,
+ "266": 849621504.0,
+ "267": 849621504.0,
+ "268": 849621504.0,
+ "269": 849621504.0,
+ "270": 849621504.0,
+ "271": 849621504.0,
+ "272": 849621504.0,
+ "273": 849621504.0,
+ "274": 849621504.0,
+ "275": 849621504.0,
+ "276": 849621504.0,
+ "277": 849621504.0,
+ "278": 849621504.0,
+ "279": 849621504.0,
+ "280": 849621504.0,
+ "281": 849621504.0,
+ "282": 849621504.0,
+ "283": 849621504.0,
+ "284": 849621504.0,
+ "285": 849621504.0,
+ "286": 849621504.0,
+ "287": 849621504.0,
+ "288": 849621504.0,
+ "289": 849621504.0,
+ "290": 849621504.0,
+ "291": 849621504.0,
+ "292": 849621504.0,
+ "293": 849621504.0,
+ "294": 849621504.0,
+ "295": 849621504.0,
+ "296": 849621504.0,
+ "297": 849621504.0,
+ "298": 849621504.0,
+ "299": 849621504.0,
+ "300": 849621504.0,
+ "301": 849621504.0,
+ "302": 849621504.0,
+ "303": 849621504.0,
+ "304": 849621504.0,
+ "305": 849621504.0,
+ "306": 849621504.0,
+ "307": 849621504.0,
+ "308": 849621504.0,
+ "309": 849621504.0,
+ "310": 849621504.0,
+ "311": 849621504.0,
+ "312": 849621504.0,
+ "313": 849621504.0,
+ "314": 849621504.0,
+ "315": 849621504.0,
+ "316": 849621504.0,
+ "317": 849621504.0,
+ "318": 849621504.0,
+ "319": 849621504.0,
+ "320": 849621504.0,
+ "321": 849621504.0,
+ "322": 849621504.0,
+ "323": 849621504.0,
+ "324": 849621504.0,
+ "325": 849621504.0,
+ "326": 849621504.0,
+ "327": 849621504.0,
+ "328": 849621504.0,
+ "329": 849621504.0,
+ "330": 849621504.0,
+ "331": 849621504.0,
+ "332": 849621504.0,
+ "333": 849621504.0,
+ "334": 849621504.0,
+ "335": 849621504.0,
+ "336": 849621504.0,
+ "337": 849621504.0,
+ "338": 849621504.0,
+ "339": 849621504.0,
+ "340": 849621504.0,
+ "341": 849621504.0,
+ "342": 849621504.0,
+ "343": 849621504.0,
+ "344": 849621504.0,
+ "345": 849621504.0,
+ "346": 849621504.0,
+ "347": 849621504.0,
+ "348": 849621504.0,
+ "349": 849621504.0,
+ "350": 849621504.0,
+ "351": 849621504.0,
+ "352": 849621504.0,
+ "353": 849621504.0,
+ "354": 849621504.0,
+ "355": 849621504.0,
+ "356": 849621504.0,
+ "357": 849621504.0,
+ "358": 849621504.0,
+ "359": 849621504.0,
+ "360": 849621504.0,
+ "361": 849621504.0,
+ "362": 849621504.0,
+ "363": 849621504.0,
+ "364": 849621504.0,
+ "365": 849621504.0,
+ "366": 849621504.0,
+ "367": 849621504.0,
+ "368": 849621504.0,
+ "369": 849621504.0,
+ "370": 849621504.0,
+ "371": 849621504.0,
+ "372": 849621504.0,
+ "373": 849621504.0,
+ "374": 849621504.0,
+ "375": 849621504.0,
+ "376": 849621504.0,
+ "377": 849621504.0,
+ "378": 849621504.0,
+ "379": 849621504.0,
+ "380": 849621504.0,
+ "381": 849621504.0,
+ "382": 849621504.0,
+ "383": 849621504.0,
+ "384": 849621504.0,
+ "385": 849621504.0,
+ "386": 849621504.0,
+ "387": 849621504.0,
+ "388": 849621504.0,
+ "389": 849621504.0,
+ "390": 849621504.0,
+ "391": 849621504.0,
+ "392": 849621504.0,
+ "393": 849621504.0,
+ "394": 849621504.0,
+ "395": 849621504.0,
+ "396": 849621504.0,
+ "397": 849621504.0,
+ "398": 849621504.0,
+ "399": 849621504.0,
+ "400": 849621504.0,
+ "401": 849621504.0,
+ "402": 849621504.0,
+ "403": 849621504.0,
+ "404": 849621504.0,
+ "405": 849621504.0,
+ "406": 849621504.0,
+ "407": 849621504.0,
+ "408": 849621504.0,
+ "409": 849621504.0,
+ "410": 849621504.0,
+ "411": 849621504.0,
+ "412": 849621504.0,
+ "413": 849621504.0,
+ "414": 849621504.0,
+ "415": 849621504.0,
+ "416": 849621504.0,
+ "417": 849621504.0,
+ "418": 849621504.0,
+ "419": 849621504.0,
+ "420": 849621504.0,
+ "421": 849621504.0,
+ "422": 849621504.0,
+ "423": 849621504.0,
+ "424": 849621504.0,
+ "425": 849621504.0,
+ "426": 849621504.0,
+ "427": 849621504.0,
+ "428": 849621504.0,
+ "429": 849621504.0,
+ "430": 849621504.0,
+ "431": 849621504.0,
+ "432": 849621504.0,
+ "433": 849621504.0,
+ "434": 849621504.0,
+ "435": 849621504.0,
+ "436": 849621504.0,
+ "437": 849621504.0,
+ "438": 849621504.0,
+ "439": 849621504.0,
+ "440": 849621504.0,
+ "441": 849621504.0,
+ "442": 849621504.0,
+ "443": 849621504.0,
+ "444": 849621504.0,
+ "445": 849621504.0,
+ "446": 849621504.0,
+ "447": 849621504.0,
+ "448": 849621504.0,
+ "449": 849621504.0,
+ "450": 849621504.0,
+ "451": 849621504.0,
+ "452": 849621504.0,
+ "453": 849621504.0,
+ "454": 849621504.0,
+ "455": 849621504.0,
+ "456": 849621504.0,
+ "457": 849621504.0,
+ "458": 849621504.0,
+ "459": 849621504.0,
+ "460": 849621504.0,
+ "461": 849621504.0,
+ "462": 849621504.0,
+ "463": 849621504.0,
+ "464": 849621504.0,
+ "465": 849621504.0,
+ "466": 849621504.0,
+ "467": 849621504.0,
+ "468": 849621504.0,
+ "469": 849621504.0,
+ "470": 849621504.0,
+ "471": 849621504.0,
+ "472": 849621504.0,
+ "473": 849621504.0,
+ "474": 849621504.0,
+ "475": 849621504.0,
+ "476": 849621504.0,
+ "477": 849621504.0,
+ "478": 849621504.0,
+ "479": 849621504.0,
+ "480": 849621504.0,
+ "481": 849621504.0,
+ "482": 849621504.0,
+ "483": 849621504.0,
+ "484": 849621504.0,
+ "485": 849621504.0,
+ "486": 849621504.0,
+ "487": 849621504.0,
+ "488": 849621504.0,
+ "489": 849621504.0,
+ "490": 849621504.0,
+ "491": 849621504.0,
+ "492": 849621504.0,
+ "493": 849621504.0,
+ "494": 849621504.0,
+ "495": 849621504.0,
+ "496": 849621504.0,
+ "497": 849621504.0,
+ "498": 849621504.0,
+ "499": 849621504.0,
+ "500": 849621504.0,
+ "501": 849621504.0,
+ "502": 849621504.0,
+ "503": 849621504.0,
+ "504": 849621504.0,
+ "505": 849621504.0,
+ "506": 849621504.0,
+ "507": 849621504.0,
+ "508": 849621504.0,
+ "509": 849621504.0,
+ "510": 849621504.0,
+ "511": 849621504.0,
+ "512": 849621504.0,
+ "513": 849621504.0,
+ "514": 849621504.0,
+ "515": 849621504.0,
+ "516": 849621504.0,
+ "517": 849621504.0,
+ "518": 849621504.0,
+ "519": 849621504.0,
+ "520": 849621504.0,
+ "521": 849621504.0,
+ "522": 849621504.0,
+ "523": 849621504.0,
+ "524": 849621504.0,
+ "525": 849621504.0,
+ "526": 849621504.0,
+ "527": 849621504.0,
+ "528": 849621504.0,
+ "529": 849621504.0,
+ "530": 849621504.0,
+ "531": 849621504.0,
+ "532": 849621504.0,
+ "533": 849621504.0,
+ "534": 849621504.0,
+ "535": 849621504.0,
+ "536": 849621504.0,
+ "537": 849621504.0,
+ "538": 849621504.0,
+ "539": 849621504.0,
+ "540": 849621504.0,
+ "541": 849621504.0,
+ "542": 849621504.0,
+ "543": 849621504.0,
+ "544": 849621504.0,
+ "545": 849621504.0,
+ "546": 849621504.0,
+ "547": 849621504.0,
+ "548": 849621504.0,
+ "549": 849621504.0,
+ "550": 849621504.0,
+ "551": 849621504.0,
+ "552": 849621504.0,
+ "553": 849621504.0,
+ "554": 849621504.0,
+ "555": 849621504.0,
+ "556": 849621504.0,
+ "557": 849621504.0,
+ "558": 849621504.0,
+ "559": 849621504.0,
+ "560": 849621504.0,
+ "561": 849621504.0,
+ "562": 849621504.0,
+ "563": 849621504.0,
+ "564": 849621504.0,
+ "565": 849621504.0,
+ "566": 849621504.0,
+ "567": 849621504.0,
+ "568": 849621504.0,
+ "569": 849621504.0,
+ "570": 849621504.0,
+ "571": 849621504.0,
+ "572": 849621504.0,
+ "573": 849621504.0,
+ "574": 849621504.0,
+ "575": 849621504.0,
+ "576": 849621504.0,
+ "577": 849621504.0,
+ "578": 849621504.0,
+ "579": 849621504.0,
+ "580": 849621504.0,
+ "581": 849621504.0,
+ "582": 849621504.0,
+ "583": 849621504.0,
+ "584": 849621504.0,
+ "585": 849621504.0,
+ "586": 849621504.0,
+ "587": 849621504.0,
+ "588": 849621504.0,
+ "589": 849621504.0,
+ "590": 849621504.0,
+ "591": 849621504.0,
+ "592": 849621504.0,
+ "593": 849621504.0,
+ "594": 849621504.0,
+ "595": 849621504.0,
+ "596": 849621504.0,
+ "597": 849621504.0,
+ "598": 849621504.0,
+ "599": 849621504.0,
+ "600": 849621504.0,
+ "601": 849621504.0,
+ "602": 849621504.0,
+ "603": 849621504.0,
+ "604": 849621504.0,
+ "605": 849621504.0,
+ "606": 849621504.0,
+ "607": 849621504.0,
+ "608": 849621504.0,
+ "609": 849621504.0,
+ "610": 849621504.0,
+ "611": 849621504.0,
+ "612": 849621504.0,
+ "613": 849621504.0,
+ "614": 849621504.0,
+ "615": 849621504.0,
+ "616": 849621504.0,
+ "617": 849621504.0,
+ "618": 849621504.0,
+ "619": 849621504.0,
+ "620": 849621504.0,
+ "621": 849621504.0,
+ "622": 849621504.0,
+ "623": 849621504.0,
+ "624": 849621504.0,
+ "625": 849621504.0,
+ "626": 849621504.0,
+ "627": 849621504.0,
+ "628": 849621504.0,
+ "629": 849621504.0,
+ "630": 849621504.0,
+ "631": 849621504.0,
+ "632": 849621504.0,
+ "633": 849621504.0,
+ "634": 849621504.0,
+ "635": 849621504.0,
+ "636": 849621504.0,
+ "637": 849621504.0,
+ "638": 849621504.0,
+ "639": 849621504.0,
+ "640": 849621504.0,
+ "641": 849621504.0,
+ "642": 849621504.0,
+ "643": 849621504.0,
+ "644": 849621504.0,
+ "645": 849621504.0,
+ "646": 849621504.0,
+ "647": 849621504.0,
+ "648": 849621504.0,
+ "649": 849621504.0,
+ "650": 849621504.0,
+ "651": 849621504.0,
+ "652": 849621504.0,
+ "653": 849621504.0,
+ "654": 849621504.0,
+ "655": 849621504.0,
+ "656": 849621504.0,
+ "657": 849621504.0,
+ "658": 849621504.0,
+ "659": 849621504.0,
+ "660": 849621504.0,
+ "661": 849621504.0,
+ "662": 849621504.0,
+ "663": 849621504.0,
+ "664": 849621504.0,
+ "665": 849621504.0,
+ "666": 849621504.0,
+ "667": 849621504.0,
+ "668": 849621504.0,
+ "669": 849621504.0,
+ "670": 849621504.0,
+ "671": 849621504.0,
+ "672": 849621504.0,
+ "673": 849621504.0,
+ "674": 849621504.0,
+ "675": 849621504.0,
+ "676": 849621504.0,
+ "677": 849621504.0,
+ "678": 849621504.0,
+ "679": 849621504.0,
+ "680": 849621504.0,
+ "681": 849621504.0,
+ "682": 849621504.0,
+ "683": 849621504.0,
+ "684": 849621504.0,
+ "685": 849621504.0,
+ "686": 849621504.0,
+ "687": 849621504.0,
+ "688": 849621504.0,
+ "689": 849621504.0,
+ "690": 849621504.0,
+ "691": 849621504.0,
+ "692": 849621504.0,
+ "693": 849621504.0,
+ "694": 849621504.0,
+ "695": 849621504.0,
+ "696": 849621504.0,
+ "697": 849621504.0,
+ "698": 849621504.0,
+ "699": 849621504.0,
+ "700": 849621504.0,
+ "701": 849621504.0,
+ "702": 849621504.0,
+ "703": 849621504.0,
+ "704": 849621504.0,
+ "705": 849621504.0,
+ "706": 849621504.0,
+ "707": 849621504.0,
+ "708": 849621504.0,
+ "709": 849621504.0,
+ "710": 849621504.0,
+ "711": 849621504.0,
+ "712": 849621504.0,
+ "713": 849621504.0,
+ "714": 849621504.0,
+ "715": 849621504.0,
+ "716": 849621504.0,
+ "717": 849621504.0,
+ "718": 849621504.0,
+ "719": 849621504.0,
+ "720": 849621504.0,
+ "721": 849621504.0,
+ "722": 849621504.0,
+ "723": 849621504.0,
+ "724": 849621504.0,
+ "725": 849621504.0,
+ "726": 849621504.0,
+ "727": 849621504.0,
+ "728": 849621504.0,
+ "729": 849621504.0,
+ "730": 849621504.0,
+ "731": 849621504.0,
+ "732": 849621504.0,
+ "733": 849621504.0,
+ "734": 849621504.0,
+ "735": 849621504.0,
+ "736": 849621504.0,
+ "737": 849621504.0,
+ "738": 849621504.0,
+ "739": 849621504.0,
+ "740": 849621504.0,
+ "741": 849621504.0,
+ "742": 849621504.0,
+ "743": 849621504.0,
+ "744": 849621504.0,
+ "745": 849621504.0,
+ "746": 849621504.0,
+ "747": 849621504.0,
+ "748": 849621504.0,
+ "749": 849621504.0,
+ "750": 849621504.0,
+ "751": 849621504.0,
+ "752": 849621504.0,
+ "753": 849621504.0,
+ "754": 849621504.0,
+ "755": 849621504.0,
+ "756": 849621504.0,
+ "757": 849621504.0,
+ "758": 849621504.0,
+ "759": 849621504.0,
+ "760": 849621504.0,
+ "761": 849621504.0,
+ "762": 849621504.0,
+ "763": 849621504.0,
+ "764": 849621504.0,
+ "765": 849621504.0,
+ "766": 849621504.0,
+ "767": 849621504.0,
+ "768": 849621504.0,
+ "769": 849621504.0,
+ "770": 849621504.0,
+ "771": 849621504.0,
+ "772": 849621504.0,
+ "773": 849621504.0,
+ "774": 849621504.0,
+ "775": 849621504.0,
+ "776": 849621504.0,
+ "777": 849621504.0,
+ "778": 849621504.0,
+ "779": 849621504.0,
+ "780": 849621504.0,
+ "781": 849621504.0,
+ "782": 849621504.0,
+ "783": 849621504.0,
+ "784": 849621504.0,
+ "785": 849621504.0,
+ "786": 849621504.0,
+ "787": 849621504.0,
+ "788": 849621504.0,
+ "789": 849621504.0,
+ "790": 849621504.0,
+ "791": 849621504.0,
+ "792": 849621504.0,
+ "793": 849621504.0,
+ "794": 849621504.0,
+ "795": 849621504.0,
+ "796": 849621504.0,
+ "797": 849621504.0,
+ "798": 849621504.0,
+ "799": 849621504.0,
+ "800": 849621504.0,
+ "801": 849621504.0,
+ "802": 849621504.0,
+ "803": 849621504.0,
+ "804": 849621504.0,
+ "805": 849621504.0,
+ "806": 849621504.0,
+ "807": 849621504.0,
+ "808": 849621504.0,
+ "809": 849621504.0,
+ "810": 849621504.0,
+ "811": 849621504.0,
+ "812": 849621504.0,
+ "813": 849621504.0,
+ "814": 849621504.0,
+ "815": 849621504.0,
+ "816": 849621504.0,
+ "817": 849621504.0,
+ "818": 849621504.0,
+ "819": 849621504.0,
+ "820": 849621504.0,
+ "821": 849621504.0,
+ "822": 849621504.0,
+ "823": 849621504.0,
+ "824": 849621504.0,
+ "825": 849621504.0,
+ "826": 849621504.0,
+ "827": 849621504.0,
+ "828": 849621504.0,
+ "829": 849621504.0,
+ "830": 849621504.0,
+ "831": 849621504.0,
+ "832": 849621504.0,
+ "833": 849621504.0,
+ "834": 849621504.0,
+ "835": 849621504.0,
+ "836": 849621504.0,
+ "837": 849621504.0,
+ "838": 849621504.0,
+ "839": 849621504.0,
+ "840": 849621504.0,
+ "841": 849621504.0,
+ "842": 849621504.0,
+ "843": 849621504.0,
+ "844": 849621504.0,
+ "845": 849621504.0,
+ "846": 849621504.0,
+ "847": 849621504.0,
+ "848": 849621504.0,
+ "849": 849621504.0,
+ "850": 849621504.0,
+ "851": 849621504.0,
+ "852": 849621504.0,
+ "853": 849621504.0,
+ "854": 849621504.0,
+ "855": 849621504.0,
+ "856": 849621504.0,
+ "857": 849621504.0,
+ "858": 849621504.0,
+ "859": 849621504.0,
+ "860": 849621504.0,
+ "861": 849621504.0,
+ "862": 849621504.0,
+ "863": 849621504.0,
+ "864": 849621504.0,
+ "865": 849621504.0,
+ "866": 849621504.0,
+ "867": 849621504.0,
+ "868": 849621504.0,
+ "869": 849621504.0,
+ "870": 849621504.0,
+ "871": 849621504.0,
+ "872": 849621504.0,
+ "873": 849621504.0,
+ "874": 849621504.0,
+ "875": 849621504.0,
+ "876": 849621504.0,
+ "877": 849621504.0,
+ "878": 849621504.0,
+ "879": 849621504.0,
+ "880": 849621504.0,
+ "881": 849621504.0,
+ "882": 849621504.0,
+ "883": 849621504.0,
+ "884": 849621504.0,
+ "885": 849621504.0,
+ "886": 849621504.0,
+ "887": 849621504.0,
+ "888": 849621504.0,
+ "889": 849621504.0,
+ "890": 849621504.0,
+ "891": 849621504.0,
+ "892": 849621504.0,
+ "893": 849621504.0,
+ "894": 849621504.0,
+ "895": 849621504.0,
+ "896": 849621504.0,
+ "897": 849621504.0,
+ "898": 849621504.0,
+ "899": 849621504.0,
+ "900": 849621504.0,
+ "901": 849621504.0,
+ "902": 849621504.0,
+ "903": 849621504.0,
+ "904": 849621504.0,
+ "905": 849621504.0,
+ "906": 849621504.0,
+ "907": 849621504.0,
+ "908": 849621504.0,
+ "909": 849621504.0,
+ "910": 849621504.0,
+ "911": 849621504.0,
+ "912": 849621504.0,
+ "913": 849621504.0,
+ "914": 849621504.0,
+ "915": 849621504.0,
+ "916": 849621504.0,
+ "917": 849621504.0,
+ "918": 849621504.0,
+ "919": 849621504.0,
+ "920": 849621504.0,
+ "921": 849621504.0,
+ "922": 849621504.0,
+ "923": 849621504.0,
+ "924": 849621504.0,
+ "925": 849621504.0,
+ "926": 849621504.0,
+ "927": 849621504.0,
+ "928": 849621504.0,
+ "929": 849621504.0,
+ "930": 849621504.0,
+ "931": 849621504.0,
+ "932": 849621504.0,
+ "933": 849621504.0,
+ "934": 849621504.0,
+ "935": 849621504.0,
+ "936": 849621504.0,
+ "937": 849621504.0,
+ "938": 849621504.0,
+ "939": 849621504.0,
+ "940": 849621504.0,
+ "941": 849621504.0,
+ "942": 849621504.0,
+ "943": 849621504.0,
+ "944": 849621504.0,
+ "945": 849621504.0,
+ "946": 849621504.0,
+ "947": 849621504.0,
+ "948": 849621504.0,
+ "949": 849621504.0,
+ "950": 849621504.0,
+ "951": 849621504.0,
+ "952": 849621504.0,
+ "953": 849621504.0,
+ "954": 849621504.0,
+ "955": 849621504.0,
+ "956": 849621504.0,
+ "957": 849621504.0,
+ "958": 849621504.0,
+ "959": 849621504.0,
+ "960": 849621504.0,
+ "961": 849621504.0,
+ "962": 849621504.0,
+ "963": 849621504.0,
+ "964": 849621504.0,
+ "965": 849621504.0,
+ "966": 849621504.0,
+ "967": 849621504.0,
+ "968": 849621504.0,
+ "969": 849621504.0,
+ "970": 849621504.0,
+ "971": 849621504.0,
+ "972": 849621504.0,
+ "973": 849621504.0,
+ "974": 849621504.0,
+ "975": 849621504.0,
+ "976": 849621504.0,
+ "977": 849621504.0,
+ "978": 849621504.0,
+ "979": 849621504.0,
+ "980": 849621504.0,
+ "981": 849621504.0,
+ "982": 849621504.0,
+ "983": 849621504.0,
+ "984": 849621504.0,
+ "985": 849621504.0,
+ "986": 849621504.0,
+ "987": 849621504.0,
+ "988": 849621504.0,
+ "989": 849621504.0,
+ "990": 849621504.0,
+ "991": 849621504.0,
+ "992": 849621504.0,
+ "993": 849621504.0,
+ "994": 849621504.0,
+ "995": 849621504.0,
+ "996": 849621504.0,
+ "997": 849621504.0,
+ "998": 849621504.0,
+ "999": 849621504.0,
+ "1000": 849621504.0,
+ "1001": 849621504.0,
+ "1002": 849621504.0,
+ "1003": 849621504.0,
+ "1004": 849621504.0,
+ "1005": 849621504.0,
+ "1006": 849621504.0,
+ "1007": 849621504.0,
+ "1008": 849621504.0,
+ "1009": 849621504.0,
+ "1010": 849621504.0,
+ "1011": 849621504.0,
+ "1012": 849621504.0,
+ "1013": 849621504.0,
+ "1014": 849621504.0,
+ "1015": 849621504.0,
+ "1016": 849621504.0,
+ "1017": 849621504.0,
+ "1018": 849621504.0,
+ "1019": 849621504.0,
+ "1020": 849621504.0,
+ "1021": 849621504.0,
+ "1022": 849621504.0,
+ "1023": 849621504.0,
+ "1024": 849621504.0,
+ "1025": 849621504.0,
+ "1026": 849621504.0,
+ "1027": 849621504.0,
+ "1028": 849621504.0,
+ "1029": 849621504.0,
+ "1030": 849621504.0,
+ "1031": 849621504.0,
+ "1032": 849621504.0,
+ "1033": 849621504.0,
+ "1034": 849621504.0,
+ "1035": 849621504.0,
+ "1036": 849621504.0,
+ "1037": 849621504.0,
+ "1038": 849621504.0,
+ "1039": 849621504.0,
+ "1040": 849621504.0,
+ "1041": 849621504.0,
+ "1042": 849621504.0,
+ "1043": 849621504.0,
+ "1044": 849621504.0,
+ "1045": 849621504.0,
+ "1046": 849621504.0,
+ "1047": 849621504.0,
+ "1048": 849621504.0,
+ "1049": 849621504.0,
+ "1050": 849621504.0,
+ "1051": 849621504.0,
+ "1052": 849621504.0,
+ "1053": 849621504.0,
+ "1054": 849621504.0,
+ "1055": 849621504.0,
+ "1056": 849621504.0,
+ "1057": 849621504.0,
+ "1058": 849621504.0,
+ "1059": 849621504.0,
+ "1060": 849621504.0,
+ "1061": 849621504.0,
+ "1062": 849621504.0,
+ "1063": 849621504.0,
+ "1064": 849621504.0,
+ "1065": 849621504.0,
+ "1066": 849621504.0,
+ "1067": 849621504.0,
+ "1068": 849621504.0,
+ "1069": 849621504.0,
+ "1070": 849621504.0,
+ "1071": 849621504.0,
+ "1072": 849621504.0,
+ "1073": 849621504.0,
+ "1074": 849621504.0,
+ "1075": 849621504.0,
+ "1076": 849621504.0,
+ "1077": 849621504.0,
+ "1078": 849621504.0,
+ "1079": 849621504.0,
+ "1080": 849621504.0,
+ "1081": 849621504.0,
+ "1082": 849621504.0,
+ "1083": 849621504.0,
+ "1084": 849621504.0,
+ "1085": 849621504.0,
+ "1086": 849621504.0,
+ "1087": 849621504.0,
+ "1088": 849621504.0,
+ "1089": 849621504.0,
+ "1090": 849621504.0,
+ "1091": 849621504.0,
+ "1092": 849621504.0,
+ "1093": 849621504.0,
+ "1094": 849621504.0,
+ "1095": 849621504.0,
+ "1096": 849621504.0,
+ "1097": 849621504.0,
+ "1098": 849621504.0,
+ "1099": 849621504.0,
+ "1100": 849621504.0,
+ "1101": 849621504.0,
+ "1102": 849621504.0,
+ "1103": 849621504.0,
+ "1104": 849621504.0,
+ "1105": 849621504.0,
+ "1106": 849621504.0,
+ "1107": 849621504.0,
+ "1108": 849621504.0,
+ "1109": 849621504.0,
+ "1110": 849621504.0,
+ "1111": 849621504.0,
+ "1112": 849621504.0,
+ "1113": 849621504.0,
+ "1114": 849621504.0,
+ "1115": 849621504.0,
+ "1116": 849621504.0,
+ "1117": 849621504.0,
+ "1118": 849621504.0,
+ "1119": 849621504.0,
+ "1120": 849621504.0,
+ "1121": 849621504.0,
+ "1122": 849621504.0,
+ "1123": 849621504.0,
+ "1124": 849621504.0,
+ "1125": 849621504.0,
+ "1126": 849621504.0,
+ "1127": 849621504.0,
+ "1128": 849621504.0,
+ "1129": 849621504.0,
+ "1130": 849621504.0,
+ "1131": 849621504.0,
+ "1132": 849621504.0,
+ "1133": 849621504.0,
+ "1134": 849621504.0,
+ "1135": 849621504.0,
+ "1136": 849621504.0,
+ "1137": 849621504.0,
+ "1138": 849621504.0,
+ "1139": 849621504.0,
+ "1140": 849621504.0,
+ "1141": 849621504.0,
+ "1142": 849621504.0,
+ "1143": 849621504.0,
+ "1144": 849621504.0,
+ "1145": 849621504.0,
+ "1146": 849621504.0,
+ "1147": 849621504.0,
+ "1148": 849621504.0,
+ "1149": 849621504.0,
+ "1150": 849621504.0,
+ "1151": 849621504.0,
+ "1152": 849621504.0,
+ "1153": 849621504.0,
+ "1154": 849621504.0,
+ "1155": 849621504.0,
+ "1156": 849621504.0,
+ "1157": 849621504.0,
+ "1158": 849621504.0,
+ "1159": 849621504.0,
+ "1160": 849621504.0,
+ "1161": 849621504.0,
+ "1162": 849621504.0,
+ "1163": 849621504.0,
+ "1164": 849621504.0,
+ "1165": 849621504.0,
+ "1166": 849621504.0,
+ "1167": 849621504.0,
+ "1168": 849621504.0,
+ "1169": 849621504.0,
+ "1170": 849621504.0,
+ "1171": 849621504.0,
+ "1172": 849621504.0,
+ "1173": 849621504.0,
+ "1174": 849621504.0,
+ "1175": 849621504.0,
+ "1176": 849621504.0,
+ "1177": 849621504.0,
+ "1178": 849621504.0,
+ "1179": 849621504.0,
+ "1180": 849621504.0,
+ "1181": 849621504.0,
+ "1182": 849621504.0,
+ "1183": 849621504.0,
+ "1184": 849621504.0,
+ "1185": 849621504.0,
+ "1186": 849621504.0,
+ "1187": 849621504.0,
+ "1188": 849621504.0,
+ "1189": 849621504.0,
+ "1190": 849621504.0,
+ "1191": 849621504.0,
+ "1192": 849621504.0,
+ "1193": 849621504.0,
+ "1194": 849621504.0,
+ "1195": 849621504.0,
+ "1196": 849621504.0,
+ "1197": 849621504.0,
+ "1198": 849621504.0,
+ "1199": 849621504.0,
+ "1200": 849621504.0,
+ "1201": 849621504.0,
+ "1202": 849621504.0,
+ "1203": 849621504.0,
+ "1204": 849621504.0,
+ "1205": 849621504.0,
+ "1206": 849621504.0,
+ "1207": 849621504.0,
+ "1208": 849621504.0,
+ "1209": 849621504.0,
+ "1210": 849621504.0,
+ "1211": 849621504.0,
+ "1212": 849621504.0,
+ "1213": 849621504.0,
+ "1214": 849621504.0,
+ "1215": 849621504.0,
+ "1216": 849621504.0,
+ "1217": 849621504.0,
+ "1218": 849621504.0,
+ "1219": 849621504.0,
+ "1220": 849621504.0,
+ "1221": 849621504.0,
+ "1222": 849621504.0,
+ "1223": 849621504.0,
+ "1224": 849621504.0,
+ "1225": 849621504.0,
+ "1226": 849621504.0,
+ "1227": 849621504.0,
+ "1228": 849621504.0,
+ "1229": 849621504.0,
+ "1230": 849621504.0,
+ "1231": 849621504.0,
+ "1232": 849621504.0,
+ "1233": 849621504.0,
+ "1234": 849621504.0,
+ "1235": 849621504.0,
+ "1236": 849621504.0,
+ "1237": 849621504.0,
+ "1238": 849621504.0,
+ "1239": 849621504.0,
+ "1240": 849621504.0,
+ "1241": 849621504.0,
+ "1242": 849621504.0,
+ "1243": 849621504.0,
+ "1244": 849621504.0,
+ "1245": 849621504.0,
+ "1246": 849621504.0,
+ "1247": 849621504.0,
+ "1248": 849621504.0,
+ "1249": 849621504.0,
+ "1250": 849621504.0,
+ "1251": 849621504.0,
+ "1252": 849621504.0,
+ "1253": 849621504.0,
+ "1254": 849621504.0,
+ "1255": 849621504.0,
+ "1256": 849621504.0,
+ "1257": 849621504.0,
+ "1258": 849621504.0,
+ "1259": 849621504.0,
+ "1260": 849621504.0,
+ "1261": 849621504.0,
+ "1262": 849621504.0,
+ "1263": 849621504.0,
+ "1264": 849621504.0,
+ "1265": 849621504.0,
+ "1266": 849621504.0,
+ "1267": 849621504.0,
+ "1268": 849621504.0,
+ "1269": 849621504.0,
+ "1270": 849621504.0,
+ "1271": 849621504.0,
+ "1272": 849621504.0,
+ "1273": 849621504.0,
+ "1274": 849621504.0,
+ "1275": 849621504.0,
+ "1276": 849621504.0,
+ "1277": 849621504.0,
+ "1278": 849621504.0,
+ "1279": 849621504.0,
+ "1280": 849621504.0,
+ "1281": 849621504.0,
+ "1282": 849621504.0,
+ "1283": 849621504.0,
+ "1284": 849621504.0,
+ "1285": 849621504.0,
+ "1286": 849621504.0,
+ "1287": 849621504.0,
+ "1288": 849621504.0,
+ "1289": 849621504.0,
+ "1290": 849621504.0,
+ "1291": 849621504.0,
+ "1292": 849621504.0,
+ "1293": 849621504.0,
+ "1294": 849621504.0,
+ "1295": 849621504.0,
+ "1296": 849621504.0,
+ "1297": 849621504.0,
+ "1298": 849621504.0,
+ "1299": 849621504.0,
+ "1300": 849621504.0,
+ "1301": 849621504.0,
+ "1302": 849621504.0,
+ "1303": 849621504.0,
+ "1304": 849621504.0,
+ "1305": 849621504.0,
+ "1306": 849621504.0,
+ "1307": 849621504.0,
+ "1308": 849621504.0,
+ "1309": 849621504.0,
+ "1310": 849621504.0,
+ "1311": 849621504.0,
+ "1312": 849621504.0,
+ "1313": 849621504.0,
+ "1314": 849621504.0,
+ "1315": 849621504.0,
+ "1316": 849621504.0,
+ "1317": 849621504.0,
+ "1318": 849621504.0,
+ "1319": 849621504.0,
+ "1320": 849621504.0,
+ "1321": 849621504.0,
+ "1322": 849621504.0,
+ "1323": 849621504.0,
+ "1324": 849621504.0,
+ "1325": 849621504.0,
+ "1326": 849621504.0,
+ "1327": 849621504.0,
+ "1328": 849621504.0,
+ "1329": 849621504.0,
+ "1330": 849621504.0,
+ "1331": 849621504.0,
+ "1332": 849621504.0,
+ "1333": 849621504.0,
+ "1334": 849621504.0,
+ "1335": 849621504.0,
+ "1336": 849621504.0,
+ "1337": 849621504.0,
+ "1338": 849621504.0,
+ "1339": 849621504.0,
+ "1340": 849621504.0,
+ "1341": 849621504.0,
+ "1342": 849621504.0,
+ "1343": 849621504.0,
+ "1344": 849621504.0,
+ "1345": 849621504.0,
+ "1346": 849621504.0,
+ "1347": 849621504.0,
+ "1348": 849621504.0,
+ "1349": 849621504.0,
+ "1350": 849621504.0,
+ "1351": 849621504.0,
+ "1352": 849621504.0,
+ "1353": 849621504.0,
+ "1354": 849621504.0,
+ "1355": 849621504.0,
+ "1356": 849621504.0,
+ "1357": 849621504.0,
+ "1358": 849621504.0,
+ "1359": 849621504.0,
+ "1360": 849621504.0,
+ "1361": 849621504.0,
+ "1362": 849621504.0,
+ "1363": 849621504.0,
+ "1364": 849621504.0,
+ "1365": 849621504.0,
+ "1366": 849621504.0,
+ "1367": 849621504.0,
+ "1368": 849621504.0,
+ "1369": 849621504.0,
+ "1370": 849621504.0,
+ "1371": 849621504.0,
+ "1372": 849621504.0,
+ "1373": 849621504.0,
+ "1374": 849621504.0,
+ "1375": 849621504.0,
+ "1376": 849621504.0,
+ "1377": 849621504.0,
+ "1378": 849621504.0,
+ "1379": 849621504.0,
+ "1380": 849621504.0,
+ "1381": 849621504.0,
+ "1382": 849621504.0,
+ "1383": 849621504.0,
+ "1384": 849621504.0,
+ "1385": 849621504.0,
+ "1386": 849621504.0,
+ "1387": 849621504.0,
+ "1388": 849621504.0,
+ "1389": 849621504.0,
+ "1390": 849621504.0,
+ "1391": 849621504.0,
+ "1392": 849621504.0,
+ "1393": 849621504.0,
+ "1394": 849621504.0,
+ "1395": 849621504.0,
+ "1396": 849621504.0,
+ "1397": 849621504.0,
+ "1398": 849621504.0,
+ "1399": 849621504.0,
+ "1400": 849621504.0,
+ "1401": 849621504.0,
+ "1402": 849621504.0,
+ "1403": 849621504.0,
+ "1404": 849621504.0,
+ "1405": 849621504.0,
+ "1406": 849621504.0,
+ "1407": 849621504.0,
+ "1408": 849621504.0,
+ "1409": 849621504.0,
+ "1410": 849621504.0,
+ "1411": 849621504.0,
+ "1412": 849621504.0,
+ "1413": 849621504.0,
+ "1414": 849621504.0,
+ "1415": 849621504.0,
+ "1416": 849621504.0,
+ "1417": 849621504.0,
+ "1418": 849621504.0,
+ "1419": 849621504.0,
+ "1420": 849621504.0,
+ "1421": 849621504.0,
+ "1422": 849621504.0,
+ "1423": 849621504.0,
+ "1424": 849621504.0,
+ "1425": 849621504.0,
+ "1426": 849621504.0,
+ "1427": 849621504.0,
+ "1428": 849621504.0,
+ "1429": 849621504.0,
+ "1430": 849621504.0,
+ "1431": 849621504.0,
+ "1432": 849621504.0,
+ "1433": 849621504.0,
+ "1434": 849621504.0,
+ "1435": 849621504.0,
+ "1436": 849621504.0,
+ "1437": 849621504.0,
+ "1438": 849621504.0,
+ "1439": 849621504.0,
+ "1440": 849621504.0,
+ "1441": 849621504.0,
+ "1442": 849621504.0,
+ "1443": 849621504.0,
+ "1444": 849621504.0,
+ "1445": 849621504.0,
+ "1446": 849621504.0,
+ "1447": 849621504.0,
+ "1448": 849621504.0,
+ "1449": 849621504.0,
+ "1450": 849621504.0,
+ "1451": 849621504.0,
+ "1452": 849621504.0,
+ "1453": 849621504.0,
+ "1454": 849621504.0,
+ "1455": 849621504.0,
+ "1456": 849621504.0,
+ "1457": 849621504.0,
+ "1458": 849621504.0,
+ "1459": 849621504.0,
+ "1460": 849621504.0,
+ "1461": 849621504.0,
+ "1462": 849621504.0,
+ "1463": 849621504.0,
+ "1464": 849621504.0,
+ "1465": 849621504.0,
+ "1466": 849621504.0,
+ "1467": 849621504.0,
+ "1468": 849621504.0,
+ "1469": 849621504.0,
+ "1470": 849621504.0,
+ "1471": 849621504.0,
+ "1472": 849621504.0,
+ "1473": 849621504.0,
+ "1474": 849621504.0,
+ "1475": 849621504.0,
+ "1476": 849621504.0,
+ "1477": 849621504.0,
+ "1478": 849621504.0,
+ "1479": 849621504.0,
+ "1480": 849621504.0,
+ "1481": 849621504.0,
+ "1482": 849621504.0,
+ "1483": 849621504.0,
+ "1484": 849621504.0,
+ "1485": 849621504.0,
+ "1486": 849621504.0,
+ "1487": 849621504.0,
+ "1488": 849621504.0,
+ "1489": 849621504.0,
+ "1490": 849621504.0,
+ "1491": 849621504.0,
+ "1492": 849621504.0,
+ "1493": 849621504.0,
+ "1494": 849621504.0,
+ "1495": 849621504.0,
+ "1496": 849621504.0,
+ "1497": 849621504.0,
+ "1498": 849621504.0,
+ "1499": 849621504.0,
+ "1500": 849621504.0,
+ "1501": 849621504.0,
+ "1502": 849621504.0,
+ "1503": 849621504.0,
+ "1504": 849621504.0,
+ "1505": 849621504.0,
+ "1506": 849621504.0,
+ "1507": 849621504.0,
+ "1508": 849621504.0,
+ "1509": 849621504.0,
+ "1510": 849621504.0,
+ "1511": 849621504.0,
+ "1512": 849621504.0,
+ "1513": 849621504.0,
+ "1514": 849621504.0,
+ "1515": 849621504.0,
+ "1516": 849621504.0,
+ "1517": 849621504.0,
+ "1518": 849621504.0,
+ "1519": 849621504.0,
+ "1520": 849621504.0,
+ "1521": 849621504.0,
+ "1522": 849621504.0,
+ "1523": 849621504.0,
+ "1524": 849621504.0,
+ "1525": 849621504.0,
+ "1526": 849621504.0,
+ "1527": 849621504.0,
+ "1528": 849621504.0,
+ "1529": 849621504.0,
+ "1530": 849621504.0,
+ "1531": 849621504.0,
+ "1532": 849621504.0,
+ "1533": 849621504.0,
+ "1534": 849621504.0,
+ "1535": 849621504.0,
+ "1536": 849621504.0,
+ "1537": 849621504.0,
+ "1538": 849621504.0,
+ "1539": 849621504.0,
+ "1540": 849621504.0,
+ "1541": 849621504.0,
+ "1542": 849621504.0,
+ "1543": 849621504.0,
+ "1544": 849621504.0,
+ "1545": 849621504.0,
+ "1546": 849621504.0,
+ "1547": 849621504.0,
+ "1548": 849621504.0,
+ "1549": 849621504.0,
+ "1550": 849621504.0,
+ "1551": 849621504.0,
+ "1552": 849621504.0,
+ "1553": 849621504.0,
+ "1554": 849621504.0,
+ "1555": 849621504.0,
+ "1556": 849621504.0,
+ "1557": 849621504.0,
+ "1558": 849621504.0,
+ "1559": 849621504.0,
+ "1560": 849621504.0,
+ "1561": 849621504.0,
+ "1562": 849621504.0,
+ "1563": 849621504.0,
+ "1564": 849621504.0,
+ "1565": 849621504.0,
+ "1566": 849621504.0,
+ "1567": 849621504.0,
+ "1568": 849621504.0,
+ "1569": 849621504.0,
+ "1570": 849621504.0,
+ "1571": 849621504.0,
+ "1572": 849621504.0,
+ "1573": 849621504.0,
+ "1574": 849621504.0,
+ "1575": 849621504.0,
+ "1576": 849621504.0,
+ "1577": 849621504.0,
+ "1578": 849621504.0,
+ "1579": 849621504.0,
+ "1580": 849621504.0,
+ "1581": 849621504.0,
+ "1582": 849621504.0,
+ "1583": 849621504.0,
+ "1584": 849621504.0,
+ "1585": 849621504.0,
+ "1586": 849621504.0,
+ "1587": 849621504.0,
+ "1588": 849621504.0,
+ "1589": 849621504.0,
+ "1590": 849621504.0,
+ "1591": 849621504.0,
+ "1592": 849621504.0,
+ "1593": 849621504.0,
+ "1594": 849621504.0,
+ "1595": 849621504.0,
+ "1596": 849621504.0,
+ "1597": 849621504.0,
+ "1598": 849621504.0,
+ "1599": 849621504.0,
+ "1600": 849621504.0,
+ "1601": 849621504.0,
+ "1602": 849621504.0,
+ "1603": 849621504.0,
+ "1604": 849621504.0,
+ "1605": 849621504.0,
+ "1606": 849621504.0,
+ "1607": 849621504.0,
+ "1608": 849621504.0,
+ "1609": 849621504.0,
+ "1610": 849621504.0,
+ "1611": 849621504.0,
+ "1612": 849621504.0,
+ "1613": 849621504.0,
+ "1614": 849621504.0,
+ "1615": 849621504.0,
+ "1616": 849621504.0,
+ "1617": 849621504.0,
+ "1618": 849621504.0,
+ "1619": 849621504.0,
+ "1620": 849621504.0,
+ "1621": 849621504.0,
+ "1622": 849621504.0,
+ "1623": 849621504.0,
+ "1624": 849621504.0,
+ "1625": 849621504.0,
+ "1626": 849621504.0,
+ "1627": 849621504.0,
+ "1628": 849621504.0,
+ "1629": 849621504.0,
+ "1630": 849621504.0,
+ "1631": 849621504.0,
+ "1632": 849621504.0,
+ "1633": 849621504.0,
+ "1634": 849621504.0,
+ "1635": 849621504.0,
+ "1636": 849621504.0,
+ "1637": 849621504.0,
+ "1638": 849621504.0,
+ "1639": 849621504.0,
+ "1640": 849621504.0,
+ "1641": 849621504.0,
+ "1642": 849621504.0,
+ "1643": 849621504.0,
+ "1644": 849621504.0,
+ "1645": 849621504.0,
+ "1646": 849621504.0,
+ "1647": 849621504.0,
+ "1648": 849621504.0,
+ "1649": 849621504.0,
+ "1650": 849621504.0,
+ "1651": 849621504.0,
+ "1652": 849621504.0,
+ "1653": 849621504.0,
+ "1654": 849621504.0,
+ "1655": 849621504.0,
+ "1656": 849621504.0,
+ "1657": 849621504.0,
+ "1658": 849621504.0,
+ "1659": 849621504.0,
+ "1660": 849621504.0,
+ "1661": 849621504.0,
+ "1662": 849621504.0,
+ "1663": 849621504.0,
+ "1664": 849621504.0,
+ "1665": 849621504.0,
+ "1666": 849621504.0,
+ "1667": 849621504.0,
+ "1668": 849621504.0,
+ "1669": 849621504.0,
+ "1670": 849621504.0,
+ "1671": 849621504.0,
+ "1672": 849621504.0,
+ "1673": 849621504.0,
+ "1674": 849621504.0,
+ "1675": 849621504.0,
+ "1676": 849621504.0,
+ "1677": 849621504.0,
+ "1678": 849621504.0,
+ "1679": 849621504.0,
+ "1680": 849621504.0,
+ "1681": 849621504.0,
+ "1682": 849621504.0,
+ "1683": 849621504.0,
+ "1684": 849621504.0,
+ "1685": 849621504.0,
+ "1686": 849621504.0,
+ "1687": 849621504.0,
+ "1688": 849621504.0,
+ "1689": 849621504.0,
+ "1690": 849621504.0,
+ "1691": 849621504.0,
+ "1692": 849621504.0,
+ "1693": 849621504.0,
+ "1694": 849621504.0,
+ "1695": 849621504.0,
+ "1696": 849621504.0,
+ "1697": 849621504.0,
+ "1698": 849621504.0,
+ "1699": 849621504.0,
+ "1700": 849621504.0,
+ "1701": 849621504.0,
+ "1702": 849621504.0,
+ "1703": 849621504.0,
+ "1704": 849621504.0,
+ "1705": 849621504.0,
+ "1706": 849621504.0,
+ "1707": 849621504.0,
+ "1708": 849621504.0,
+ "1709": 849621504.0,
+ "1710": 849621504.0,
+ "1711": 849621504.0,
+ "1712": 849621504.0,
+ "1713": 849621504.0,
+ "1714": 849621504.0,
+ "1715": 849621504.0,
+ "1716": 849621504.0,
+ "1717": 849621504.0,
+ "1718": 849621504.0,
+ "1719": 849621504.0,
+ "1720": 849621504.0,
+ "1721": 849621504.0,
+ "1722": 849621504.0,
+ "1723": 849621504.0,
+ "1724": 849621504.0,
+ "1725": 849621504.0,
+ "1726": 849621504.0,
+ "1727": 849621504.0,
+ "1728": 849621504.0,
+ "1729": 849621504.0,
+ "1730": 849621504.0,
+ "1731": 849621504.0,
+ "1732": 849621504.0,
+ "1733": 849621504.0,
+ "1734": 849621504.0,
+ "1735": 849621504.0,
+ "1736": 849621504.0,
+ "1737": 849621504.0,
+ "1738": 849621504.0,
+ "1739": 849621504.0,
+ "1740": 849621504.0,
+ "1741": 849621504.0,
+ "1742": 849621504.0,
+ "1743": 849621504.0,
+ "1744": 849621504.0,
+ "1745": 849621504.0,
+ "1746": 849621504.0,
+ "1747": 849621504.0,
+ "1748": 849621504.0,
+ "1749": 849621504.0,
+ "1750": 849621504.0,
+ "1751": 849621504.0,
+ "1752": 849621504.0,
+ "1753": 849621504.0,
+ "1754": 849621504.0,
+ "1755": 849621504.0,
+ "1756": 849621504.0,
+ "1757": 849621504.0,
+ "1758": 849621504.0,
+ "1759": 849621504.0,
+ "1760": 849621504.0,
+ "1761": 849621504.0,
+ "1762": 849621504.0,
+ "1763": 849621504.0,
+ "1764": 849621504.0,
+ "1765": 849621504.0,
+ "1766": 849621504.0,
+ "1767": 849621504.0,
+ "1768": 849621504.0,
+ "1769": 849621504.0,
+ "1770": 849621504.0,
+ "1771": 849621504.0,
+ "1772": 849621504.0,
+ "1773": 849621504.0,
+ "1774": 849621504.0,
+ "1775": 849621504.0,
+ "1776": 849621504.0,
+ "1777": 849621504.0,
+ "1778": 849621504.0,
+ "1779": 849621504.0,
+ "1780": 849621504.0,
+ "1781": 849621504.0,
+ "1782": 849621504.0,
+ "1783": 849621504.0,
+ "1784": 849621504.0,
+ "1785": 849621504.0,
+ "1786": 849621504.0,
+ "1787": 849621504.0,
+ "1788": 849621504.0,
+ "1789": 849621504.0,
+ "1790": 849621504.0,
+ "1791": 849621504.0,
+ "1792": 849621504.0,
+ "1793": 849621504.0,
+ "1794": 849621504.0,
+ "1795": 849621504.0,
+ "1796": 849621504.0,
+ "1797": 849621504.0,
+ "1798": 849621504.0,
+ "1799": 849621504.0,
+ "1800": 849621504.0,
+ "1801": 849621504.0,
+ "1802": 849621504.0,
+ "1803": 849621504.0,
+ "1804": 849621504.0,
+ "1805": 849621504.0,
+ "1806": 849621504.0,
+ "1807": 849621504.0,
+ "1808": 849621504.0,
+ "1809": 849621504.0,
+ "1810": 849621504.0,
+ "1811": 849621504.0,
+ "1812": 849621504.0,
+ "1813": 849621504.0,
+ "1814": 849621504.0,
+ "1815": 849621504.0,
+ "1816": 849621504.0,
+ "1817": 849621504.0,
+ "1818": 849621504.0,
+ "1819": 849621504.0,
+ "1820": 849621504.0,
+ "1821": 849621504.0,
+ "1822": 849621504.0,
+ "1823": 849621504.0,
+ "1824": 849621504.0,
+ "1825": 849621504.0,
+ "1826": 849621504.0,
+ "1827": 849621504.0,
+ "1828": 849621504.0,
+ "1829": 849621504.0,
+ "1830": 849621504.0,
+ "1831": 849621504.0,
+ "1832": 849621504.0,
+ "1833": 849621504.0,
+ "1834": 849621504.0,
+ "1835": 849621504.0,
+ "1836": 849621504.0,
+ "1837": 849621504.0,
+ "1838": 849621504.0,
+ "1839": 849621504.0,
+ "1840": 849621504.0,
+ "1841": 849621504.0,
+ "1842": 849621504.0,
+ "1843": 849621504.0,
+ "1844": 849621504.0,
+ "1845": 849621504.0,
+ "1846": 849621504.0,
+ "1847": 849621504.0,
+ "1848": 849621504.0,
+ "1849": 849621504.0,
+ "1850": 849621504.0,
+ "1851": 849621504.0,
+ "1852": 849621504.0,
+ "1853": 849621504.0,
+ "1854": 849621504.0,
+ "1855": 849621504.0,
+ "1856": 849621504.0,
+ "1857": 849621504.0,
+ "1858": 849621504.0,
+ "1859": 849621504.0,
+ "1860": 849621504.0,
+ "1861": 849621504.0,
+ "1862": 849621504.0,
+ "1863": 849621504.0,
+ "1864": 849621504.0,
+ "1865": 849621504.0,
+ "1866": 849621504.0,
+ "1867": 849621504.0,
+ "1868": 849621504.0,
+ "1869": 849621504.0,
+ "1870": 849621504.0,
+ "1871": 849621504.0,
+ "1872": 849621504.0,
+ "1873": 849621504.0,
+ "1874": 849621504.0,
+ "1875": 849621504.0,
+ "1876": 849621504.0,
+ "1877": 849621504.0,
+ "1878": 849621504.0,
+ "1879": 849621504.0,
+ "1880": 849621504.0,
+ "1881": 849621504.0,
+ "1882": 849621504.0,
+ "1883": 849621504.0,
+ "1884": 849621504.0,
+ "1885": 849621504.0,
+ "1886": 849621504.0,
+ "1887": 849621504.0,
+ "1888": 849621504.0,
+ "1889": 849621504.0,
+ "1890": 849621504.0,
+ "1891": 849621504.0,
+ "1892": 849621504.0,
+ "1893": 849621504.0,
+ "1894": 849621504.0,
+ "1895": 849621504.0,
+ "1896": 849621504.0,
+ "1897": 849621504.0,
+ "1898": 849621504.0,
+ "1899": 849621504.0,
+ "1900": 849621504.0,
+ "1901": 849621504.0,
+ "1902": 849621504.0,
+ "1903": 849621504.0,
+ "1904": 849621504.0,
+ "1905": 849621504.0,
+ "1906": 849621504.0,
+ "1907": 849621504.0,
+ "1908": 849621504.0,
+ "1909": 849621504.0,
+ "1910": 849621504.0,
+ "1911": 849621504.0,
+ "1912": 849621504.0,
+ "1913": 849621504.0,
+ "1914": 849621504.0,
+ "1915": 849621504.0,
+ "1916": 849621504.0,
+ "1917": 849621504.0,
+ "1918": 849621504.0,
+ "1919": 849621504.0,
+ "1920": 849621504.0,
+ "1921": 849621504.0,
+ "1922": 849621504.0,
+ "1923": 849621504.0,
+ "1924": 849621504.0,
+ "1925": 849621504.0,
+ "1926": 849621504.0,
+ "1927": 849621504.0,
+ "1928": 849621504.0,
+ "1929": 849621504.0,
+ "1930": 849621504.0,
+ "1931": 849621504.0,
+ "1932": 849621504.0,
+ "1933": 849621504.0,
+ "1934": 849621504.0,
+ "1935": 849621504.0,
+ "1936": 849621504.0,
+ "1937": 849621504.0,
+ "1938": 849621504.0,
+ "1939": 849621504.0,
+ "1940": 849621504.0,
+ "1941": 849621504.0,
+ "1942": 849621504.0,
+ "1943": 849621504.0,
+ "1944": 849621504.0,
+ "1945": 849621504.0,
+ "1946": 849621504.0,
+ "1947": 849621504.0,
+ "1948": 849621504.0,
+ "1949": 849621504.0,
+ "1950": 849621504.0,
+ "1951": 849621504.0,
+ "1952": 849621504.0,
+ "1953": 849621504.0,
+ "1954": 849621504.0,
+ "1955": 849621504.0,
+ "1956": 849621504.0,
+ "1957": 849621504.0,
+ "1958": 849621504.0,
+ "1959": 849621504.0,
+ "1960": 849621504.0,
+ "1961": 849621504.0,
+ "1962": 849621504.0,
+ "1963": 849621504.0,
+ "1964": 849621504.0,
+ "1965": 849621504.0,
+ "1966": 849621504.0,
+ "1967": 849621504.0,
+ "1968": 849621504.0,
+ "1969": 849621504.0,
+ "1970": 849621504.0,
+ "1971": 849621504.0,
+ "1972": 849621504.0,
+ "1973": 849621504.0,
+ "1974": 849621504.0,
+ "1975": 849621504.0,
+ "1976": 849621504.0,
+ "1977": 849621504.0,
+ "1978": 849621504.0,
+ "1979": 849621504.0,
+ "1980": 849621504.0,
+ "1981": 849621504.0,
+ "1982": 849621504.0,
+ "1983": 849621504.0,
+ "1984": 849621504.0,
+ "1985": 849621504.0,
+ "1986": 849621504.0,
+ "1987": 849621504.0,
+ "1988": 849621504.0,
+ "1989": 849621504.0,
+ "1990": 849621504.0,
+ "1991": 849621504.0,
+ "1992": 849621504.0,
+ "1993": 849621504.0,
+ "1994": 849621504.0,
+ "1995": 849621504.0,
+ "1996": 849621504.0,
+ "1997": 849621504.0,
+ "1998": 849621504.0,
+ "1999": 849621504.0,
+ "2000": 849621504.0
+ }
+ },
+ "iteration-time": {
+ "start_step": 1,
+ "end_step": 2000,
+ "step_interval": 1,
+ "values": {
+ "1": 14.94115,
+ "2": 1.30868,
+ "3": 1.13391,
+ "4": 1.12792,
+ "5": 1.13103,
+ "6": 1.1383,
+ "7": 1.13573,
+ "8": 1.15789,
+ "9": 1.12704,
+ "10": 1.1241,
+ "11": 1.12786,
+ "12": 1.1288,
+ "13": 1.1399,
+ "14": 1.13165,
+ "15": 1.12333,
+ "16": 1.12398,
+ "17": 1.12493,
+ "18": 1.11586,
+ "19": 1.1123,
+ "20": 1.11192,
+ "21": 1.1266,
+ "22": 1.13629,
+ "23": 1.13171,
+ "24": 1.14969,
+ "25": 1.17022,
+ "26": 1.14634,
+ "27": 1.14242,
+ "28": 1.14353,
+ "29": 1.14554,
+ "30": 1.28826,
+ "31": 1.14265,
+ "32": 1.14023,
+ "33": 1.15286,
+ "34": 1.14975,
+ "35": 1.13988,
+ "36": 1.62757,
+ "37": 2.22703,
+ "38": 1.36074,
+ "39": 1.1325,
+ "40": 1.14106,
+ "41": 1.14114,
+ "42": 1.13305,
+ "43": 1.12375,
+ "44": 1.12631,
+ "45": 1.12358,
+ "46": 1.12334,
+ "47": 1.12398,
+ "48": 1.12749,
+ "49": 1.13897,
+ "50": 1.13563,
+ "51": 1.13628,
+ "52": 1.12935,
+ "53": 1.12779,
+ "54": 1.13147,
+ "55": 1.1279,
+ "56": 1.12777,
+ "57": 1.1269,
+ "58": 1.13989,
+ "59": 1.13378,
+ "60": 1.13552,
+ "61": 1.12879,
+ "62": 1.4796,
+ "63": 1.12843,
+ "64": 1.12488,
+ "65": 1.12888,
+ "66": 1.14028,
+ "67": 1.13532,
+ "68": 1.13278,
+ "69": 1.12779,
+ "70": 1.12468,
+ "71": 1.12483,
+ "72": 1.12423,
+ "73": 1.12335,
+ "74": 1.12699,
+ "75": 1.13379,
+ "76": 1.13001,
+ "77": 1.12994,
+ "78": 1.13166,
+ "79": 1.12415,
+ "80": 1.126,
+ "81": 1.16016,
+ "82": 1.13845,
+ "83": 1.13882,
+ "84": 1.14455,
+ "85": 1.46908,
+ "86": 1.1259,
+ "87": 1.12119,
+ "88": 1.12312,
+ "89": 1.12593,
+ "90": 1.51995,
+ "91": 1.16022,
+ "92": 1.1304,
+ "93": 1.13161,
+ "94": 1.13511,
+ "95": 1.13911,
+ "96": 1.80205,
+ "97": 1.13368,
+ "98": 1.13335,
+ "99": 1.13549,
+ "100": 1.13409,
+ "101": 1.13703,
+ "102": 1.14592,
+ "103": 1.13516,
+ "104": 1.13661,
+ "105": 1.13299,
+ "106": 1.13577,
+ "107": 1.13657,
+ "108": 1.13144,
+ "109": 1.14828,
+ "110": 1.15036,
+ "111": 1.1486,
+ "112": 1.14183,
+ "113": 1.14297,
+ "114": 1.1411,
+ "115": 1.14318,
+ "116": 1.14291,
+ "117": 1.14168,
+ "118": 1.15055,
+ "119": 1.1482,
+ "120": 1.15352,
+ "121": 1.13046,
+ "122": 1.145,
+ "123": 1.14278,
+ "124": 1.1428,
+ "125": 1.14189,
+ "126": 1.13609,
+ "127": 1.14025,
+ "128": 1.14097,
+ "129": 1.13489,
+ "130": 1.13417,
+ "131": 1.13581,
+ "132": 1.13708,
+ "133": 1.17896,
+ "134": 1.13176,
+ "135": 1.12984,
+ "136": 1.1435,
+ "137": 1.15088,
+ "138": 1.14391,
+ "139": 1.14409,
+ "140": 1.14238,
+ "141": 1.14313,
+ "142": 1.1493,
+ "143": 1.13518,
+ "144": 1.13229,
+ "145": 1.13749,
+ "146": 1.15049,
+ "147": 1.16077,
+ "148": 1.14254,
+ "149": 1.14071,
+ "150": 1.14075,
+ "151": 1.13943,
+ "152": 1.15276,
+ "153": 1.15369,
+ "154": 1.14618,
+ "155": 1.14225,
+ "156": 1.14285,
+ "157": 1.14106,
+ "158": 1.14415,
+ "159": 1.14445,
+ "160": 1.14934,
+ "161": 1.14229,
+ "162": 1.14167,
+ "163": 1.14058,
+ "164": 1.14064,
+ "165": 1.14012,
+ "166": 1.15198,
+ "167": 1.15221,
+ "168": 1.1471,
+ "169": 1.14122,
+ "170": 1.14769,
+ "171": 1.14073,
+ "172": 1.14205,
+ "173": 1.14583,
+ "174": 1.14217,
+ "175": 1.14015,
+ "176": 1.14319,
+ "177": 1.14097,
+ "178": 1.14115,
+ "179": 1.14122,
+ "180": 1.15137,
+ "181": 1.14856,
+ "182": 1.15203,
+ "183": 1.14535,
+ "184": 1.13997,
+ "185": 1.15174,
+ "186": 1.18192,
+ "187": 1.14929,
+ "188": 1.14842,
+ "189": 1.14724,
+ "190": 1.14922,
+ "191": 1.14932,
+ "192": 1.14856,
+ "193": 1.1562,
+ "194": 1.153,
+ "195": 1.16371,
+ "196": 1.14525,
+ "197": 1.1411,
+ "198": 1.14592,
+ "199": 1.14301,
+ "200": 1.15088,
+ "201": 1.14229,
+ "202": 1.14171,
+ "203": 1.14083,
+ "204": 1.13968,
+ "205": 1.13977,
+ "206": 1.14177,
+ "207": 1.15548,
+ "208": 1.15609,
+ "209": 1.14509,
+ "210": 1.1487,
+ "211": 1.14163,
+ "212": 1.13971,
+ "213": 1.15326,
+ "214": 1.14129,
+ "215": 1.14055,
+ "216": 1.13893,
+ "217": 1.14191,
+ "218": 1.1418,
+ "219": 1.14249,
+ "220": 1.14162,
+ "221": 1.14077,
+ "222": 1.15513,
+ "223": 1.15668,
+ "224": 1.14515,
+ "225": 1.14589,
+ "226": 1.14548,
+ "227": 1.14318,
+ "228": 1.14204,
+ "229": 1.14391,
+ "230": 1.14565,
+ "231": 1.1439,
+ "232": 1.14309,
+ "233": 1.14396,
+ "234": 1.14146,
+ "235": 1.14229,
+ "236": 1.14106,
+ "237": 1.14362,
+ "238": 1.15203,
+ "239": 1.1942,
+ "240": 1.18025,
+ "241": 1.15197,
+ "242": 1.15276,
+ "243": 1.15399,
+ "244": 1.15628,
+ "245": 1.14958,
+ "246": 1.14931,
+ "247": 1.14093,
+ "248": 1.13869,
+ "249": 1.1385,
+ "250": 1.13897,
+ "251": 1.13787,
+ "252": 1.13939,
+ "253": 1.17282,
+ "254": 1.13361,
+ "255": 1.13502,
+ "256": 1.13895,
+ "257": 1.16245,
+ "258": 1.1352,
+ "259": 1.15685,
+ "260": 1.14637,
+ "261": 1.2867,
+ "262": 1.13699,
+ "263": 1.13959,
+ "264": 1.15414,
+ "265": 1.14324,
+ "266": 1.14515,
+ "267": 1.14328,
+ "268": 1.14359,
+ "269": 1.144,
+ "270": 1.15446,
+ "271": 1.15182,
+ "272": 1.15575,
+ "273": 1.15561,
+ "274": 1.15762,
+ "275": 1.15307,
+ "276": 1.1516,
+ "277": 1.1569,
+ "278": 1.15789,
+ "279": 1.168,
+ "280": 1.16711,
+ "281": 1.16858,
+ "282": 1.16899,
+ "283": 1.15631,
+ "284": 1.15543,
+ "285": 1.15685,
+ "286": 1.15663,
+ "287": 1.15204,
+ "288": 1.15333,
+ "289": 1.15257,
+ "290": 1.14865,
+ "291": 1.15067,
+ "292": 1.15626,
+ "293": 1.15161,
+ "294": 1.15116,
+ "295": 1.15102,
+ "296": 1.15104,
+ "297": 1.17304,
+ "298": 1.17562,
+ "299": 1.17694,
+ "300": 1.15026,
+ "301": 1.15562,
+ "302": 1.15582,
+ "303": 1.15039,
+ "304": 1.14517,
+ "305": 1.14745,
+ "306": 1.15392,
+ "307": 1.15054,
+ "308": 1.14391,
+ "309": 1.1426,
+ "310": 1.1434,
+ "311": 1.14297,
+ "312": 1.14164,
+ "313": 1.15234,
+ "314": 1.14891,
+ "315": 1.14745,
+ "316": 1.15325,
+ "317": 1.15145,
+ "318": 1.51061,
+ "319": 1.13797,
+ "320": 1.13871,
+ "321": 1.20976,
+ "322": 1.19788,
+ "323": 1.14258,
+ "324": 1.14169,
+ "325": 1.14227,
+ "326": 1.1426,
+ "327": 1.14596,
+ "328": 1.14584,
+ "329": 1.14606,
+ "330": 1.13676,
+ "331": 1.14712,
+ "332": 1.14502,
+ "333": 1.14602,
+ "334": 1.14598,
+ "335": 1.15781,
+ "336": 1.15666,
+ "337": 1.1498,
+ "338": 1.15651,
+ "339": 1.15267,
+ "340": 1.14703,
+ "341": 1.14889,
+ "342": 1.14863,
+ "343": 1.14731,
+ "344": 1.1479,
+ "345": 1.20819,
+ "346": 1.15653,
+ "347": 1.15548,
+ "348": 1.15594,
+ "349": 1.15558,
+ "350": 1.15652,
+ "351": 1.15348,
+ "352": 1.15517,
+ "353": 1.15665,
+ "354": 1.15895,
+ "355": 1.15829,
+ "356": 1.16229,
+ "357": 1.17016,
+ "358": 1.16317,
+ "359": 1.18492,
+ "360": 1.20126,
+ "361": 1.19034,
+ "362": 1.18723,
+ "363": 1.16724,
+ "364": 1.14627,
+ "365": 1.14394,
+ "366": 1.14503,
+ "367": 1.14264,
+ "368": 1.14464,
+ "369": 1.14478,
+ "370": 1.14447,
+ "371": 1.15012,
+ "372": 1.14509,
+ "373": 1.14362,
+ "374": 1.14617,
+ "375": 1.14658,
+ "376": 1.13748,
+ "377": 1.15141,
+ "378": 1.14564,
+ "379": 1.14278,
+ "380": 1.14166,
+ "381": 1.14361,
+ "382": 1.14293,
+ "383": 1.14196,
+ "384": 1.14178,
+ "385": 1.14053,
+ "386": 1.14184,
+ "387": 1.14451,
+ "388": 1.14162,
+ "389": 1.1419,
+ "390": 1.14477,
+ "391": 1.15539,
+ "392": 1.16117,
+ "393": 1.16925,
+ "394": 1.16815,
+ "395": 1.1561,
+ "396": 1.15146,
+ "397": 1.15422,
+ "398": 1.14884,
+ "399": 1.14136,
+ "400": 1.14059,
+ "401": 1.14105,
+ "402": 1.14013,
+ "403": 1.15094,
+ "404": 1.13492,
+ "405": 1.1425,
+ "406": 1.14173,
+ "407": 1.14385,
+ "408": 1.14421,
+ "409": 1.14226,
+ "410": 1.1417,
+ "411": 1.1511,
+ "412": 1.15763,
+ "413": 1.15891,
+ "414": 1.15294,
+ "415": 1.15191,
+ "416": 1.15346,
+ "417": 1.15001,
+ "418": 1.15279,
+ "419": 1.14974,
+ "420": 1.14848,
+ "421": 1.14722,
+ "422": 1.15396,
+ "423": 1.1499,
+ "424": 1.15269,
+ "425": 1.15087,
+ "426": 1.14945,
+ "427": 1.15106,
+ "428": 1.15515,
+ "429": 1.14379,
+ "430": 1.16231,
+ "431": 1.18658,
+ "432": 1.17212,
+ "433": 1.16725,
+ "434": 1.17832,
+ "435": 1.16254,
+ "436": 1.16094,
+ "437": 1.15865,
+ "438": 1.16104,
+ "439": 1.1621,
+ "440": 1.13911,
+ "441": 1.13485,
+ "442": 1.13534,
+ "443": 1.13627,
+ "444": 1.13432,
+ "445": 1.13868,
+ "446": 1.13561,
+ "447": 1.13518,
+ "448": 1.1365,
+ "449": 1.13444,
+ "450": 1.13455,
+ "451": 1.14098,
+ "452": 1.15368,
+ "453": 1.1566,
+ "454": 1.15931,
+ "455": 1.18151,
+ "456": 1.16215,
+ "457": 1.16012,
+ "458": 1.15916,
+ "459": 1.15837,
+ "460": 1.16214,
+ "461": 1.1652,
+ "462": 1.16044,
+ "463": 1.16179,
+ "464": 1.163,
+ "465": 1.16332,
+ "466": 1.15968,
+ "467": 1.16196,
+ "468": 1.1592,
+ "469": 1.15988,
+ "470": 1.16081,
+ "471": 1.16128,
+ "472": 1.15868,
+ "473": 1.16004,
+ "474": 1.16125,
+ "475": 1.15956,
+ "476": 1.16733,
+ "477": 1.18857,
+ "478": 1.15838,
+ "479": 1.16068,
+ "480": 1.16004,
+ "481": 1.15956,
+ "482": 1.15757,
+ "483": 1.15802,
+ "484": 1.16061,
+ "485": 1.15848,
+ "486": 1.16058,
+ "487": 1.15819,
+ "488": 1.15991,
+ "489": 1.15831,
+ "490": 1.1589,
+ "491": 1.16144,
+ "492": 1.15934,
+ "493": 1.15973,
+ "494": 1.16104,
+ "495": 1.15933,
+ "496": 1.16173,
+ "497": 1.16203,
+ "498": 1.16059,
+ "499": 1.16461,
+ "500": 1.16533,
+ "501": 1.1723,
+ "502": 1.17075,
+ "503": 1.17256,
+ "504": 1.16176,
+ "505": 1.15972,
+ "506": 1.16185,
+ "507": 1.21311,
+ "508": 1.16326,
+ "509": 1.15384,
+ "510": 1.15071,
+ "511": 1.15307,
+ "512": 1.15748,
+ "513": 1.1518,
+ "514": 1.15181,
+ "515": 1.15338,
+ "516": 1.1524,
+ "517": 1.15481,
+ "518": 1.15358,
+ "519": 1.16302,
+ "520": 1.16218,
+ "521": 1.15461,
+ "522": 1.157,
+ "523": 1.15817,
+ "524": 1.15517,
+ "525": 1.15361,
+ "526": 1.15183,
+ "527": 1.15237,
+ "528": 1.15423,
+ "529": 1.15637,
+ "530": 1.15521,
+ "531": 1.15012,
+ "532": 1.15132,
+ "533": 1.1495,
+ "534": 1.14919,
+ "535": 1.1546,
+ "536": 1.15442,
+ "537": 1.1514,
+ "538": 1.15195,
+ "539": 1.15221,
+ "540": 1.15639,
+ "541": 1.1549,
+ "542": 1.15495,
+ "543": 1.15683,
+ "544": 1.16361,
+ "545": 1.16186,
+ "546": 1.15697,
+ "547": 1.15978,
+ "548": 1.16151,
+ "549": 1.15737,
+ "550": 1.15451,
+ "551": 1.16057,
+ "552": 1.20604,
+ "553": 1.15937,
+ "554": 1.21638,
+ "555": 1.16193,
+ "556": 1.16004,
+ "557": 1.15937,
+ "558": 1.15924,
+ "559": 1.15864,
+ "560": 1.16064,
+ "561": 1.15935,
+ "562": 1.43389,
+ "563": 1.16041,
+ "564": 1.16122,
+ "565": 1.49173,
+ "566": 1.15954,
+ "567": 1.17345,
+ "568": 1.16261,
+ "569": 1.15966,
+ "570": 1.1607,
+ "571": 1.15553,
+ "572": 1.1568,
+ "573": 1.15385,
+ "574": 1.15701,
+ "575": 1.15849,
+ "576": 1.15634,
+ "577": 1.15908,
+ "578": 1.15576,
+ "579": 1.15627,
+ "580": 1.14973,
+ "581": 1.16027,
+ "582": 1.16176,
+ "583": 1.15493,
+ "584": 1.15722,
+ "585": 1.15744,
+ "586": 1.15502,
+ "587": 1.1559,
+ "588": 1.15496,
+ "589": 1.16378,
+ "590": 1.16595,
+ "591": 1.16611,
+ "592": 1.16989,
+ "593": 1.16842,
+ "594": 1.17261,
+ "595": 1.15925,
+ "596": 1.16083,
+ "597": 1.16113,
+ "598": 1.16297,
+ "599": 1.16456,
+ "600": 1.15983,
+ "601": 1.16187,
+ "602": 1.15943,
+ "603": 1.15985,
+ "604": 1.1592,
+ "605": 1.15871,
+ "606": 1.16032,
+ "607": 1.15919,
+ "608": 1.17988,
+ "609": 1.16067,
+ "610": 1.18157,
+ "611": 1.15299,
+ "612": 1.15282,
+ "613": 1.15274,
+ "614": 1.15344,
+ "615": 1.15192,
+ "616": 1.15757,
+ "617": 1.15404,
+ "618": 1.16198,
+ "619": 1.12381,
+ "620": 1.11492,
+ "621": 1.14943,
+ "622": 1.16512,
+ "623": 1.16958,
+ "624": 1.16409,
+ "625": 1.15844,
+ "626": 1.14917,
+ "627": 1.15285,
+ "628": 1.15477,
+ "629": 1.15363,
+ "630": 1.15213,
+ "631": 1.14647,
+ "632": 1.14867,
+ "633": 1.15423,
+ "634": 1.15566,
+ "635": 1.15345,
+ "636": 1.15319,
+ "637": 1.1511,
+ "638": 1.15409,
+ "639": 1.15188,
+ "640": 1.15258,
+ "641": 1.15414,
+ "642": 1.15983,
+ "643": 1.15819,
+ "644": 1.15887,
+ "645": 1.15631,
+ "646": 1.15765,
+ "647": 1.16277,
+ "648": 1.16768,
+ "649": 1.17095,
+ "650": 1.16972,
+ "651": 1.16894,
+ "652": 1.16584,
+ "653": 1.1612,
+ "654": 1.17303,
+ "655": 1.16406,
+ "656": 1.1617,
+ "657": 1.16573,
+ "658": 1.16082,
+ "659": 1.16677,
+ "660": 1.16969,
+ "661": 1.16374,
+ "662": 1.16155,
+ "663": 1.16674,
+ "664": 1.16865,
+ "665": 1.16719,
+ "666": 1.16772,
+ "667": 1.16872,
+ "668": 1.16616,
+ "669": 1.16505,
+ "670": 1.16449,
+ "671": 1.16777,
+ "672": 1.16457,
+ "673": 1.16059,
+ "674": 1.16013,
+ "675": 1.1589,
+ "676": 1.1645,
+ "677": 1.16737,
+ "678": 1.16262,
+ "679": 1.44417,
+ "680": 1.16641,
+ "681": 1.16441,
+ "682": 1.16834,
+ "683": 1.17163,
+ "684": 1.16041,
+ "685": 1.16815,
+ "686": 1.16615,
+ "687": 1.1689,
+ "688": 1.16377,
+ "689": 1.16277,
+ "690": 1.15926,
+ "691": 1.15823,
+ "692": 1.15747,
+ "693": 1.15897,
+ "694": 1.15722,
+ "695": 1.15679,
+ "696": 1.15619,
+ "697": 1.15686,
+ "698": 1.15548,
+ "699": 1.15619,
+ "700": 1.15662,
+ "701": 1.15701,
+ "702": 1.15611,
+ "703": 1.1578,
+ "704": 1.15921,
+ "705": 1.15626,
+ "706": 1.15696,
+ "707": 1.15676,
+ "708": 1.15718,
+ "709": 1.15643,
+ "710": 1.16154,
+ "711": 1.15995,
+ "712": 1.159,
+ "713": 1.16786,
+ "714": 1.15799,
+ "715": 1.15749,
+ "716": 1.52131,
+ "717": 1.15676,
+ "718": 1.16066,
+ "719": 1.15878,
+ "720": 1.16243,
+ "721": 1.15801,
+ "722": 1.16032,
+ "723": 1.15929,
+ "724": 1.16338,
+ "725": 1.15949,
+ "726": 1.16444,
+ "727": 1.31697,
+ "728": 1.15571,
+ "729": 1.15513,
+ "730": 1.15845,
+ "731": 1.16172,
+ "732": 1.15814,
+ "733": 1.1597,
+ "734": 1.15388,
+ "735": 1.15282,
+ "736": 1.15589,
+ "737": 1.15547,
+ "738": 1.1547,
+ "739": 1.15614,
+ "740": 1.15546,
+ "741": 1.15558,
+ "742": 1.15607,
+ "743": 1.15425,
+ "744": 1.15442,
+ "745": 1.16502,
+ "746": 1.15566,
+ "747": 1.15865,
+ "748": 1.15828,
+ "749": 1.16418,
+ "750": 1.15709,
+ "751": 1.15988,
+ "752": 1.15915,
+ "753": 1.15069,
+ "754": 1.15176,
+ "755": 1.15161,
+ "756": 1.1502,
+ "757": 1.14643,
+ "758": 1.7155,
+ "759": 1.15471,
+ "760": 1.15638,
+ "761": 1.15684,
+ "762": 1.16005,
+ "763": 1.1585,
+ "764": 1.16197,
+ "765": 1.22988,
+ "766": 1.16563,
+ "767": 1.16594,
+ "768": 1.16751,
+ "769": 1.16167,
+ "770": 1.16736,
+ "771": 1.16232,
+ "772": 1.16021,
+ "773": 1.16138,
+ "774": 1.16446,
+ "775": 1.15216,
+ "776": 1.15086,
+ "777": 1.15506,
+ "778": 1.15465,
+ "779": 1.15872,
+ "780": 1.15533,
+ "781": 1.15836,
+ "782": 1.15778,
+ "783": 1.21735,
+ "784": 1.15535,
+ "785": 1.14905,
+ "786": 1.14868,
+ "787": 1.14899,
+ "788": 1.1521,
+ "789": 1.1498,
+ "790": 1.15389,
+ "791": 1.15198,
+ "792": 1.14834,
+ "793": 1.14935,
+ "794": 1.14986,
+ "795": 1.15066,
+ "796": 1.15229,
+ "797": 1.15036,
+ "798": 1.15026,
+ "799": 1.15231,
+ "800": 1.15717,
+ "801": 1.15355,
+ "802": 1.15502,
+ "803": 1.15201,
+ "804": 1.15023,
+ "805": 1.15209,
+ "806": 1.15072,
+ "807": 1.48449,
+ "808": 1.15218,
+ "809": 1.1522,
+ "810": 1.15111,
+ "811": 1.15134,
+ "812": 1.15187,
+ "813": 1.15379,
+ "814": 1.15585,
+ "815": 1.16392,
+ "816": 1.15452,
+ "817": 1.15487,
+ "818": 1.15245,
+ "819": 1.14836,
+ "820": 1.14547,
+ "821": 1.74382,
+ "822": 1.14655,
+ "823": 1.13629,
+ "824": 1.15244,
+ "825": 1.14064,
+ "826": 1.14002,
+ "827": 1.14234,
+ "828": 1.1401,
+ "829": 1.13945,
+ "830": 1.14243,
+ "831": 1.14339,
+ "832": 1.13963,
+ "833": 1.14165,
+ "834": 1.13931,
+ "835": 1.13828,
+ "836": 1.13924,
+ "837": 1.13918,
+ "838": 1.14038,
+ "839": 1.14023,
+ "840": 1.13827,
+ "841": 1.14334,
+ "842": 1.26736,
+ "843": 1.15235,
+ "844": 1.16327,
+ "845": 1.15615,
+ "846": 1.15656,
+ "847": 1.14563,
+ "848": 1.14836,
+ "849": 1.14901,
+ "850": 1.14852,
+ "851": 1.15019,
+ "852": 1.14893,
+ "853": 1.14907,
+ "854": 1.14895,
+ "855": 1.14997,
+ "856": 1.14951,
+ "857": 1.15014,
+ "858": 1.14881,
+ "859": 1.15072,
+ "860": 1.16126,
+ "861": 1.15807,
+ "862": 1.15716,
+ "863": 1.15555,
+ "864": 1.15038,
+ "865": 1.15177,
+ "866": 1.15177,
+ "867": 1.14884,
+ "868": 1.14782,
+ "869": 1.15086,
+ "870": 1.14982,
+ "871": 1.14833,
+ "872": 1.14875,
+ "873": 1.15147,
+ "874": 1.15225,
+ "875": 1.29099,
+ "876": 2.39847,
+ "877": 2.16612,
+ "878": 1.53276,
+ "879": 1.14604,
+ "880": 1.1515,
+ "881": 1.16208,
+ "882": 1.15925,
+ "883": 1.14916,
+ "884": 1.14927,
+ "885": 1.1758,
+ "886": 1.17545,
+ "887": 1.17369,
+ "888": 1.17655,
+ "889": 1.16376,
+ "890": 1.14874,
+ "891": 1.148,
+ "892": 1.14787,
+ "893": 1.15123,
+ "894": 1.15168,
+ "895": 1.15419,
+ "896": 1.15535,
+ "897": 1.15242,
+ "898": 1.15508,
+ "899": 1.15225,
+ "900": 1.15072,
+ "901": 1.1534,
+ "902": 1.15136,
+ "903": 1.15481,
+ "904": 1.15989,
+ "905": 1.16184,
+ "906": 1.14716,
+ "907": 1.15192,
+ "908": 1.15696,
+ "909": 1.15328,
+ "910": 1.14059,
+ "911": 1.1604,
+ "912": 1.14941,
+ "913": 1.14972,
+ "914": 1.14954,
+ "915": 1.15073,
+ "916": 1.14475,
+ "917": 1.15414,
+ "918": 1.1385,
+ "919": 1.14185,
+ "920": 1.14089,
+ "921": 1.13784,
+ "922": 1.13875,
+ "923": 1.13882,
+ "924": 1.14141,
+ "925": 1.13908,
+ "926": 1.13874,
+ "927": 1.13823,
+ "928": 1.13737,
+ "929": 1.13836,
+ "930": 1.13809,
+ "931": 1.14893,
+ "932": 1.13972,
+ "933": 1.1369,
+ "934": 1.1362,
+ "935": 1.13765,
+ "936": 1.14369,
+ "937": 1.1504,
+ "938": 1.14208,
+ "939": 1.14841,
+ "940": 1.14975,
+ "941": 1.14225,
+ "942": 1.14185,
+ "943": 1.13864,
+ "944": 1.13915,
+ "945": 1.14062,
+ "946": 1.15111,
+ "947": 1.14071,
+ "948": 1.13898,
+ "949": 1.1399,
+ "950": 1.15937,
+ "951": 1.16785,
+ "952": 1.16807,
+ "953": 1.1506,
+ "954": 1.15006,
+ "955": 1.15045,
+ "956": 1.17067,
+ "957": 1.14856,
+ "958": 1.14992,
+ "959": 1.15251,
+ "960": 1.15045,
+ "961": 1.15121,
+ "962": 1.14957,
+ "963": 1.15095,
+ "964": 1.15,
+ "965": 1.15089,
+ "966": 1.15156,
+ "967": 1.15423,
+ "968": 1.16332,
+ "969": 1.15359,
+ "970": 1.15613,
+ "971": 1.15232,
+ "972": 1.15652,
+ "973": 1.15399,
+ "974": 1.15065,
+ "975": 1.1485,
+ "976": 1.15243,
+ "977": 1.15368,
+ "978": 1.14828,
+ "979": 1.14969,
+ "980": 1.15374,
+ "981": 1.1505,
+ "982": 1.15031,
+ "983": 1.15033,
+ "984": 1.14921,
+ "985": 1.15504,
+ "986": 1.15572,
+ "987": 1.153,
+ "988": 1.15573,
+ "989": 1.14747,
+ "990": 1.14636,
+ "991": 1.14517,
+ "992": 1.1463,
+ "993": 1.14805,
+ "994": 1.14644,
+ "995": 1.14583,
+ "996": 1.14485,
+ "997": 1.14418,
+ "998": 1.14622,
+ "999": 1.14662,
+ "1000": 1.14312,
+ "1001": 1.15227,
+ "1002": 1.14681,
+ "1003": 1.14794,
+ "1004": 1.14889,
+ "1005": 1.15067,
+ "1006": 1.14757,
+ "1007": 1.14767,
+ "1008": 1.15061,
+ "1009": 1.15075,
+ "1010": 1.14894,
+ "1011": 1.14975,
+ "1012": 1.14667,
+ "1013": 1.14688,
+ "1014": 1.14788,
+ "1015": 1.167,
+ "1016": 1.44606,
+ "1017": 1.14923,
+ "1018": 1.15268,
+ "1019": 1.14981,
+ "1020": 1.15011,
+ "1021": 1.47391,
+ "1022": 1.15277,
+ "1023": 1.14774,
+ "1024": 1.146,
+ "1025": 1.15253,
+ "1026": 1.14633,
+ "1027": 1.14525,
+ "1028": 1.14728,
+ "1029": 1.14654,
+ "1030": 1.14663,
+ "1031": 1.14708,
+ "1032": 1.14715,
+ "1033": 1.1454,
+ "1034": 1.14763,
+ "1035": 1.14591,
+ "1036": 1.14493,
+ "1037": 1.14584,
+ "1038": 1.14665,
+ "1039": 1.14812,
+ "1040": 1.14495,
+ "1041": 1.15044,
+ "1042": 1.14701,
+ "1043": 1.14657,
+ "1044": 1.14631,
+ "1045": 1.14822,
+ "1046": 1.14789,
+ "1047": 1.14525,
+ "1048": 1.14815,
+ "1049": 1.14939,
+ "1050": 1.14592,
+ "1051": 1.14667,
+ "1052": 1.15232,
+ "1053": 1.14863,
+ "1054": 1.14908,
+ "1055": 1.14931,
+ "1056": 1.14644,
+ "1057": 1.149,
+ "1058": 1.14751,
+ "1059": 1.14668,
+ "1060": 1.14758,
+ "1061": 1.14789,
+ "1062": 1.43562,
+ "1063": 1.14875,
+ "1064": 1.14846,
+ "1065": 1.14888,
+ "1066": 1.15486,
+ "1067": 1.15212,
+ "1068": 1.14934,
+ "1069": 1.14526,
+ "1070": 1.14506,
+ "1071": 1.14599,
+ "1072": 1.14774,
+ "1073": 1.14651,
+ "1074": 1.14609,
+ "1075": 1.14817,
+ "1076": 1.14662,
+ "1077": 1.15159,
+ "1078": 1.14735,
+ "1079": 1.14525,
+ "1080": 1.1516,
+ "1081": 1.14601,
+ "1082": 1.13989,
+ "1083": 1.13569,
+ "1084": 1.1371,
+ "1085": 1.1366,
+ "1086": 1.13713,
+ "1087": 1.13756,
+ "1088": 1.13768,
+ "1089": 1.13917,
+ "1090": 1.13759,
+ "1091": 1.13884,
+ "1092": 1.13707,
+ "1093": 1.13679,
+ "1094": 1.13513,
+ "1095": 1.1351,
+ "1096": 1.13494,
+ "1097": 1.13589,
+ "1098": 1.14132,
+ "1099": 1.13697,
+ "1100": 1.14195,
+ "1101": 1.14189,
+ "1102": 1.13736,
+ "1103": 1.13781,
+ "1104": 1.14284,
+ "1105": 1.13518,
+ "1106": 1.13585,
+ "1107": 1.13621,
+ "1108": 1.13665,
+ "1109": 1.13792,
+ "1110": 1.13764,
+ "1111": 1.13778,
+ "1112": 1.13619,
+ "1113": 1.13651,
+ "1114": 1.13628,
+ "1115": 1.13802,
+ "1116": 1.13792,
+ "1117": 1.13642,
+ "1118": 1.13784,
+ "1119": 1.14898,
+ "1120": 1.15049,
+ "1121": 1.15028,
+ "1122": 1.14509,
+ "1123": 1.1445,
+ "1124": 1.14756,
+ "1125": 1.15117,
+ "1126": 1.14917,
+ "1127": 1.1475,
+ "1128": 1.1481,
+ "1129": 1.14683,
+ "1130": 1.14088,
+ "1131": 1.13493,
+ "1132": 1.13613,
+ "1133": 1.13537,
+ "1134": 1.13473,
+ "1135": 1.13657,
+ "1136": 1.13516,
+ "1137": 1.13606,
+ "1138": 1.13473,
+ "1139": 1.13442,
+ "1140": 1.13398,
+ "1141": 1.13591,
+ "1142": 1.13975,
+ "1143": 1.13478,
+ "1144": 1.13376,
+ "1145": 1.13428,
+ "1146": 1.1348,
+ "1147": 1.13462,
+ "1148": 1.1351,
+ "1149": 1.13494,
+ "1150": 1.13506,
+ "1151": 1.13487,
+ "1152": 1.14039,
+ "1153": 1.13991,
+ "1154": 1.13825,
+ "1155": 1.1373,
+ "1156": 1.13451,
+ "1157": 1.13683,
+ "1158": 1.13335,
+ "1159": 1.13548,
+ "1160": 1.1339,
+ "1161": 1.13613,
+ "1162": 1.13429,
+ "1163": 1.13448,
+ "1164": 1.13542,
+ "1165": 1.13453,
+ "1166": 1.13398,
+ "1167": 1.13549,
+ "1168": 1.1342,
+ "1169": 1.13502,
+ "1170": 1.13535,
+ "1171": 1.13581,
+ "1172": 1.13532,
+ "1173": 1.13552,
+ "1174": 1.13371,
+ "1175": 1.13456,
+ "1176": 1.13401,
+ "1177": 1.1335,
+ "1178": 1.13628,
+ "1179": 1.13907,
+ "1180": 1.13757,
+ "1181": 1.1538,
+ "1182": 1.15712,
+ "1183": 1.16123,
+ "1184": 1.15318,
+ "1185": 1.14801,
+ "1186": 1.14711,
+ "1187": 1.1471,
+ "1188": 1.15109,
+ "1189": 1.14707,
+ "1190": 1.14787,
+ "1191": 1.1451,
+ "1192": 1.14677,
+ "1193": 1.14621,
+ "1194": 1.14554,
+ "1195": 1.14738,
+ "1196": 1.14756,
+ "1197": 1.14799,
+ "1198": 1.1487,
+ "1199": 1.14616,
+ "1200": 1.14688,
+ "1201": 1.14531,
+ "1202": 1.14639,
+ "1203": 1.14696,
+ "1204": 1.1469,
+ "1205": 1.1472,
+ "1206": 1.14687,
+ "1207": 1.1494,
+ "1208": 1.14873,
+ "1209": 1.15175,
+ "1210": 1.14868,
+ "1211": 1.14793,
+ "1212": 1.14766,
+ "1213": 1.14823,
+ "1214": 1.15557,
+ "1215": 1.15986,
+ "1216": 1.14175,
+ "1217": 1.1392,
+ "1218": 1.13591,
+ "1219": 1.13796,
+ "1220": 1.14086,
+ "1221": 1.14081,
+ "1222": 1.13816,
+ "1223": 1.13977,
+ "1224": 1.14436,
+ "1225": 1.13986,
+ "1226": 1.13821,
+ "1227": 1.13854,
+ "1228": 1.13738,
+ "1229": 1.1384,
+ "1230": 1.13897,
+ "1231": 1.13732,
+ "1232": 1.13852,
+ "1233": 1.14144,
+ "1234": 1.13711,
+ "1235": 1.14105,
+ "1236": 1.13578,
+ "1237": 1.13838,
+ "1238": 1.13809,
+ "1239": 1.13782,
+ "1240": 1.13859,
+ "1241": 1.1381,
+ "1242": 1.13717,
+ "1243": 1.14814,
+ "1244": 1.16451,
+ "1245": 1.17765,
+ "1246": 1.17167,
+ "1247": 1.15708,
+ "1248": 1.15406,
+ "1249": 1.17391,
+ "1250": 1.14803,
+ "1251": 1.14601,
+ "1252": 1.14796,
+ "1253": 1.14706,
+ "1254": 1.14679,
+ "1255": 1.14306,
+ "1256": 1.14387,
+ "1257": 1.14608,
+ "1258": 1.14617,
+ "1259": 1.14999,
+ "1260": 1.1468,
+ "1261": 1.14332,
+ "1262": 1.15005,
+ "1263": 1.1449,
+ "1264": 1.14544,
+ "1265": 1.14292,
+ "1266": 1.14481,
+ "1267": 1.154,
+ "1268": 1.15455,
+ "1269": 1.15329,
+ "1270": 1.15008,
+ "1271": 1.15345,
+ "1272": 1.14616,
+ "1273": 1.15423,
+ "1274": 1.15349,
+ "1275": 1.14785,
+ "1276": 1.14536,
+ "1277": 1.14467,
+ "1278": 1.1456,
+ "1279": 1.14593,
+ "1280": 1.1462,
+ "1281": 1.14599,
+ "1282": 1.14837,
+ "1283": 1.14585,
+ "1284": 1.14656,
+ "1285": 1.14618,
+ "1286": 1.14615,
+ "1287": 1.14657,
+ "1288": 1.44686,
+ "1289": 1.14572,
+ "1290": 1.14398,
+ "1291": 1.1431,
+ "1292": 1.14524,
+ "1293": 1.14421,
+ "1294": 1.14593,
+ "1295": 1.16051,
+ "1296": 1.16214,
+ "1297": 1.15606,
+ "1298": 1.14439,
+ "1299": 1.14445,
+ "1300": 1.1445,
+ "1301": 1.1455,
+ "1302": 1.14117,
+ "1303": 1.14365,
+ "1304": 1.14474,
+ "1305": 1.14456,
+ "1306": 1.14522,
+ "1307": 1.144,
+ "1308": 1.14453,
+ "1309": 1.14471,
+ "1310": 1.1456,
+ "1311": 1.15495,
+ "1312": 1.15256,
+ "1313": 1.14805,
+ "1314": 1.14996,
+ "1315": 1.14425,
+ "1316": 1.14401,
+ "1317": 1.14262,
+ "1318": 1.14556,
+ "1319": 1.14661,
+ "1320": 1.14567,
+ "1321": 1.14648,
+ "1322": 1.14709,
+ "1323": 1.14522,
+ "1324": 1.14764,
+ "1325": 1.14331,
+ "1326": 1.14538,
+ "1327": 1.1453,
+ "1328": 1.14734,
+ "1329": 1.18619,
+ "1330": 1.48212,
+ "1331": 1.14651,
+ "1332": 1.15204,
+ "1333": 1.14629,
+ "1334": 1.14624,
+ "1335": 1.14927,
+ "1336": 1.14601,
+ "1337": 1.15642,
+ "1338": 1.14811,
+ "1339": 1.14508,
+ "1340": 1.15069,
+ "1341": 1.14629,
+ "1342": 1.14635,
+ "1343": 1.14657,
+ "1344": 1.14655,
+ "1345": 1.14564,
+ "1346": 1.14633,
+ "1347": 1.14523,
+ "1348": 1.14691,
+ "1349": 1.14575,
+ "1350": 1.14592,
+ "1351": 1.14631,
+ "1352": 1.14436,
+ "1353": 1.14573,
+ "1354": 1.14471,
+ "1355": 1.14554,
+ "1356": 1.14492,
+ "1357": 1.14301,
+ "1358": 1.141,
+ "1359": 1.14219,
+ "1360": 1.14228,
+ "1361": 1.14109,
+ "1362": 1.1413,
+ "1363": 1.14096,
+ "1364": 1.15355,
+ "1365": 1.14229,
+ "1366": 1.14615,
+ "1367": 1.14174,
+ "1368": 1.13953,
+ "1369": 1.14014,
+ "1370": 1.14132,
+ "1371": 1.14139,
+ "1372": 1.13849,
+ "1373": 1.14304,
+ "1374": 1.14028,
+ "1375": 1.13912,
+ "1376": 1.14082,
+ "1377": 1.1416,
+ "1378": 1.13936,
+ "1379": 1.13866,
+ "1380": 1.13826,
+ "1381": 1.14443,
+ "1382": 1.14029,
+ "1383": 1.13913,
+ "1384": 1.14177,
+ "1385": 1.14492,
+ "1386": 1.1415,
+ "1387": 1.1398,
+ "1388": 1.14017,
+ "1389": 1.14077,
+ "1390": 1.14782,
+ "1391": 1.15011,
+ "1392": 1.15174,
+ "1393": 1.14605,
+ "1394": 1.14761,
+ "1395": 1.14735,
+ "1396": 1.14827,
+ "1397": 1.14566,
+ "1398": 1.14659,
+ "1399": 1.14187,
+ "1400": 1.14737,
+ "1401": 1.14674,
+ "1402": 1.14468,
+ "1403": 1.14534,
+ "1404": 1.14726,
+ "1405": 1.14773,
+ "1406": 1.14711,
+ "1407": 1.14543,
+ "1408": 1.14568,
+ "1409": 1.14559,
+ "1410": 1.14443,
+ "1411": 1.14591,
+ "1412": 1.14444,
+ "1413": 1.14904,
+ "1414": 1.14806,
+ "1415": 1.14757,
+ "1416": 1.14307,
+ "1417": 1.14119,
+ "1418": 1.14392,
+ "1419": 1.14104,
+ "1420": 1.14278,
+ "1421": 1.13949,
+ "1422": 1.14028,
+ "1423": 1.14112,
+ "1424": 1.14151,
+ "1425": 1.14321,
+ "1426": 1.14894,
+ "1427": 1.14281,
+ "1428": 1.14881,
+ "1429": 1.14225,
+ "1430": 1.13905,
+ "1431": 1.14148,
+ "1432": 1.14895,
+ "1433": 1.15186,
+ "1434": 1.14773,
+ "1435": 1.14968,
+ "1436": 1.14689,
+ "1437": 1.1487,
+ "1438": 1.14731,
+ "1439": 1.14746,
+ "1440": 1.14835,
+ "1441": 1.15151,
+ "1442": 1.15182,
+ "1443": 1.15073,
+ "1444": 1.14751,
+ "1445": 1.15081,
+ "1446": 1.15106,
+ "1447": 1.14876,
+ "1448": 1.15178,
+ "1449": 1.15117,
+ "1450": 1.1479,
+ "1451": 1.14851,
+ "1452": 1.14502,
+ "1453": 1.1454,
+ "1454": 1.14722,
+ "1455": 1.14628,
+ "1456": 1.14413,
+ "1457": 1.14761,
+ "1458": 1.14681,
+ "1459": 1.14632,
+ "1460": 1.14804,
+ "1461": 1.14676,
+ "1462": 1.14566,
+ "1463": 1.14599,
+ "1464": 1.14679,
+ "1465": 1.14572,
+ "1466": 1.14995,
+ "1467": 1.14848,
+ "1468": 1.14679,
+ "1469": 1.15027,
+ "1470": 1.14636,
+ "1471": 1.14406,
+ "1472": 1.14039,
+ "1473": 1.13768,
+ "1474": 1.13897,
+ "1475": 1.14331,
+ "1476": 1.1403,
+ "1477": 1.14139,
+ "1478": 1.14985,
+ "1479": 1.14611,
+ "1480": 1.47655,
+ "1481": 1.45511,
+ "1482": 1.14381,
+ "1483": 1.13941,
+ "1484": 1.13782,
+ "1485": 1.13771,
+ "1486": 1.13796,
+ "1487": 1.13795,
+ "1488": 1.13829,
+ "1489": 1.13758,
+ "1490": 1.13822,
+ "1491": 1.13667,
+ "1492": 1.13847,
+ "1493": 1.13787,
+ "1494": 1.14072,
+ "1495": 1.14614,
+ "1496": 1.14436,
+ "1497": 1.14422,
+ "1498": 1.1393,
+ "1499": 1.13987,
+ "1500": 1.13991,
+ "1501": 1.14215,
+ "1502": 1.13842,
+ "1503": 1.13883,
+ "1504": 1.1496,
+ "1505": 1.14028,
+ "1506": 1.13931,
+ "1507": 1.13949,
+ "1508": 1.14063,
+ "1509": 1.13913,
+ "1510": 1.1402,
+ "1511": 1.13931,
+ "1512": 1.13839,
+ "1513": 1.13771,
+ "1514": 1.13848,
+ "1515": 1.13796,
+ "1516": 1.13782,
+ "1517": 1.13889,
+ "1518": 1.13716,
+ "1519": 1.13908,
+ "1520": 1.13972,
+ "1521": 1.13966,
+ "1522": 1.13875,
+ "1523": 1.15781,
+ "1524": 1.15885,
+ "1525": 1.15802,
+ "1526": 1.14191,
+ "1527": 1.14054,
+ "1528": 1.1385,
+ "1529": 1.13922,
+ "1530": 1.12994,
+ "1531": 1.12552,
+ "1532": 1.27166,
+ "1533": 1.12707,
+ "1534": 1.12638,
+ "1535": 1.12608,
+ "1536": 1.12654,
+ "1537": 1.12511,
+ "1538": 1.16008,
+ "1539": 1.13169,
+ "1540": 1.13294,
+ "1541": 1.13386,
+ "1542": 1.13461,
+ "1543": 1.13337,
+ "1544": 1.1331,
+ "1545": 1.13294,
+ "1546": 1.13283,
+ "1547": 1.13316,
+ "1548": 1.13651,
+ "1549": 1.13626,
+ "1550": 1.13638,
+ "1551": 1.13187,
+ "1552": 1.20522,
+ "1553": 1.15894,
+ "1554": 1.14738,
+ "1555": 1.14563,
+ "1556": 1.14409,
+ "1557": 1.15018,
+ "1558": 1.14323,
+ "1559": 1.14591,
+ "1560": 1.14645,
+ "1561": 1.14673,
+ "1562": 1.14543,
+ "1563": 1.14518,
+ "1564": 1.14589,
+ "1565": 1.14486,
+ "1566": 1.14436,
+ "1567": 1.14357,
+ "1568": 1.1454,
+ "1569": 1.14493,
+ "1570": 1.14347,
+ "1571": 1.14477,
+ "1572": 1.14203,
+ "1573": 1.14441,
+ "1574": 1.14468,
+ "1575": 1.14607,
+ "1576": 1.14532,
+ "1577": 1.14389,
+ "1578": 1.1433,
+ "1579": 1.14321,
+ "1580": 1.14391,
+ "1581": 1.1421,
+ "1582": 1.14368,
+ "1583": 1.1444,
+ "1584": 1.14356,
+ "1585": 1.14875,
+ "1586": 1.14497,
+ "1587": 1.14521,
+ "1588": 1.14708,
+ "1589": 1.14631,
+ "1590": 1.14662,
+ "1591": 1.14949,
+ "1592": 1.15354,
+ "1593": 1.14014,
+ "1594": 1.1408,
+ "1595": 1.14166,
+ "1596": 1.14151,
+ "1597": 1.14228,
+ "1598": 1.14126,
+ "1599": 1.14028,
+ "1600": 1.14528,
+ "1601": 1.14125,
+ "1602": 1.14085,
+ "1603": 1.13862,
+ "1604": 1.13487,
+ "1605": 1.13314,
+ "1606": 1.13467,
+ "1607": 1.13153,
+ "1608": 1.12971,
+ "1609": 1.13044,
+ "1610": 1.14013,
+ "1611": 1.13008,
+ "1612": 1.13161,
+ "1613": 1.13128,
+ "1614": 1.13059,
+ "1615": 1.13169,
+ "1616": 1.13043,
+ "1617": 1.13141,
+ "1618": 1.12976,
+ "1619": 1.13071,
+ "1620": 1.12907,
+ "1621": 1.13138,
+ "1622": 1.12994,
+ "1623": 1.12985,
+ "1624": 1.12999,
+ "1625": 1.13035,
+ "1626": 1.13761,
+ "1627": 1.13703,
+ "1628": 1.15487,
+ "1629": 1.13257,
+ "1630": 1.13549,
+ "1631": 1.13358,
+ "1632": 1.13488,
+ "1633": 1.13601,
+ "1634": 1.13282,
+ "1635": 1.13439,
+ "1636": 1.13078,
+ "1637": 1.13147,
+ "1638": 1.13065,
+ "1639": 1.13181,
+ "1640": 1.13227,
+ "1641": 1.13282,
+ "1642": 1.13305,
+ "1643": 1.19491,
+ "1644": 1.15821,
+ "1645": 1.15349,
+ "1646": 1.1437,
+ "1647": 1.1416,
+ "1648": 1.14282,
+ "1649": 1.1408,
+ "1650": 1.13388,
+ "1651": 1.13396,
+ "1652": 1.15414,
+ "1653": 1.13734,
+ "1654": 1.13143,
+ "1655": 1.13124,
+ "1656": 1.13417,
+ "1657": 1.13376,
+ "1658": 1.12932,
+ "1659": 1.13161,
+ "1660": 1.13178,
+ "1661": 1.1315,
+ "1662": 1.13209,
+ "1663": 1.13118,
+ "1664": 1.13332,
+ "1665": 1.12981,
+ "1666": 1.13001,
+ "1667": 1.12943,
+ "1668": 1.12938,
+ "1669": 1.12973,
+ "1670": 1.13031,
+ "1671": 1.14164,
+ "1672": 1.14108,
+ "1673": 1.14165,
+ "1674": 1.14189,
+ "1675": 1.14174,
+ "1676": 1.14802,
+ "1677": 1.14434,
+ "1678": 1.14543,
+ "1679": 1.14285,
+ "1680": 1.14529,
+ "1681": 1.14548,
+ "1682": 1.14333,
+ "1683": 1.14553,
+ "1684": 1.14327,
+ "1685": 1.1476,
+ "1686": 1.1406,
+ "1687": 1.13769,
+ "1688": 1.13364,
+ "1689": 1.13418,
+ "1690": 1.13026,
+ "1691": 1.13222,
+ "1692": 1.13195,
+ "1693": 1.13247,
+ "1694": 1.13264,
+ "1695": 1.13167,
+ "1696": 1.13234,
+ "1697": 1.13335,
+ "1698": 1.13463,
+ "1699": 1.1337,
+ "1700": 1.13362,
+ "1701": 1.13339,
+ "1702": 1.13335,
+ "1703": 1.13412,
+ "1704": 1.1332,
+ "1705": 1.13109,
+ "1706": 1.13306,
+ "1707": 1.42699,
+ "1708": 1.14258,
+ "1709": 1.13227,
+ "1710": 1.13333,
+ "1711": 1.13316,
+ "1712": 1.13147,
+ "1713": 1.1325,
+ "1714": 1.13279,
+ "1715": 1.13509,
+ "1716": 1.132,
+ "1717": 1.13183,
+ "1718": 1.13123,
+ "1719": 1.13209,
+ "1720": 1.13195,
+ "1721": 1.12891,
+ "1722": 1.12633,
+ "1723": 1.12872,
+ "1724": 1.1269,
+ "1725": 1.12641,
+ "1726": 1.12585,
+ "1727": 1.12446,
+ "1728": 1.12583,
+ "1729": 1.1336,
+ "1730": 1.1322,
+ "1731": 1.13153,
+ "1732": 1.132,
+ "1733": 1.13239,
+ "1734": 1.13216,
+ "1735": 1.13252,
+ "1736": 1.13132,
+ "1737": 1.13165,
+ "1738": 1.13359,
+ "1739": 1.126,
+ "1740": 1.124,
+ "1741": 1.12533,
+ "1742": 1.12379,
+ "1743": 1.12474,
+ "1744": 1.12432,
+ "1745": 1.13505,
+ "1746": 1.13795,
+ "1747": 1.13914,
+ "1748": 1.17805,
+ "1749": 1.13962,
+ "1750": 1.13602,
+ "1751": 1.13778,
+ "1752": 1.13639,
+ "1753": 1.14452,
+ "1754": 1.14424,
+ "1755": 1.14388,
+ "1756": 1.14572,
+ "1757": 1.17074,
+ "1758": 1.14596,
+ "1759": 1.14637,
+ "1760": 1.14576,
+ "1761": 1.1441,
+ "1762": 1.13385,
+ "1763": 1.13833,
+ "1764": 1.13995,
+ "1765": 1.14229,
+ "1766": 1.2706,
+ "1767": 1.15999,
+ "1768": 1.13873,
+ "1769": 1.1421,
+ "1770": 1.13078,
+ "1771": 1.13059,
+ "1772": 1.13076,
+ "1773": 1.13527,
+ "1774": 1.13153,
+ "1775": 1.1299,
+ "1776": 1.13144,
+ "1777": 1.13048,
+ "1778": 1.1312,
+ "1779": 1.13109,
+ "1780": 1.13227,
+ "1781": 1.1318,
+ "1782": 1.13195,
+ "1783": 1.13076,
+ "1784": 1.13371,
+ "1785": 1.13513,
+ "1786": 1.13544,
+ "1787": 1.13286,
+ "1788": 1.13114,
+ "1789": 1.12859,
+ "1790": 1.13136,
+ "1791": 1.13775,
+ "1792": 1.1401,
+ "1793": 1.13769,
+ "1794": 1.13564,
+ "1795": 1.13638,
+ "1796": 1.13621,
+ "1797": 1.13614,
+ "1798": 1.13707,
+ "1799": 1.13631,
+ "1800": 1.13547,
+ "1801": 1.13673,
+ "1802": 1.13706,
+ "1803": 1.13765,
+ "1804": 1.13506,
+ "1805": 1.13603,
+ "1806": 1.13717,
+ "1807": 1.13637,
+ "1808": 1.13841,
+ "1809": 1.13734,
+ "1810": 1.1379,
+ "1811": 1.13795,
+ "1812": 1.13826,
+ "1813": 1.13875,
+ "1814": 1.13885,
+ "1815": 1.13773,
+ "1816": 1.13726,
+ "1817": 1.14087,
+ "1818": 1.1378,
+ "1819": 1.13714,
+ "1820": 1.13737,
+ "1821": 1.13928,
+ "1822": 1.1371,
+ "1823": 1.13901,
+ "1824": 1.14485,
+ "1825": 1.12803,
+ "1826": 1.12264,
+ "1827": 1.12651,
+ "1828": 1.13421,
+ "1829": 1.13198,
+ "1830": 1.13242,
+ "1831": 1.13488,
+ "1832": 1.13287,
+ "1833": 1.13394,
+ "1834": 1.13403,
+ "1835": 1.13598,
+ "1836": 1.13357,
+ "1837": 1.13518,
+ "1838": 1.13404,
+ "1839": 1.13577,
+ "1840": 1.13254,
+ "1841": 1.13422,
+ "1842": 1.13496,
+ "1843": 1.135,
+ "1844": 1.13791,
+ "1845": 1.13082,
+ "1846": 1.13135,
+ "1847": 1.13026,
+ "1848": 1.13098,
+ "1849": 1.13032,
+ "1850": 1.13038,
+ "1851": 1.13107,
+ "1852": 1.13535,
+ "1853": 1.1311,
+ "1854": 1.13935,
+ "1855": 1.13148,
+ "1856": 1.13042,
+ "1857": 1.13238,
+ "1858": 1.13034,
+ "1859": 1.13083,
+ "1860": 1.13262,
+ "1861": 1.13117,
+ "1862": 1.13181,
+ "1863": 1.13237,
+ "1864": 1.13125,
+ "1865": 1.13519,
+ "1866": 1.14006,
+ "1867": 1.13476,
+ "1868": 1.13101,
+ "1869": 1.13227,
+ "1870": 1.13399,
+ "1871": 1.13455,
+ "1872": 1.13237,
+ "1873": 1.13088,
+ "1874": 1.13163,
+ "1875": 1.13336,
+ "1876": 1.13121,
+ "1877": 1.13209,
+ "1878": 1.13199,
+ "1879": 1.13177,
+ "1880": 1.13322,
+ "1881": 1.13141,
+ "1882": 1.13236,
+ "1883": 1.12859,
+ "1884": 1.12504,
+ "1885": 1.12493,
+ "1886": 1.12502,
+ "1887": 1.12484,
+ "1888": 1.1248,
+ "1889": 1.12719,
+ "1890": 1.13286,
+ "1891": 1.1293,
+ "1892": 1.13422,
+ "1893": 1.12646,
+ "1894": 1.12508,
+ "1895": 1.12422,
+ "1896": 1.12724,
+ "1897": 1.12903,
+ "1898": 1.13203,
+ "1899": 1.12741,
+ "1900": 1.12527,
+ "1901": 1.12359,
+ "1902": 1.12382,
+ "1903": 1.12536,
+ "1904": 1.12683,
+ "1905": 1.12606,
+ "1906": 1.12607,
+ "1907": 1.12626,
+ "1908": 1.44717,
+ "1909": 1.12543,
+ "1910": 1.12376,
+ "1911": 1.12429,
+ "1912": 1.12442,
+ "1913": 1.12355,
+ "1914": 1.12476,
+ "1915": 1.12331,
+ "1916": 1.12342,
+ "1917": 1.12442,
+ "1918": 1.12472,
+ "1919": 1.12536,
+ "1920": 1.12387,
+ "1921": 1.12347,
+ "1922": 1.12561,
+ "1923": 1.12391,
+ "1924": 1.12342,
+ "1925": 1.12607,
+ "1926": 1.12383,
+ "1927": 1.12305,
+ "1928": 1.125,
+ "1929": 1.12399,
+ "1930": 1.1237,
+ "1931": 1.12459,
+ "1932": 1.12475,
+ "1933": 1.12278,
+ "1934": 1.12413,
+ "1935": 1.12588,
+ "1936": 1.12473,
+ "1937": 1.12412,
+ "1938": 1.12444,
+ "1939": 1.12303,
+ "1940": 1.12421,
+ "1941": 1.12404,
+ "1942": 1.12568,
+ "1943": 1.12645,
+ "1944": 1.12388,
+ "1945": 1.44561,
+ "1946": 1.12748,
+ "1947": 1.44404,
+ "1948": 1.12309,
+ "1949": 1.12591,
+ "1950": 1.124,
+ "1951": 1.12953,
+ "1952": 1.12429,
+ "1953": 1.48105,
+ "1954": 1.12576,
+ "1955": 1.1274,
+ "1956": 1.12693,
+ "1957": 1.1261,
+ "1958": 1.1276,
+ "1959": 1.18913,
+ "1960": 1.12817,
+ "1961": 1.12615,
+ "1962": 1.12581,
+ "1963": 1.12682,
+ "1964": 1.12747,
+ "1965": 1.14301,
+ "1966": 1.14417,
+ "1967": 1.14427,
+ "1968": 1.14017,
+ "1969": 1.13872,
+ "1970": 1.13824,
+ "1971": 1.14731,
+ "1972": 1.13727,
+ "1973": 1.13816,
+ "1974": 1.13684,
+ "1975": 1.13985,
+ "1976": 1.13777,
+ "1977": 1.13833,
+ "1978": 1.14247,
+ "1979": 1.14554,
+ "1980": 1.14074,
+ "1981": 1.1396,
+ "1982": 1.13784,
+ "1983": 1.19896,
+ "1984": 1.13952,
+ "1985": 1.13865,
+ "1986": 1.13959,
+ "1987": 1.13909,
+ "1988": 1.13875,
+ "1989": 1.13947,
+ "1990": 1.13762,
+ "1991": 1.13799,
+ "1992": 1.13904,
+ "1993": 1.13674,
+ "1994": 1.13869,
+ "1995": 1.13884,
+ "1996": 1.13807,
+ "1997": 1.13986,
+ "1998": 1.14151,
+ "1999": 1.13582,
+ "2000": 1.16726
+ }
+ }
+}
\ No newline at end of file
diff --git a/tests/functional_tests/test_cases/gpt/gpt3_weekly_dgx_h100_mcore_tp2_pp2_current_scaling_native_fp8_tp_pp_sp_tp_overlap/model_config.yaml b/tests/functional_tests/test_cases/gpt/gpt3_weekly_dgx_h100_mcore_tp2_pp2_current_scaling_native_fp8_tp_pp_sp_tp_overlap/model_config.yaml
index 5668a7575e2..15ac9782df5 100644
--- a/tests/functional_tests/test_cases/gpt/gpt3_weekly_dgx_h100_mcore_tp2_pp2_current_scaling_native_fp8_tp_pp_sp_tp_overlap/model_config.yaml
+++ b/tests/functional_tests/test_cases/gpt/gpt3_weekly_dgx_h100_mcore_tp2_pp2_current_scaling_native_fp8_tp_pp_sp_tp_overlap/model_config.yaml
@@ -42,7 +42,7 @@ MODEL_ARGS:
--pipeline-model-parallel-size: 2
--sequence-parallel: true
--tp-comm-overlap: true
- --tp-comm-overlap-cfg: tests/functional_tests/test_cases/gpt/gpt3_345m_weekly_dgx_h100_1N8G_mcore_tp2_pp2_current_scaling_native_fp8_tp_pp_sp_tp_overlap/tp_comm_overlap_cfg.yaml
+ --tp-comm-overlap-cfg: tests/functional_tests/test_cases/gpt/gpt3_weekly_dgx_h100_mcore_tp2_pp2_current_scaling_native_fp8_tp_pp_sp_tp_overlap/tp_comm_overlap_cfg.yaml
--deterministic-mode: true
--no-gradient-accumulation-fusion: true
--fp8-format: hybrid
diff --git a/tests/functional_tests/test_cases/gpt/gpt3_weekly_dgx_h100_mcore_tp4_cp2_native_fp8_tp_sp_cp_tp_overlap/golden_values_dev_dgx_h100.json b/tests/functional_tests/test_cases/gpt/gpt3_weekly_dgx_h100_mcore_tp4_cp2_native_fp8_tp_sp_cp_tp_overlap/golden_values_dev_dgx_h100.json
new file mode 100644
index 00000000000..b6e543e2cf8
--- /dev/null
+++ b/tests/functional_tests/test_cases/gpt/gpt3_weekly_dgx_h100_mcore_tp4_cp2_native_fp8_tp_sp_cp_tp_overlap/golden_values_dev_dgx_h100.json
@@ -0,0 +1,10037 @@
+{
+ "lm loss": {
+ "start_step": 1,
+ "end_step": 2000,
+ "step_interval": 1,
+ "values": {
+ "1": 10.85229,
+ "2": 10.85951,
+ "3": 10.85469,
+ "4": 10.86843,
+ "5": 10.85304,
+ "6": 10.85362,
+ "7": 10.8602,
+ "8": 10.85298,
+ "9": 10.84874,
+ "10": 10.84674,
+ "11": 10.83863,
+ "12": 10.83549,
+ "13": 10.82524,
+ "14": 10.84078,
+ "15": 10.78613,
+ "16": 10.79372,
+ "17": 10.76553,
+ "18": 10.78902,
+ "19": 10.73057,
+ "20": 10.69489,
+ "21": 10.64595,
+ "22": 10.64791,
+ "23": 10.65524,
+ "24": 10.55349,
+ "25": 10.56424,
+ "26": 10.63262,
+ "27": 10.47084,
+ "28": 10.471,
+ "29": 10.36495,
+ "30": 10.27406,
+ "31": 10.43126,
+ "32": 10.35361,
+ "33": 10.22439,
+ "34": 10.17135,
+ "35": 10.23744,
+ "36": 10.15766,
+ "37": 10.10704,
+ "38": 10.03631,
+ "39": 10.04895,
+ "40": 10.06978,
+ "41": 9.95276,
+ "42": 9.95577,
+ "43": 9.87217,
+ "44": 9.99154,
+ "45": 10.00766,
+ "46": 9.84803,
+ "47": 10.00018,
+ "48": 9.81816,
+ "49": 9.94941,
+ "50": 9.94449,
+ "51": 9.5964,
+ "52": 9.79483,
+ "53": 9.63207,
+ "54": 9.8854,
+ "55": 9.74063,
+ "56": 9.85006,
+ "57": 9.86123,
+ "58": 9.87737,
+ "59": 9.54716,
+ "60": 9.64756,
+ "61": 9.87994,
+ "62": 9.76465,
+ "63": 9.68066,
+ "64": 9.82801,
+ "65": 9.59733,
+ "66": 9.62928,
+ "67": 9.74212,
+ "68": 9.60593,
+ "69": 9.29694,
+ "70": 9.42495,
+ "71": 9.79013,
+ "72": 9.71358,
+ "73": 9.61909,
+ "74": 9.45334,
+ "75": 9.24289,
+ "76": 9.50821,
+ "77": 9.57857,
+ "78": 9.56035,
+ "79": 9.31048,
+ "80": 9.36161,
+ "81": 9.46136,
+ "82": 9.55628,
+ "83": 9.53353,
+ "84": 9.35526,
+ "85": 9.40111,
+ "86": 9.65137,
+ "87": 9.23621,
+ "88": 9.48942,
+ "89": 9.22457,
+ "90": 9.41443,
+ "91": 9.39014,
+ "92": 9.3793,
+ "93": 9.36366,
+ "94": 9.51552,
+ "95": 9.42012,
+ "96": 9.33698,
+ "97": 9.20729,
+ "98": 9.49265,
+ "99": 9.29333,
+ "100": 9.35883,
+ "101": 9.24766,
+ "102": 9.24259,
+ "103": 9.07796,
+ "104": 9.16832,
+ "105": 9.37671,
+ "106": 9.15179,
+ "107": 9.17832,
+ "108": 9.31483,
+ "109": 9.28984,
+ "110": 9.36705,
+ "111": 9.17605,
+ "112": 9.23281,
+ "113": 9.35413,
+ "114": 9.35742,
+ "115": 9.32337,
+ "116": 9.00364,
+ "117": 9.06445,
+ "118": 9.06523,
+ "119": 9.22504,
+ "120": 9.08324,
+ "121": 9.19428,
+ "122": 9.14006,
+ "123": 9.25894,
+ "124": 9.45689,
+ "125": 9.21857,
+ "126": 9.0614,
+ "127": 9.01413,
+ "128": 9.22025,
+ "129": 8.98394,
+ "130": 9.14098,
+ "131": 9.15643,
+ "132": 9.03479,
+ "133": 8.86261,
+ "134": 9.18468,
+ "135": 8.88922,
+ "136": 9.1645,
+ "137": 9.15944,
+ "138": 9.23186,
+ "139": 9.08834,
+ "140": 8.87267,
+ "141": 9.29752,
+ "142": 9.19877,
+ "143": 9.12079,
+ "144": 9.24324,
+ "145": 9.10527,
+ "146": 8.98338,
+ "147": 8.9881,
+ "148": 9.1361,
+ "149": 9.06877,
+ "150": 9.01122,
+ "151": 8.93192,
+ "152": 8.87852,
+ "153": 9.06711,
+ "154": 9.1802,
+ "155": 9.13786,
+ "156": 9.05095,
+ "157": 9.15163,
+ "158": 9.05301,
+ "159": 9.03638,
+ "160": 8.89244,
+ "161": 9.04764,
+ "162": 8.89639,
+ "163": 8.84472,
+ "164": 8.97496,
+ "165": 8.93105,
+ "166": 8.65677,
+ "167": 8.83411,
+ "168": 8.8203,
+ "169": 8.65961,
+ "170": 9.04726,
+ "171": 8.72167,
+ "172": 8.82105,
+ "173": 8.91105,
+ "174": 8.85007,
+ "175": 8.70985,
+ "176": 8.7611,
+ "177": 8.76567,
+ "178": 8.72394,
+ "179": 8.64132,
+ "180": 8.74357,
+ "181": 8.6941,
+ "182": 8.72315,
+ "183": 9.08667,
+ "184": 8.60959,
+ "185": 8.88334,
+ "186": 8.74346,
+ "187": 8.57546,
+ "188": 8.6841,
+ "189": 8.86656,
+ "190": 8.53754,
+ "191": 8.66593,
+ "192": 8.61152,
+ "193": 8.5763,
+ "194": 8.75183,
+ "195": 8.5938,
+ "196": 8.7761,
+ "197": 8.744,
+ "198": 8.63042,
+ "199": 8.77202,
+ "200": 8.73627,
+ "201": 8.67068,
+ "202": 8.55099,
+ "203": 8.54134,
+ "204": 8.71213,
+ "205": 8.22486,
+ "206": 8.85986,
+ "207": 8.67928,
+ "208": 8.70826,
+ "209": 8.75243,
+ "210": 8.58226,
+ "211": 8.84167,
+ "212": 8.4913,
+ "213": 8.57316,
+ "214": 8.51316,
+ "215": 8.56549,
+ "216": 8.50617,
+ "217": 8.53369,
+ "218": 8.53635,
+ "219": 8.64298,
+ "220": 8.54526,
+ "221": 8.39761,
+ "222": 8.50474,
+ "223": 8.44078,
+ "224": 8.52901,
+ "225": 8.5708,
+ "226": 8.44247,
+ "227": 8.67823,
+ "228": 8.3859,
+ "229": 8.4537,
+ "230": 8.4985,
+ "231": 8.50257,
+ "232": 8.49898,
+ "233": 8.49438,
+ "234": 8.64018,
+ "235": 8.5617,
+ "236": 8.39791,
+ "237": 8.49075,
+ "238": 8.30637,
+ "239": 8.56099,
+ "240": 8.67125,
+ "241": 8.447,
+ "242": 8.47179,
+ "243": 8.51685,
+ "244": 8.36975,
+ "245": 8.59641,
+ "246": 8.59557,
+ "247": 8.43962,
+ "248": 8.50986,
+ "249": 8.52277,
+ "250": 8.42301,
+ "251": 8.3783,
+ "252": 8.54698,
+ "253": 8.3164,
+ "254": 8.35246,
+ "255": 8.29609,
+ "256": 8.20858,
+ "257": 8.39462,
+ "258": 8.45148,
+ "259": 8.23213,
+ "260": 8.24039,
+ "261": 8.23733,
+ "262": 8.34866,
+ "263": 8.30632,
+ "264": 8.1907,
+ "265": 8.33202,
+ "266": 8.2336,
+ "267": 7.9013,
+ "268": 8.37861,
+ "269": 8.40384,
+ "270": 8.26475,
+ "271": 8.27885,
+ "272": 8.31844,
+ "273": 8.13253,
+ "274": 8.09818,
+ "275": 8.00901,
+ "276": 7.92522,
+ "277": 8.23699,
+ "278": 8.04701,
+ "279": 7.96356,
+ "280": 7.75515,
+ "281": 8.10016,
+ "282": 8.14722,
+ "283": 8.15666,
+ "284": 8.10022,
+ "285": 8.06894,
+ "286": 7.90037,
+ "287": 7.99127,
+ "288": 8.24359,
+ "289": 8.17176,
+ "290": 8.12684,
+ "291": 8.25357,
+ "292": 8.0756,
+ "293": 8.11914,
+ "294": 7.97501,
+ "295": 7.96533,
+ "296": 8.23576,
+ "297": 7.79081,
+ "298": 8.04236,
+ "299": 7.93831,
+ "300": 7.8498,
+ "301": 8.00964,
+ "302": 7.94515,
+ "303": 7.99053,
+ "304": 7.95899,
+ "305": 7.9946,
+ "306": 7.9738,
+ "307": 7.98707,
+ "308": 7.9953,
+ "309": 8.0059,
+ "310": 7.97168,
+ "311": 7.92562,
+ "312": 7.88182,
+ "313": 7.82955,
+ "314": 7.82035,
+ "315": 7.82475,
+ "316": 7.74495,
+ "317": 7.92567,
+ "318": 7.97631,
+ "319": 7.82443,
+ "320": 7.563,
+ "321": 7.74534,
+ "322": 7.82917,
+ "323": 7.76703,
+ "324": 7.90668,
+ "325": 7.79387,
+ "326": 7.64901,
+ "327": 7.86137,
+ "328": 7.7832,
+ "329": 7.87669,
+ "330": 7.74815,
+ "331": 7.52005,
+ "332": 7.81037,
+ "333": 7.8379,
+ "334": 7.67759,
+ "335": 7.69435,
+ "336": 7.90998,
+ "337": 7.64618,
+ "338": 7.89178,
+ "339": 7.7192,
+ "340": 7.75318,
+ "341": 7.70375,
+ "342": 7.81451,
+ "343": 7.61028,
+ "344": 7.58433,
+ "345": 7.60474,
+ "346": 7.45825,
+ "347": 7.55021,
+ "348": 7.67669,
+ "349": 7.57925,
+ "350": 7.65118,
+ "351": 7.74172,
+ "352": 7.69877,
+ "353": 7.4955,
+ "354": 7.73645,
+ "355": 7.75823,
+ "356": 7.76871,
+ "357": 7.8083,
+ "358": 7.59223,
+ "359": 7.54129,
+ "360": 7.62161,
+ "361": 7.53913,
+ "362": 7.75707,
+ "363": 7.58184,
+ "364": 7.57393,
+ "365": 7.61381,
+ "366": 7.30007,
+ "367": 7.55433,
+ "368": 7.4381,
+ "369": 7.34072,
+ "370": 7.45786,
+ "371": 7.45479,
+ "372": 7.64528,
+ "373": 7.51803,
+ "374": 7.43579,
+ "375": 7.52279,
+ "376": 7.33856,
+ "377": 7.23275,
+ "378": 7.53208,
+ "379": 7.48549,
+ "380": 7.37893,
+ "381": 7.46259,
+ "382": 7.28593,
+ "383": 7.26774,
+ "384": 7.4035,
+ "385": 7.38617,
+ "386": 7.2246,
+ "387": 7.41197,
+ "388": 7.27354,
+ "389": 7.42884,
+ "390": 7.23295,
+ "391": 7.63854,
+ "392": 7.32743,
+ "393": 7.41119,
+ "394": 7.46811,
+ "395": 7.43164,
+ "396": 7.27624,
+ "397": 7.22237,
+ "398": 7.41314,
+ "399": 7.14965,
+ "400": 7.28882,
+ "401": 7.34645,
+ "402": 7.38389,
+ "403": 7.27445,
+ "404": 7.29549,
+ "405": 7.25441,
+ "406": 7.20955,
+ "407": 7.35305,
+ "408": 7.17476,
+ "409": 7.15738,
+ "410": 7.30843,
+ "411": 7.21046,
+ "412": 7.19143,
+ "413": 7.22421,
+ "414": 6.90584,
+ "415": 7.32329,
+ "416": 7.41955,
+ "417": 7.01436,
+ "418": 7.26656,
+ "419": 7.03251,
+ "420": 7.40294,
+ "421": 7.17304,
+ "422": 7.22884,
+ "423": 7.08611,
+ "424": 7.2354,
+ "425": 7.3087,
+ "426": 7.28003,
+ "427": 7.12262,
+ "428": 7.08425,
+ "429": 6.87125,
+ "430": 7.19779,
+ "431": 6.99763,
+ "432": 7.22298,
+ "433": 6.96906,
+ "434": 6.95232,
+ "435": 7.01097,
+ "436": 7.00141,
+ "437": 6.9848,
+ "438": 6.99447,
+ "439": 6.93128,
+ "440": 7.05472,
+ "441": 7.03406,
+ "442": 7.09324,
+ "443": 7.0854,
+ "444": 6.69941,
+ "445": 6.98741,
+ "446": 7.13474,
+ "447": 7.11726,
+ "448": 6.97509,
+ "449": 7.04203,
+ "450": 7.00855,
+ "451": 6.82317,
+ "452": 6.90281,
+ "453": 7.00796,
+ "454": 6.96028,
+ "455": 7.02393,
+ "456": 6.98781,
+ "457": 6.96156,
+ "458": 6.89735,
+ "459": 6.68323,
+ "460": 7.05439,
+ "461": 7.088,
+ "462": 6.86315,
+ "463": 7.04576,
+ "464": 6.64275,
+ "465": 7.02272,
+ "466": 6.99895,
+ "467": 6.99097,
+ "468": 6.94728,
+ "469": 6.82004,
+ "470": 7.0355,
+ "471": 6.87321,
+ "472": 6.95214,
+ "473": 6.81396,
+ "474": 6.96547,
+ "475": 7.1584,
+ "476": 6.75391,
+ "477": 6.88861,
+ "478": 6.89832,
+ "479": 6.69636,
+ "480": 7.01803,
+ "481": 6.98503,
+ "482": 6.72248,
+ "483": 6.77484,
+ "484": 6.74297,
+ "485": 6.92045,
+ "486": 7.05544,
+ "487": 6.62222,
+ "488": 6.87375,
+ "489": 6.76024,
+ "490": 6.81377,
+ "491": 6.69837,
+ "492": 6.68149,
+ "493": 6.75646,
+ "494": 6.66282,
+ "495": 6.62263,
+ "496": 6.57706,
+ "497": 6.8292,
+ "498": 6.63548,
+ "499": 6.84385,
+ "500": 6.64283,
+ "501": 6.71966,
+ "502": 6.82988,
+ "503": 6.69833,
+ "504": 6.60751,
+ "505": 6.6112,
+ "506": 6.73586,
+ "507": 6.85391,
+ "508": 6.84629,
+ "509": 6.6384,
+ "510": 6.81034,
+ "511": 6.72977,
+ "512": 6.72804,
+ "513": 6.64821,
+ "514": 6.70064,
+ "515": 6.43824,
+ "516": 6.73421,
+ "517": 6.69542,
+ "518": 6.52993,
+ "519": 6.62474,
+ "520": 6.84935,
+ "521": 6.65329,
+ "522": 6.6979,
+ "523": 6.73262,
+ "524": 6.72634,
+ "525": 6.6655,
+ "526": 6.40663,
+ "527": 6.79088,
+ "528": 6.65206,
+ "529": 6.62295,
+ "530": 6.61639,
+ "531": 6.63503,
+ "532": 6.62382,
+ "533": 6.75435,
+ "534": 6.60296,
+ "535": 6.74138,
+ "536": 6.61812,
+ "537": 6.63086,
+ "538": 6.52418,
+ "539": 6.54299,
+ "540": 6.57593,
+ "541": 6.44382,
+ "542": 6.66189,
+ "543": 6.67325,
+ "544": 6.66927,
+ "545": 6.80511,
+ "546": 6.6246,
+ "547": 6.40979,
+ "548": 6.71663,
+ "549": 6.68986,
+ "550": 6.51987,
+ "551": 6.74092,
+ "552": 6.63227,
+ "553": 6.47534,
+ "554": 6.62778,
+ "555": 6.45222,
+ "556": 6.60749,
+ "557": 6.62431,
+ "558": 6.37676,
+ "559": 6.36118,
+ "560": 6.5756,
+ "561": 6.72381,
+ "562": 6.62768,
+ "563": 6.73287,
+ "564": 6.34176,
+ "565": 6.50706,
+ "566": 6.6902,
+ "567": 6.55838,
+ "568": 6.50084,
+ "569": 6.44415,
+ "570": 6.35619,
+ "571": 6.62259,
+ "572": 6.30471,
+ "573": 6.5721,
+ "574": 6.46259,
+ "575": 6.63541,
+ "576": 6.50701,
+ "577": 6.51656,
+ "578": 6.47574,
+ "579": 6.45618,
+ "580": 6.5583,
+ "581": 6.59714,
+ "582": 6.46959,
+ "583": 6.50413,
+ "584": 6.51087,
+ "585": 6.41424,
+ "586": 6.40258,
+ "587": 6.4501,
+ "588": 6.55622,
+ "589": 6.61456,
+ "590": 6.27891,
+ "591": 6.66415,
+ "592": 6.2545,
+ "593": 6.46521,
+ "594": 6.37467,
+ "595": 6.34819,
+ "596": 6.25003,
+ "597": 6.18054,
+ "598": 6.44279,
+ "599": 6.38602,
+ "600": 6.44414,
+ "601": 6.25051,
+ "602": 6.51804,
+ "603": 6.50819,
+ "604": 6.37382,
+ "605": 6.48026,
+ "606": 6.3013,
+ "607": 6.51999,
+ "608": 6.66049,
+ "609": 6.16075,
+ "610": 6.55805,
+ "611": 6.38737,
+ "612": 6.56702,
+ "613": 6.41056,
+ "614": 6.18827,
+ "615": 6.38286,
+ "616": 6.34421,
+ "617": 6.36273,
+ "618": 6.43626,
+ "619": 6.12502,
+ "620": 6.3943,
+ "621": 6.44427,
+ "622": 6.38402,
+ "623": 6.56769,
+ "624": 6.34417,
+ "625": 6.26521,
+ "626": 6.28634,
+ "627": 6.4276,
+ "628": 6.24043,
+ "629": 6.57298,
+ "630": 6.3523,
+ "631": 6.33431,
+ "632": 6.29554,
+ "633": 6.24213,
+ "634": 6.29476,
+ "635": 6.53142,
+ "636": 6.23005,
+ "637": 6.62121,
+ "638": 6.00686,
+ "639": 6.26506,
+ "640": 6.2796,
+ "641": 6.19435,
+ "642": 6.27007,
+ "643": 6.44413,
+ "644": 6.2445,
+ "645": 6.23092,
+ "646": 6.38932,
+ "647": 6.3209,
+ "648": 6.34188,
+ "649": 6.33297,
+ "650": 6.47025,
+ "651": 6.31782,
+ "652": 6.23993,
+ "653": 6.36817,
+ "654": 6.43495,
+ "655": 6.5135,
+ "656": 6.31371,
+ "657": 6.4163,
+ "658": 6.22993,
+ "659": 6.1432,
+ "660": 6.3808,
+ "661": 6.15725,
+ "662": 6.2613,
+ "663": 6.36151,
+ "664": 6.32043,
+ "665": 6.39194,
+ "666": 6.15182,
+ "667": 6.18562,
+ "668": 6.22741,
+ "669": 6.20408,
+ "670": 6.23602,
+ "671": 6.22904,
+ "672": 6.47492,
+ "673": 6.32812,
+ "674": 6.28343,
+ "675": 6.37362,
+ "676": 6.38018,
+ "677": 6.29511,
+ "678": 6.26804,
+ "679": 6.22803,
+ "680": 6.28357,
+ "681": 6.19077,
+ "682": 6.07906,
+ "683": 6.26403,
+ "684": 6.31575,
+ "685": 6.2874,
+ "686": 6.14011,
+ "687": 6.27685,
+ "688": 6.19835,
+ "689": 6.61075,
+ "690": 6.16856,
+ "691": 6.17286,
+ "692": 6.2649,
+ "693": 6.13689,
+ "694": 6.22553,
+ "695": 6.31786,
+ "696": 6.1061,
+ "697": 6.14556,
+ "698": 6.21959,
+ "699": 6.45326,
+ "700": 6.03519,
+ "701": 6.05302,
+ "702": 6.23703,
+ "703": 6.17441,
+ "704": 6.20621,
+ "705": 6.11844,
+ "706": 6.06567,
+ "707": 6.24456,
+ "708": 6.30245,
+ "709": 5.99551,
+ "710": 6.15229,
+ "711": 6.2479,
+ "712": 6.17146,
+ "713": 5.88608,
+ "714": 6.09975,
+ "715": 6.10497,
+ "716": 6.40586,
+ "717": 6.18363,
+ "718": 6.23537,
+ "719": 6.26862,
+ "720": 6.25804,
+ "721": 6.25605,
+ "722": 6.22472,
+ "723": 6.07187,
+ "724": 6.22017,
+ "725": 6.0314,
+ "726": 6.29244,
+ "727": 6.00644,
+ "728": 6.03616,
+ "729": 6.0826,
+ "730": 6.17412,
+ "731": 6.09163,
+ "732": 6.07888,
+ "733": 6.11348,
+ "734": 6.37763,
+ "735": 6.26791,
+ "736": 6.17709,
+ "737": 6.36077,
+ "738": 6.13247,
+ "739": 6.14636,
+ "740": 5.87836,
+ "741": 6.00499,
+ "742": 5.98594,
+ "743": 6.17515,
+ "744": 6.02317,
+ "745": 6.14565,
+ "746": 6.03122,
+ "747": 6.09452,
+ "748": 6.22864,
+ "749": 5.93308,
+ "750": 6.16381,
+ "751": 5.95292,
+ "752": 6.01389,
+ "753": 6.02392,
+ "754": 6.28379,
+ "755": 6.12598,
+ "756": 6.2443,
+ "757": 6.01404,
+ "758": 6.19738,
+ "759": 6.22084,
+ "760": 6.02115,
+ "761": 6.1856,
+ "762": 6.21798,
+ "763": 6.02971,
+ "764": 5.95856,
+ "765": 5.92315,
+ "766": 5.96127,
+ "767": 5.81063,
+ "768": 6.18012,
+ "769": 6.27004,
+ "770": 6.28915,
+ "771": 5.78425,
+ "772": 6.0231,
+ "773": 6.17908,
+ "774": 5.87868,
+ "775": 6.02111,
+ "776": 6.12258,
+ "777": 5.875,
+ "778": 6.04901,
+ "779": 5.86583,
+ "780": 6.13275,
+ "781": 5.8451,
+ "782": 6.03644,
+ "783": 5.94982,
+ "784": 5.91239,
+ "785": 6.08718,
+ "786": 6.0949,
+ "787": 5.6498,
+ "788": 5.99117,
+ "789": 6.20208,
+ "790": 6.25533,
+ "791": 5.78584,
+ "792": 5.98398,
+ "793": 6.17232,
+ "794": 6.02303,
+ "795": 5.99758,
+ "796": 6.15575,
+ "797": 6.04799,
+ "798": 6.04773,
+ "799": 6.10394,
+ "800": 6.00523,
+ "801": 6.13976,
+ "802": 5.97143,
+ "803": 6.14303,
+ "804": 5.99897,
+ "805": 5.8162,
+ "806": 6.08016,
+ "807": 6.03933,
+ "808": 5.91779,
+ "809": 5.76774,
+ "810": 6.00748,
+ "811": 5.92407,
+ "812": 5.89853,
+ "813": 5.95603,
+ "814": 6.0199,
+ "815": 5.80113,
+ "816": 6.10732,
+ "817": 5.92704,
+ "818": 6.05349,
+ "819": 5.99954,
+ "820": 5.71925,
+ "821": 5.93871,
+ "822": 6.18742,
+ "823": 5.82051,
+ "824": 5.97479,
+ "825": 6.17898,
+ "826": 6.18992,
+ "827": 6.04811,
+ "828": 6.0618,
+ "829": 5.8808,
+ "830": 5.9338,
+ "831": 5.89066,
+ "832": 5.95946,
+ "833": 6.05775,
+ "834": 5.98694,
+ "835": 5.99225,
+ "836": 5.78808,
+ "837": 6.1001,
+ "838": 5.85774,
+ "839": 5.82603,
+ "840": 6.17451,
+ "841": 5.77389,
+ "842": 5.88244,
+ "843": 5.93827,
+ "844": 6.0037,
+ "845": 6.08214,
+ "846": 5.68388,
+ "847": 5.75348,
+ "848": 5.96075,
+ "849": 6.0909,
+ "850": 5.83839,
+ "851": 6.01221,
+ "852": 5.74277,
+ "853": 5.9819,
+ "854": 6.00994,
+ "855": 5.81104,
+ "856": 5.99027,
+ "857": 5.99462,
+ "858": 6.04349,
+ "859": 5.94378,
+ "860": 6.08776,
+ "861": 6.05806,
+ "862": 5.99259,
+ "863": 5.83184,
+ "864": 5.83727,
+ "865": 5.93014,
+ "866": 5.88373,
+ "867": 5.87071,
+ "868": 6.0603,
+ "869": 6.08011,
+ "870": 5.96321,
+ "871": 6.03762,
+ "872": 5.89053,
+ "873": 5.83933,
+ "874": 6.02181,
+ "875": 5.90658,
+ "876": 5.96303,
+ "877": 5.92074,
+ "878": 6.09702,
+ "879": 5.76213,
+ "880": 6.0073,
+ "881": 5.98795,
+ "882": 5.90217,
+ "883": 5.67039,
+ "884": 5.95748,
+ "885": 5.74054,
+ "886": 5.98445,
+ "887": 5.90648,
+ "888": 5.8314,
+ "889": 6.00733,
+ "890": 6.01123,
+ "891": 5.94286,
+ "892": 5.70277,
+ "893": 6.08459,
+ "894": 5.72165,
+ "895": 5.83588,
+ "896": 5.83978,
+ "897": 5.84943,
+ "898": 5.92347,
+ "899": 5.93201,
+ "900": 5.8958,
+ "901": 5.94689,
+ "902": 5.82987,
+ "903": 6.04738,
+ "904": 5.92586,
+ "905": 5.89894,
+ "906": 5.61575,
+ "907": 5.90522,
+ "908": 5.73333,
+ "909": 5.98526,
+ "910": 5.85686,
+ "911": 5.69844,
+ "912": 5.69856,
+ "913": 5.76407,
+ "914": 5.82436,
+ "915": 5.79681,
+ "916": 5.88608,
+ "917": 5.867,
+ "918": 5.8166,
+ "919": 5.80848,
+ "920": 5.88971,
+ "921": 5.8407,
+ "922": 5.62064,
+ "923": 6.03383,
+ "924": 5.60482,
+ "925": 5.61823,
+ "926": 5.85786,
+ "927": 5.95554,
+ "928": 5.83872,
+ "929": 5.82237,
+ "930": 5.95411,
+ "931": 5.75622,
+ "932": 5.59098,
+ "933": 5.63134,
+ "934": 5.80496,
+ "935": 5.63538,
+ "936": 5.8317,
+ "937": 5.96485,
+ "938": 5.58943,
+ "939": 5.79158,
+ "940": 5.96089,
+ "941": 5.72676,
+ "942": 5.83595,
+ "943": 5.87091,
+ "944": 5.95881,
+ "945": 5.70173,
+ "946": 5.55832,
+ "947": 5.74676,
+ "948": 5.79172,
+ "949": 5.82702,
+ "950": 5.84636,
+ "951": 5.72232,
+ "952": 5.6926,
+ "953": 5.67846,
+ "954": 5.72814,
+ "955": 5.52701,
+ "956": 5.6247,
+ "957": 5.84082,
+ "958": 5.79725,
+ "959": 5.57236,
+ "960": 5.8033,
+ "961": 5.83318,
+ "962": 5.76931,
+ "963": 5.768,
+ "964": 5.70825,
+ "965": 5.63755,
+ "966": 5.60344,
+ "967": 5.72795,
+ "968": 5.74037,
+ "969": 5.82565,
+ "970": 5.64868,
+ "971": 5.70857,
+ "972": 5.85255,
+ "973": 5.67308,
+ "974": 5.7177,
+ "975": 5.86027,
+ "976": 5.71074,
+ "977": 5.77363,
+ "978": 5.68598,
+ "979": 5.5901,
+ "980": 5.76431,
+ "981": 5.89808,
+ "982": 5.47164,
+ "983": 5.61909,
+ "984": 5.54693,
+ "985": 5.58914,
+ "986": 5.6395,
+ "987": 5.57215,
+ "988": 5.71212,
+ "989": 5.69568,
+ "990": 5.62713,
+ "991": 5.85071,
+ "992": 5.77178,
+ "993": 5.87182,
+ "994": 5.69827,
+ "995": 5.7311,
+ "996": 5.73947,
+ "997": 5.81776,
+ "998": 5.83946,
+ "999": 5.83213,
+ "1000": 5.68618,
+ "1001": 5.86902,
+ "1002": 5.75759,
+ "1003": 5.64206,
+ "1004": 5.80056,
+ "1005": 5.53357,
+ "1006": 5.3287,
+ "1007": 5.7697,
+ "1008": 5.79391,
+ "1009": 5.65438,
+ "1010": 5.78459,
+ "1011": 5.89696,
+ "1012": 5.62269,
+ "1013": 5.61367,
+ "1014": 5.67992,
+ "1015": 5.56146,
+ "1016": 5.87263,
+ "1017": 5.83169,
+ "1018": 5.62357,
+ "1019": 5.73336,
+ "1020": 5.61404,
+ "1021": 5.85353,
+ "1022": 5.49696,
+ "1023": 5.65062,
+ "1024": 5.74334,
+ "1025": 5.57222,
+ "1026": 5.40994,
+ "1027": 5.59905,
+ "1028": 5.68935,
+ "1029": 5.68346,
+ "1030": 5.68799,
+ "1031": 5.40526,
+ "1032": 5.78443,
+ "1033": 5.57561,
+ "1034": 5.6274,
+ "1035": 5.71529,
+ "1036": 5.62368,
+ "1037": 5.36621,
+ "1038": 5.66561,
+ "1039": 5.6477,
+ "1040": 5.57324,
+ "1041": 5.59731,
+ "1042": 5.81493,
+ "1043": 5.56271,
+ "1044": 5.46406,
+ "1045": 5.9683,
+ "1046": 5.48617,
+ "1047": 5.39181,
+ "1048": 5.49562,
+ "1049": 5.67791,
+ "1050": 5.69881,
+ "1051": 5.5776,
+ "1052": 5.68149,
+ "1053": 5.63114,
+ "1054": 5.45857,
+ "1055": 5.59887,
+ "1056": 5.67508,
+ "1057": 5.75628,
+ "1058": 5.56524,
+ "1059": 5.74843,
+ "1060": 5.82162,
+ "1061": 5.47233,
+ "1062": 5.65043,
+ "1063": 5.50248,
+ "1064": 5.59125,
+ "1065": 5.55564,
+ "1066": 5.74466,
+ "1067": 5.67043,
+ "1068": 5.44061,
+ "1069": 5.61122,
+ "1070": 5.81207,
+ "1071": 5.51069,
+ "1072": 5.62291,
+ "1073": 5.6192,
+ "1074": 5.52379,
+ "1075": 5.70748,
+ "1076": 5.5951,
+ "1077": 5.70681,
+ "1078": 5.56223,
+ "1079": 5.61677,
+ "1080": 5.64259,
+ "1081": 5.62201,
+ "1082": 5.50149,
+ "1083": 5.64213,
+ "1084": 5.55087,
+ "1085": 5.40393,
+ "1086": 5.62042,
+ "1087": 5.44171,
+ "1088": 5.51111,
+ "1089": 5.76887,
+ "1090": 5.52736,
+ "1091": 5.51307,
+ "1092": 5.40781,
+ "1093": 5.69672,
+ "1094": 5.56925,
+ "1095": 5.5731,
+ "1096": 5.61367,
+ "1097": 5.6454,
+ "1098": 5.65292,
+ "1099": 5.51436,
+ "1100": 5.63973,
+ "1101": 5.67989,
+ "1102": 5.53567,
+ "1103": 5.54943,
+ "1104": 5.53818,
+ "1105": 5.55271,
+ "1106": 5.68243,
+ "1107": 5.68309,
+ "1108": 5.78112,
+ "1109": 5.54014,
+ "1110": 5.6617,
+ "1111": 5.59215,
+ "1112": 5.58702,
+ "1113": 5.62687,
+ "1114": 5.61504,
+ "1115": 5.59863,
+ "1116": 5.66461,
+ "1117": 5.64732,
+ "1118": 5.65418,
+ "1119": 5.70846,
+ "1120": 5.63501,
+ "1121": 5.37809,
+ "1122": 5.23308,
+ "1123": 5.47298,
+ "1124": 5.65454,
+ "1125": 5.68419,
+ "1126": 5.68674,
+ "1127": 5.56954,
+ "1128": 5.62438,
+ "1129": 5.29406,
+ "1130": 5.54548,
+ "1131": 5.6238,
+ "1132": 5.72077,
+ "1133": 5.51615,
+ "1134": 5.55302,
+ "1135": 5.51992,
+ "1136": 5.42021,
+ "1137": 5.46757,
+ "1138": 5.5657,
+ "1139": 5.41524,
+ "1140": 5.26144,
+ "1141": 5.58424,
+ "1142": 5.64054,
+ "1143": 5.385,
+ "1144": 5.3823,
+ "1145": 5.36615,
+ "1146": 5.62886,
+ "1147": 5.49181,
+ "1148": 5.50478,
+ "1149": 5.51839,
+ "1150": 5.39997,
+ "1151": 5.5553,
+ "1152": 5.42174,
+ "1153": 5.4602,
+ "1154": 5.50372,
+ "1155": 5.44072,
+ "1156": 5.34868,
+ "1157": 5.66217,
+ "1158": 5.39889,
+ "1159": 5.33332,
+ "1160": 5.79511,
+ "1161": 5.53597,
+ "1162": 5.45589,
+ "1163": 5.52529,
+ "1164": 5.38319,
+ "1165": 5.52473,
+ "1166": 5.48721,
+ "1167": 5.36058,
+ "1168": 5.49334,
+ "1169": 5.40387,
+ "1170": 5.58667,
+ "1171": 5.48535,
+ "1172": 5.64049,
+ "1173": 5.62012,
+ "1174": 5.51308,
+ "1175": 5.34473,
+ "1176": 5.38256,
+ "1177": 5.55838,
+ "1178": 5.46714,
+ "1179": 5.49373,
+ "1180": 5.46571,
+ "1181": 5.55314,
+ "1182": 5.59825,
+ "1183": 5.76884,
+ "1184": 5.54748,
+ "1185": 5.28691,
+ "1186": 5.60427,
+ "1187": 5.55401,
+ "1188": 5.51546,
+ "1189": 5.38634,
+ "1190": 5.40233,
+ "1191": 5.38976,
+ "1192": 5.49689,
+ "1193": 5.46486,
+ "1194": 5.45443,
+ "1195": 5.32542,
+ "1196": 5.52268,
+ "1197": 5.47666,
+ "1198": 5.52589,
+ "1199": 5.38688,
+ "1200": 5.33164,
+ "1201": 5.49012,
+ "1202": 5.43748,
+ "1203": 5.49375,
+ "1204": 5.40666,
+ "1205": 5.48999,
+ "1206": 5.33478,
+ "1207": 5.58651,
+ "1208": 5.42414,
+ "1209": 5.2931,
+ "1210": 5.49969,
+ "1211": 5.5071,
+ "1212": 5.59732,
+ "1213": 5.41745,
+ "1214": 5.49785,
+ "1215": 5.23706,
+ "1216": 5.41194,
+ "1217": 5.38264,
+ "1218": 5.4506,
+ "1219": 5.48501,
+ "1220": 5.38351,
+ "1221": 5.4519,
+ "1222": 5.31254,
+ "1223": 5.47747,
+ "1224": 5.41418,
+ "1225": 5.42845,
+ "1226": 5.32249,
+ "1227": 5.47547,
+ "1228": 5.73249,
+ "1229": 5.32716,
+ "1230": 5.41211,
+ "1231": 5.07649,
+ "1232": 5.78792,
+ "1233": 5.28531,
+ "1234": 5.24399,
+ "1235": 5.36824,
+ "1236": 5.47881,
+ "1237": 5.20655,
+ "1238": 5.41404,
+ "1239": 5.40719,
+ "1240": 5.46621,
+ "1241": 5.57221,
+ "1242": 5.45465,
+ "1243": 5.43424,
+ "1244": 5.51633,
+ "1245": 5.19115,
+ "1246": 5.71566,
+ "1247": 5.43,
+ "1248": 5.29843,
+ "1249": 5.40246,
+ "1250": 5.34088,
+ "1251": 5.41904,
+ "1252": 5.57108,
+ "1253": 5.489,
+ "1254": 5.31099,
+ "1255": 5.51387,
+ "1256": 5.60708,
+ "1257": 5.42325,
+ "1258": 5.55956,
+ "1259": 5.47585,
+ "1260": 5.50779,
+ "1261": 5.63801,
+ "1262": 5.39496,
+ "1263": 5.32432,
+ "1264": 5.50348,
+ "1265": 5.30656,
+ "1266": 5.23675,
+ "1267": 5.37031,
+ "1268": 5.38615,
+ "1269": 5.14823,
+ "1270": 5.39882,
+ "1271": 5.27753,
+ "1272": 5.52297,
+ "1273": 5.29632,
+ "1274": 5.34638,
+ "1275": 5.37784,
+ "1276": 5.3975,
+ "1277": 5.4606,
+ "1278": 5.35501,
+ "1279": 5.43897,
+ "1280": 5.45708,
+ "1281": 5.4056,
+ "1282": 5.38482,
+ "1283": 5.42347,
+ "1284": 5.34377,
+ "1285": 5.50505,
+ "1286": 5.33544,
+ "1287": 5.58814,
+ "1288": 5.2615,
+ "1289": 5.42995,
+ "1290": 5.49991,
+ "1291": 5.49987,
+ "1292": 5.44631,
+ "1293": 5.4171,
+ "1294": 5.49492,
+ "1295": 5.34499,
+ "1296": 5.18358,
+ "1297": 5.16726,
+ "1298": 5.11761,
+ "1299": 5.30129,
+ "1300": 5.21142,
+ "1301": 5.30283,
+ "1302": 5.27612,
+ "1303": 5.35547,
+ "1304": 5.43158,
+ "1305": 5.36825,
+ "1306": 5.25293,
+ "1307": 5.19217,
+ "1308": 5.27071,
+ "1309": 5.40774,
+ "1310": 5.26053,
+ "1311": 5.37774,
+ "1312": 5.35324,
+ "1313": 5.29428,
+ "1314": 5.29224,
+ "1315": 5.41906,
+ "1316": 5.25856,
+ "1317": 5.27981,
+ "1318": 5.21136,
+ "1319": 5.34401,
+ "1320": 5.4177,
+ "1321": 5.44957,
+ "1322": 5.46219,
+ "1323": 5.37269,
+ "1324": 5.24973,
+ "1325": 5.40538,
+ "1326": 5.53891,
+ "1327": 5.38638,
+ "1328": 5.21164,
+ "1329": 5.41667,
+ "1330": 5.39695,
+ "1331": 5.30979,
+ "1332": 5.3112,
+ "1333": 5.36823,
+ "1334": 5.44451,
+ "1335": 5.36788,
+ "1336": 5.43552,
+ "1337": 5.46933,
+ "1338": 5.30246,
+ "1339": 5.1362,
+ "1340": 5.41205,
+ "1341": 5.34033,
+ "1342": 5.35625,
+ "1343": 5.47387,
+ "1344": 5.37842,
+ "1345": 5.34238,
+ "1346": 5.07927,
+ "1347": 5.38404,
+ "1348": 5.49312,
+ "1349": 5.40746,
+ "1350": 5.02698,
+ "1351": 5.31566,
+ "1352": 5.15947,
+ "1353": 5.3409,
+ "1354": 5.35878,
+ "1355": 5.11364,
+ "1356": 5.25842,
+ "1357": 5.28929,
+ "1358": 5.15831,
+ "1359": 5.10775,
+ "1360": 5.17385,
+ "1361": 5.30604,
+ "1362": 5.06672,
+ "1363": 5.29722,
+ "1364": 5.3953,
+ "1365": 5.01953,
+ "1366": 5.1147,
+ "1367": 5.33054,
+ "1368": 5.18248,
+ "1369": 5.22391,
+ "1370": 5.1961,
+ "1371": 5.27906,
+ "1372": 5.25988,
+ "1373": 5.28404,
+ "1374": 5.2779,
+ "1375": 5.46001,
+ "1376": 5.26713,
+ "1377": 5.26807,
+ "1378": 5.31427,
+ "1379": 5.22765,
+ "1380": 5.25807,
+ "1381": 5.47919,
+ "1382": 5.08739,
+ "1383": 5.37543,
+ "1384": 5.36108,
+ "1385": 5.39028,
+ "1386": 5.16582,
+ "1387": 5.16244,
+ "1388": 5.27616,
+ "1389": 5.30262,
+ "1390": 5.25131,
+ "1391": 5.26406,
+ "1392": 5.36794,
+ "1393": 5.37824,
+ "1394": 5.40104,
+ "1395": 5.32383,
+ "1396": 5.21137,
+ "1397": 5.2828,
+ "1398": 5.36587,
+ "1399": 5.35557,
+ "1400": 5.26522,
+ "1401": 5.35981,
+ "1402": 5.42507,
+ "1403": 5.19768,
+ "1404": 5.27957,
+ "1405": 5.11754,
+ "1406": 4.98933,
+ "1407": 5.39818,
+ "1408": 5.1921,
+ "1409": 5.39429,
+ "1410": 5.37153,
+ "1411": 4.91585,
+ "1412": 5.35244,
+ "1413": 5.41055,
+ "1414": 5.21699,
+ "1415": 5.44044,
+ "1416": 5.32598,
+ "1417": 5.39078,
+ "1418": 5.29894,
+ "1419": 5.31316,
+ "1420": 5.43638,
+ "1421": 5.39683,
+ "1422": 5.41859,
+ "1423": 4.99867,
+ "1424": 5.33177,
+ "1425": 5.58491,
+ "1426": 5.23068,
+ "1427": 5.31742,
+ "1428": 5.33463,
+ "1429": 5.07871,
+ "1430": 5.32748,
+ "1431": 5.32237,
+ "1432": 5.34216,
+ "1433": 5.18496,
+ "1434": 5.16175,
+ "1435": 5.20122,
+ "1436": 5.10715,
+ "1437": 5.22566,
+ "1438": 5.31423,
+ "1439": 5.34769,
+ "1440": 5.34295,
+ "1441": 5.16777,
+ "1442": 5.21935,
+ "1443": 5.20553,
+ "1444": 5.12984,
+ "1445": 5.07414,
+ "1446": 5.26456,
+ "1447": 5.25775,
+ "1448": 5.29302,
+ "1449": 5.24616,
+ "1450": 5.34316,
+ "1451": 5.07004,
+ "1452": 5.26796,
+ "1453": 5.1741,
+ "1454": 5.01458,
+ "1455": 5.12771,
+ "1456": 5.27213,
+ "1457": 5.1882,
+ "1458": 5.00695,
+ "1459": 5.2215,
+ "1460": 5.23955,
+ "1461": 5.08,
+ "1462": 4.97269,
+ "1463": 5.15114,
+ "1464": 5.22113,
+ "1465": 5.27344,
+ "1466": 5.36076,
+ "1467": 5.34631,
+ "1468": 5.2303,
+ "1469": 5.05117,
+ "1470": 5.12322,
+ "1471": 5.25302,
+ "1472": 5.12175,
+ "1473": 5.10167,
+ "1474": 5.21744,
+ "1475": 5.18613,
+ "1476": 5.15517,
+ "1477": 5.26215,
+ "1478": 5.30407,
+ "1479": 5.01063,
+ "1480": 5.182,
+ "1481": 5.25124,
+ "1482": 5.3494,
+ "1483": 5.27058,
+ "1484": 4.92644,
+ "1485": 5.29103,
+ "1486": 5.04435,
+ "1487": 4.88432,
+ "1488": 5.18325,
+ "1489": 5.10139,
+ "1490": 5.04545,
+ "1491": 5.3188,
+ "1492": 5.22283,
+ "1493": 4.94061,
+ "1494": 5.10891,
+ "1495": 5.13402,
+ "1496": 5.05779,
+ "1497": 5.36536,
+ "1498": 5.30609,
+ "1499": 5.143,
+ "1500": 5.09554,
+ "1501": 5.0349,
+ "1502": 5.15423,
+ "1503": 5.43131,
+ "1504": 5.32574,
+ "1505": 5.00836,
+ "1506": 5.14423,
+ "1507": 5.16501,
+ "1508": 5.16864,
+ "1509": 5.3204,
+ "1510": 5.02703,
+ "1511": 5.1198,
+ "1512": 4.98354,
+ "1513": 5.1699,
+ "1514": 5.33407,
+ "1515": 5.36306,
+ "1516": 5.27572,
+ "1517": 5.2256,
+ "1518": 5.02899,
+ "1519": 5.29833,
+ "1520": 5.13757,
+ "1521": 5.15715,
+ "1522": 5.33462,
+ "1523": 5.24144,
+ "1524": 5.06791,
+ "1525": 5.20708,
+ "1526": 5.27861,
+ "1527": 5.25864,
+ "1528": 5.2395,
+ "1529": 5.18253,
+ "1530": 5.23913,
+ "1531": 5.09996,
+ "1532": 5.15679,
+ "1533": 5.05231,
+ "1534": 5.21917,
+ "1535": 5.16769,
+ "1536": 5.102,
+ "1537": 5.0318,
+ "1538": 4.91991,
+ "1539": 5.2394,
+ "1540": 5.11391,
+ "1541": 5.25502,
+ "1542": 5.23775,
+ "1543": 5.05438,
+ "1544": 5.08156,
+ "1545": 5.11794,
+ "1546": 5.32713,
+ "1547": 5.10763,
+ "1548": 5.23418,
+ "1549": 5.23089,
+ "1550": 4.97536,
+ "1551": 5.25942,
+ "1552": 5.0226,
+ "1553": 5.14887,
+ "1554": 5.11051,
+ "1555": 5.11223,
+ "1556": 5.19882,
+ "1557": 5.08844,
+ "1558": 5.22982,
+ "1559": 5.00137,
+ "1560": 5.11269,
+ "1561": 5.14639,
+ "1562": 5.18443,
+ "1563": 5.24639,
+ "1564": 5.26429,
+ "1565": 5.08809,
+ "1566": 5.29393,
+ "1567": 5.04372,
+ "1568": 5.08304,
+ "1569": 5.2002,
+ "1570": 5.17168,
+ "1571": 4.95228,
+ "1572": 5.04524,
+ "1573": 5.02748,
+ "1574": 4.99831,
+ "1575": 5.23124,
+ "1576": 5.20891,
+ "1577": 5.12722,
+ "1578": 5.36355,
+ "1579": 4.94343,
+ "1580": 5.12556,
+ "1581": 5.09739,
+ "1582": 5.28014,
+ "1583": 5.04619,
+ "1584": 5.0566,
+ "1585": 5.11727,
+ "1586": 5.30646,
+ "1587": 5.13281,
+ "1588": 5.22351,
+ "1589": 4.83814,
+ "1590": 5.09825,
+ "1591": 5.18082,
+ "1592": 5.14078,
+ "1593": 5.23646,
+ "1594": 5.11532,
+ "1595": 5.10761,
+ "1596": 5.19194,
+ "1597": 5.11362,
+ "1598": 5.16252,
+ "1599": 5.18865,
+ "1600": 4.86676,
+ "1601": 5.11898,
+ "1602": 5.22827,
+ "1603": 5.19524,
+ "1604": 5.05797,
+ "1605": 5.03277,
+ "1606": 4.98991,
+ "1607": 5.06915,
+ "1608": 4.97927,
+ "1609": 5.07061,
+ "1610": 5.04561,
+ "1611": 4.9918,
+ "1612": 4.75806,
+ "1613": 5.03141,
+ "1614": 4.87811,
+ "1615": 5.07817,
+ "1616": 5.22549,
+ "1617": 5.06182,
+ "1618": 4.98945,
+ "1619": 5.18486,
+ "1620": 5.14429,
+ "1621": 5.31666,
+ "1622": 5.06737,
+ "1623": 5.15063,
+ "1624": 5.1305,
+ "1625": 5.12197,
+ "1626": 5.10206,
+ "1627": 5.1085,
+ "1628": 5.06234,
+ "1629": 4.93316,
+ "1630": 5.06616,
+ "1631": 5.05719,
+ "1632": 5.10145,
+ "1633": 4.97087,
+ "1634": 4.92194,
+ "1635": 5.05013,
+ "1636": 4.9202,
+ "1637": 5.22863,
+ "1638": 5.15783,
+ "1639": 4.9808,
+ "1640": 5.00716,
+ "1641": 5.12367,
+ "1642": 5.0869,
+ "1643": 5.05029,
+ "1644": 5.12283,
+ "1645": 4.96415,
+ "1646": 5.12257,
+ "1647": 5.03267,
+ "1648": 5.1903,
+ "1649": 4.92263,
+ "1650": 5.0596,
+ "1651": 4.93391,
+ "1652": 5.21143,
+ "1653": 5.1587,
+ "1654": 5.13384,
+ "1655": 5.16235,
+ "1656": 5.34793,
+ "1657": 5.21074,
+ "1658": 5.04155,
+ "1659": 4.92889,
+ "1660": 4.8117,
+ "1661": 5.02968,
+ "1662": 5.14515,
+ "1663": 5.15868,
+ "1664": 4.98471,
+ "1665": 5.11027,
+ "1666": 5.10315,
+ "1667": 4.84929,
+ "1668": 5.10956,
+ "1669": 5.07311,
+ "1670": 5.11152,
+ "1671": 5.16545,
+ "1672": 4.77709,
+ "1673": 5.03502,
+ "1674": 4.91572,
+ "1675": 5.04406,
+ "1676": 5.0023,
+ "1677": 4.80013,
+ "1678": 5.02745,
+ "1679": 4.88908,
+ "1680": 5.03791,
+ "1681": 5.06371,
+ "1682": 5.03586,
+ "1683": 4.90255,
+ "1684": 5.06133,
+ "1685": 5.13096,
+ "1686": 5.075,
+ "1687": 4.97679,
+ "1688": 5.17279,
+ "1689": 5.1507,
+ "1690": 4.99681,
+ "1691": 4.99961,
+ "1692": 4.91412,
+ "1693": 5.02305,
+ "1694": 4.94741,
+ "1695": 4.91895,
+ "1696": 5.0846,
+ "1697": 5.05067,
+ "1698": 4.95116,
+ "1699": 5.00638,
+ "1700": 4.94576,
+ "1701": 5.16681,
+ "1702": 5.07316,
+ "1703": 5.16582,
+ "1704": 5.14235,
+ "1705": 4.96408,
+ "1706": 4.98303,
+ "1707": 4.78833,
+ "1708": 5.03283,
+ "1709": 5.2281,
+ "1710": 5.02918,
+ "1711": 5.18873,
+ "1712": 5.19088,
+ "1713": 5.03631,
+ "1714": 5.04689,
+ "1715": 4.91662,
+ "1716": 4.93663,
+ "1717": 4.86445,
+ "1718": 5.02654,
+ "1719": 5.12575,
+ "1720": 5.02353,
+ "1721": 4.9343,
+ "1722": 5.06572,
+ "1723": 4.93302,
+ "1724": 5.03906,
+ "1725": 5.19169,
+ "1726": 5.06497,
+ "1727": 4.91076,
+ "1728": 5.01922,
+ "1729": 5.04885,
+ "1730": 4.91107,
+ "1731": 5.00108,
+ "1732": 4.91468,
+ "1733": 5.12873,
+ "1734": 4.83023,
+ "1735": 5.21293,
+ "1736": 4.91729,
+ "1737": 4.86164,
+ "1738": 4.97933,
+ "1739": 5.16149,
+ "1740": 4.84041,
+ "1741": 4.78298,
+ "1742": 4.91062,
+ "1743": 5.09353,
+ "1744": 4.98531,
+ "1745": 4.82544,
+ "1746": 4.94973,
+ "1747": 4.86843,
+ "1748": 5.06696,
+ "1749": 4.86793,
+ "1750": 5.01333,
+ "1751": 5.12023,
+ "1752": 4.90813,
+ "1753": 5.09204,
+ "1754": 5.05813,
+ "1755": 4.89777,
+ "1756": 5.02216,
+ "1757": 5.14157,
+ "1758": 4.87188,
+ "1759": 4.94434,
+ "1760": 4.83222,
+ "1761": 5.02427,
+ "1762": 4.81507,
+ "1763": 4.77391,
+ "1764": 4.93175,
+ "1765": 5.14727,
+ "1766": 5.33614,
+ "1767": 5.22331,
+ "1768": 4.94712,
+ "1769": 5.0043,
+ "1770": 4.98512,
+ "1771": 4.96473,
+ "1772": 4.98299,
+ "1773": 4.97266,
+ "1774": 4.87138,
+ "1775": 4.9493,
+ "1776": 4.9958,
+ "1777": 4.94665,
+ "1778": 4.99288,
+ "1779": 5.08212,
+ "1780": 4.83608,
+ "1781": 5.05478,
+ "1782": 4.99549,
+ "1783": 5.01236,
+ "1784": 4.93254,
+ "1785": 5.16842,
+ "1786": 4.80892,
+ "1787": 4.9699,
+ "1788": 4.82948,
+ "1789": 4.88554,
+ "1790": 4.80386,
+ "1791": 4.74542,
+ "1792": 4.87988,
+ "1793": 5.11081,
+ "1794": 4.98659,
+ "1795": 4.97147,
+ "1796": 5.00354,
+ "1797": 4.79101,
+ "1798": 4.77029,
+ "1799": 5.01913,
+ "1800": 4.91155,
+ "1801": 5.04891,
+ "1802": 4.82591,
+ "1803": 4.95313,
+ "1804": 4.88492,
+ "1805": 4.90634,
+ "1806": 4.88167,
+ "1807": 4.92894,
+ "1808": 4.92469,
+ "1809": 5.15028,
+ "1810": 5.09708,
+ "1811": 4.96325,
+ "1812": 4.8059,
+ "1813": 5.1023,
+ "1814": 4.7819,
+ "1815": 4.86518,
+ "1816": 5.05104,
+ "1817": 4.79238,
+ "1818": 4.80401,
+ "1819": 5.02672,
+ "1820": 4.68884,
+ "1821": 5.02319,
+ "1822": 4.66224,
+ "1823": 4.86936,
+ "1824": 4.7914,
+ "1825": 5.06607,
+ "1826": 4.81841,
+ "1827": 4.79544,
+ "1828": 4.9506,
+ "1829": 5.10848,
+ "1830": 4.9163,
+ "1831": 4.89965,
+ "1832": 4.83328,
+ "1833": 4.78854,
+ "1834": 4.94794,
+ "1835": 4.96175,
+ "1836": 4.91339,
+ "1837": 4.6762,
+ "1838": 4.80703,
+ "1839": 4.89949,
+ "1840": 4.91213,
+ "1841": 4.84083,
+ "1842": 4.9567,
+ "1843": 4.71182,
+ "1844": 4.6194,
+ "1845": 5.00584,
+ "1846": 4.75435,
+ "1847": 4.86491,
+ "1848": 4.9035,
+ "1849": 4.85124,
+ "1850": 4.87005,
+ "1851": 5.01617,
+ "1852": 4.97859,
+ "1853": 4.82821,
+ "1854": 4.86426,
+ "1855": 4.82455,
+ "1856": 4.75214,
+ "1857": 4.96641,
+ "1858": 4.96711,
+ "1859": 4.7484,
+ "1860": 4.86558,
+ "1861": 5.21257,
+ "1862": 4.61253,
+ "1863": 4.83567,
+ "1864": 4.74748,
+ "1865": 4.86472,
+ "1866": 4.78934,
+ "1867": 5.00307,
+ "1868": 4.72073,
+ "1869": 4.76301,
+ "1870": 4.93972,
+ "1871": 5.00163,
+ "1872": 4.68713,
+ "1873": 4.70038,
+ "1874": 4.85131,
+ "1875": 4.85367,
+ "1876": 4.74378,
+ "1877": 4.80696,
+ "1878": 4.8139,
+ "1879": 4.82462,
+ "1880": 4.89248,
+ "1881": 4.79379,
+ "1882": 4.79882,
+ "1883": 4.78556,
+ "1884": 4.97714,
+ "1885": 4.92363,
+ "1886": 4.82454,
+ "1887": 4.82091,
+ "1888": 4.97246,
+ "1889": 4.96553,
+ "1890": 4.71236,
+ "1891": 4.65764,
+ "1892": 4.85277,
+ "1893": 4.65022,
+ "1894": 4.90165,
+ "1895": 4.79,
+ "1896": 4.66068,
+ "1897": 4.79617,
+ "1898": 4.92161,
+ "1899": 4.77736,
+ "1900": 4.91325,
+ "1901": 4.84998,
+ "1902": 4.787,
+ "1903": 4.76372,
+ "1904": 4.65638,
+ "1905": 4.55077,
+ "1906": 4.81577,
+ "1907": 4.9106,
+ "1908": 5.03029,
+ "1909": 4.89294,
+ "1910": 4.7884,
+ "1911": 4.81269,
+ "1912": 4.653,
+ "1913": 4.95098,
+ "1914": 4.88806,
+ "1915": 4.86687,
+ "1916": 4.9302,
+ "1917": 4.85504,
+ "1918": 4.87427,
+ "1919": 4.99557,
+ "1920": 4.77001,
+ "1921": 4.88729,
+ "1922": 4.8196,
+ "1923": 4.75752,
+ "1924": 4.8297,
+ "1925": 5.05687,
+ "1926": 4.94229,
+ "1927": 4.93308,
+ "1928": 4.92739,
+ "1929": 4.93147,
+ "1930": 4.917,
+ "1931": 4.77692,
+ "1932": 4.86743,
+ "1933": 4.83532,
+ "1934": 4.84373,
+ "1935": 5.11279,
+ "1936": 4.88728,
+ "1937": 4.8824,
+ "1938": 4.80623,
+ "1939": 4.70831,
+ "1940": 4.83067,
+ "1941": 4.74224,
+ "1942": 4.87785,
+ "1943": 4.74082,
+ "1944": 4.7536,
+ "1945": 4.69017,
+ "1946": 4.91953,
+ "1947": 4.87613,
+ "1948": 4.60452,
+ "1949": 4.89888,
+ "1950": 4.79826,
+ "1951": 4.9677,
+ "1952": 4.73855,
+ "1953": 4.79852,
+ "1954": 4.7398,
+ "1955": 4.85209,
+ "1956": 4.88278,
+ "1957": 4.73599,
+ "1958": 4.70215,
+ "1959": 4.76471,
+ "1960": 4.76967,
+ "1961": 4.71471,
+ "1962": 4.83443,
+ "1963": 4.82459,
+ "1964": 4.85019,
+ "1965": 4.87867,
+ "1966": 4.79219,
+ "1967": 4.60013,
+ "1968": 4.83399,
+ "1969": 4.59632,
+ "1970": 4.58346,
+ "1971": 4.90585,
+ "1972": 4.89941,
+ "1973": 4.55559,
+ "1974": 4.8295,
+ "1975": 4.83261,
+ "1976": 4.71818,
+ "1977": 4.58171,
+ "1978": 5.00781,
+ "1979": 4.6663,
+ "1980": 4.74961,
+ "1981": 4.87741,
+ "1982": 4.72647,
+ "1983": 4.89363,
+ "1984": 4.64954,
+ "1985": 4.78941,
+ "1986": 4.70195,
+ "1987": 4.8185,
+ "1988": 4.89272,
+ "1989": 4.63799,
+ "1990": 4.79789,
+ "1991": 4.70399,
+ "1992": 4.80349,
+ "1993": 4.74121,
+ "1994": 4.85611,
+ "1995": 4.5595,
+ "1996": 4.65792,
+ "1997": 4.8133,
+ "1998": 4.68041,
+ "1999": 4.73244,
+ "2000": 4.6301
+ }
+ },
+ "num-zeros": {
+ "start_step": 1,
+ "end_step": 2000,
+ "step_interval": 1,
+ "values": {
+ "1": 26.0,
+ "2": 32.0,
+ "3": 38.0,
+ "4": 33.0,
+ "5": 32.0,
+ "6": 30.0,
+ "7": 33.0,
+ "8": 34.0,
+ "9": 40.0,
+ "10": 31.0,
+ "11": 26.0,
+ "12": 33.0,
+ "13": 28.0,
+ "14": 29.0,
+ "15": 28.0,
+ "16": 27.0,
+ "17": 32.0,
+ "18": 28.0,
+ "19": 31.0,
+ "20": 39.0,
+ "21": 22.0,
+ "22": 29.0,
+ "23": 39.0,
+ "24": 35.0,
+ "25": 31.0,
+ "26": 40.0,
+ "27": 39.0,
+ "28": 42.0,
+ "29": 53.0,
+ "30": 51.0,
+ "31": 48.0,
+ "32": 51.0,
+ "33": 38.0,
+ "34": 48.0,
+ "35": 47.0,
+ "36": 49.0,
+ "37": 42.0,
+ "38": 43.0,
+ "39": 52.0,
+ "40": 55.0,
+ "41": 39.0,
+ "42": 54.0,
+ "43": 57.0,
+ "44": 53.0,
+ "45": 46.0,
+ "46": 61.0,
+ "47": 52.0,
+ "48": 54.0,
+ "49": 64.0,
+ "50": 64.0,
+ "51": 42.0,
+ "52": 55.0,
+ "53": 48.0,
+ "54": 71.0,
+ "55": 56.0,
+ "56": 74.0,
+ "57": 70.0,
+ "58": 57.0,
+ "59": 53.0,
+ "60": 67.0,
+ "61": 63.0,
+ "62": 59.0,
+ "63": 66.0,
+ "64": 70.0,
+ "65": 59.0,
+ "66": 74.0,
+ "67": 81.0,
+ "68": 74.0,
+ "69": 60.0,
+ "70": 60.0,
+ "71": 66.0,
+ "72": 75.0,
+ "73": 67.0,
+ "74": 63.0,
+ "75": 60.0,
+ "76": 60.0,
+ "77": 78.0,
+ "78": 78.0,
+ "79": 58.0,
+ "80": 63.0,
+ "81": 63.0,
+ "82": 50.0,
+ "83": 63.0,
+ "84": 72.0,
+ "85": 69.0,
+ "86": 80.0,
+ "87": 70.0,
+ "88": 68.0,
+ "89": 69.0,
+ "90": 63.0,
+ "91": 58.0,
+ "92": 87.0,
+ "93": 65.0,
+ "94": 50.0,
+ "95": 67.0,
+ "96": 71.0,
+ "97": 70.0,
+ "98": 81.0,
+ "99": 66.0,
+ "100": 76.0,
+ "101": 67.0,
+ "102": 44.0,
+ "103": 60.0,
+ "104": 68.0,
+ "105": 84.0,
+ "106": 61.0,
+ "107": 76.0,
+ "108": 68.0,
+ "109": 76.0,
+ "110": 74.0,
+ "111": 75.0,
+ "112": 78.0,
+ "113": 58.0,
+ "114": 66.0,
+ "115": 71.0,
+ "116": 63.0,
+ "117": 74.0,
+ "118": 52.0,
+ "119": 74.0,
+ "120": 52.0,
+ "121": 76.0,
+ "122": 66.0,
+ "123": 81.0,
+ "124": 76.0,
+ "125": 87.0,
+ "126": 49.0,
+ "127": 56.0,
+ "128": 78.0,
+ "129": 53.0,
+ "130": 76.0,
+ "131": 86.0,
+ "132": 61.0,
+ "133": 72.0,
+ "134": 62.0,
+ "135": 59.0,
+ "136": 60.0,
+ "137": 57.0,
+ "138": 81.0,
+ "139": 74.0,
+ "140": 59.0,
+ "141": 50.0,
+ "142": 64.0,
+ "143": 54.0,
+ "144": 49.0,
+ "145": 57.0,
+ "146": 51.0,
+ "147": 49.0,
+ "148": 69.0,
+ "149": 49.0,
+ "150": 66.0,
+ "151": 57.0,
+ "152": 51.0,
+ "153": 61.0,
+ "154": 58.0,
+ "155": 68.0,
+ "156": 68.0,
+ "157": 51.0,
+ "158": 68.0,
+ "159": 60.0,
+ "160": 64.0,
+ "161": 66.0,
+ "162": 75.0,
+ "163": 40.0,
+ "164": 84.0,
+ "165": 50.0,
+ "166": 68.0,
+ "167": 54.0,
+ "168": 58.0,
+ "169": 65.0,
+ "170": 71.0,
+ "171": 54.0,
+ "172": 64.0,
+ "173": 81.0,
+ "174": 55.0,
+ "175": 63.0,
+ "176": 69.0,
+ "177": 80.0,
+ "178": 68.0,
+ "179": 69.0,
+ "180": 64.0,
+ "181": 41.0,
+ "182": 63.0,
+ "183": 66.0,
+ "184": 67.0,
+ "185": 77.0,
+ "186": 77.0,
+ "187": 61.0,
+ "188": 62.0,
+ "189": 50.0,
+ "190": 57.0,
+ "191": 60.0,
+ "192": 67.0,
+ "193": 70.0,
+ "194": 72.0,
+ "195": 60.0,
+ "196": 81.0,
+ "197": 56.0,
+ "198": 47.0,
+ "199": 50.0,
+ "200": 86.0,
+ "201": 52.0,
+ "202": 64.0,
+ "203": 58.0,
+ "204": 63.0,
+ "205": 40.0,
+ "206": 72.0,
+ "207": 50.0,
+ "208": 42.0,
+ "209": 69.0,
+ "210": 68.0,
+ "211": 56.0,
+ "212": 64.0,
+ "213": 60.0,
+ "214": 62.0,
+ "215": 66.0,
+ "216": 58.0,
+ "217": 59.0,
+ "218": 70.0,
+ "219": 80.0,
+ "220": 81.0,
+ "221": 51.0,
+ "222": 57.0,
+ "223": 67.0,
+ "224": 53.0,
+ "225": 61.0,
+ "226": 68.0,
+ "227": 76.0,
+ "228": 59.0,
+ "229": 44.0,
+ "230": 50.0,
+ "231": 58.0,
+ "232": 65.0,
+ "233": 90.0,
+ "234": 60.0,
+ "235": 98.0,
+ "236": 49.0,
+ "237": 92.0,
+ "238": 71.0,
+ "239": 68.0,
+ "240": 79.0,
+ "241": 67.0,
+ "242": 75.0,
+ "243": 66.0,
+ "244": 59.0,
+ "245": 81.0,
+ "246": 80.0,
+ "247": 88.0,
+ "248": 81.0,
+ "249": 79.0,
+ "250": 80.0,
+ "251": 74.0,
+ "252": 72.0,
+ "253": 57.0,
+ "254": 67.0,
+ "255": 79.0,
+ "256": 86.0,
+ "257": 66.0,
+ "258": 94.0,
+ "259": 69.0,
+ "260": 70.0,
+ "261": 64.0,
+ "262": 77.0,
+ "263": 74.0,
+ "264": 70.0,
+ "265": 68.0,
+ "266": 67.0,
+ "267": 66.0,
+ "268": 59.0,
+ "269": 73.0,
+ "270": 85.0,
+ "271": 67.0,
+ "272": 81.0,
+ "273": 71.0,
+ "274": 69.0,
+ "275": 72.0,
+ "276": 72.0,
+ "277": 82.0,
+ "278": 61.0,
+ "279": 94.0,
+ "280": 56.0,
+ "281": 55.0,
+ "282": 73.0,
+ "283": 90.0,
+ "284": 85.0,
+ "285": 49.0,
+ "286": 50.0,
+ "287": 90.0,
+ "288": 71.0,
+ "289": 85.0,
+ "290": 75.0,
+ "291": 88.0,
+ "292": 88.0,
+ "293": 91.0,
+ "294": 84.0,
+ "295": 85.0,
+ "296": 102.0,
+ "297": 70.0,
+ "298": 65.0,
+ "299": 80.0,
+ "300": 80.0,
+ "301": 91.0,
+ "302": 94.0,
+ "303": 71.0,
+ "304": 74.0,
+ "305": 59.0,
+ "306": 72.0,
+ "307": 73.0,
+ "308": 91.0,
+ "309": 88.0,
+ "310": 82.0,
+ "311": 84.0,
+ "312": 73.0,
+ "313": 97.0,
+ "314": 74.0,
+ "315": 69.0,
+ "316": 96.0,
+ "317": 61.0,
+ "318": 99.0,
+ "319": 67.0,
+ "320": 77.0,
+ "321": 86.0,
+ "322": 70.0,
+ "323": 86.0,
+ "324": 96.0,
+ "325": 74.0,
+ "326": 97.0,
+ "327": 73.0,
+ "328": 99.0,
+ "329": 93.0,
+ "330": 96.0,
+ "331": 81.0,
+ "332": 79.0,
+ "333": 97.0,
+ "334": 81.0,
+ "335": 84.0,
+ "336": 81.0,
+ "337": 99.0,
+ "338": 89.0,
+ "339": 93.0,
+ "340": 101.0,
+ "341": 93.0,
+ "342": 57.0,
+ "343": 81.0,
+ "344": 105.0,
+ "345": 88.0,
+ "346": 85.0,
+ "347": 91.0,
+ "348": 82.0,
+ "349": 78.0,
+ "350": 101.0,
+ "351": 105.0,
+ "352": 76.0,
+ "353": 112.0,
+ "354": 72.0,
+ "355": 79.0,
+ "356": 104.0,
+ "357": 86.0,
+ "358": 77.0,
+ "359": 99.0,
+ "360": 102.0,
+ "361": 64.0,
+ "362": 123.0,
+ "363": 96.0,
+ "364": 95.0,
+ "365": 85.0,
+ "366": 82.0,
+ "367": 84.0,
+ "368": 83.0,
+ "369": 77.0,
+ "370": 118.0,
+ "371": 76.0,
+ "372": 77.0,
+ "373": 96.0,
+ "374": 68.0,
+ "375": 92.0,
+ "376": 84.0,
+ "377": 98.0,
+ "378": 99.0,
+ "379": 108.0,
+ "380": 96.0,
+ "381": 92.0,
+ "382": 75.0,
+ "383": 89.0,
+ "384": 100.0,
+ "385": 73.0,
+ "386": 85.0,
+ "387": 73.0,
+ "388": 93.0,
+ "389": 88.0,
+ "390": 90.0,
+ "391": 115.0,
+ "392": 88.0,
+ "393": 99.0,
+ "394": 104.0,
+ "395": 125.0,
+ "396": 80.0,
+ "397": 78.0,
+ "398": 67.0,
+ "399": 104.0,
+ "400": 96.0,
+ "401": 105.0,
+ "402": 88.0,
+ "403": 97.0,
+ "404": 101.0,
+ "405": 85.0,
+ "406": 114.0,
+ "407": 76.0,
+ "408": 98.0,
+ "409": 84.0,
+ "410": 102.0,
+ "411": 81.0,
+ "412": 56.0,
+ "413": 68.0,
+ "414": 90.0,
+ "415": 95.0,
+ "416": 93.0,
+ "417": 90.0,
+ "418": 60.0,
+ "419": 86.0,
+ "420": 76.0,
+ "421": 110.0,
+ "422": 89.0,
+ "423": 78.0,
+ "424": 82.0,
+ "425": 94.0,
+ "426": 80.0,
+ "427": 96.0,
+ "428": 86.0,
+ "429": 92.0,
+ "430": 84.0,
+ "431": 87.0,
+ "432": 80.0,
+ "433": 81.0,
+ "434": 93.0,
+ "435": 83.0,
+ "436": 82.0,
+ "437": 91.0,
+ "438": 62.0,
+ "439": 72.0,
+ "440": 79.0,
+ "441": 87.0,
+ "442": 106.0,
+ "443": 106.0,
+ "444": 58.0,
+ "445": 93.0,
+ "446": 89.0,
+ "447": 97.0,
+ "448": 79.0,
+ "449": 90.0,
+ "450": 83.0,
+ "451": 63.0,
+ "452": 70.0,
+ "453": 63.0,
+ "454": 80.0,
+ "455": 114.0,
+ "456": 98.0,
+ "457": 101.0,
+ "458": 70.0,
+ "459": 69.0,
+ "460": 65.0,
+ "461": 115.0,
+ "462": 63.0,
+ "463": 73.0,
+ "464": 69.0,
+ "465": 95.0,
+ "466": 76.0,
+ "467": 77.0,
+ "468": 90.0,
+ "469": 65.0,
+ "470": 91.0,
+ "471": 76.0,
+ "472": 60.0,
+ "473": 94.0,
+ "474": 69.0,
+ "475": 90.0,
+ "476": 66.0,
+ "477": 75.0,
+ "478": 78.0,
+ "479": 63.0,
+ "480": 73.0,
+ "481": 80.0,
+ "482": 77.0,
+ "483": 78.0,
+ "484": 84.0,
+ "485": 70.0,
+ "486": 84.0,
+ "487": 69.0,
+ "488": 88.0,
+ "489": 77.0,
+ "490": 59.0,
+ "491": 83.0,
+ "492": 57.0,
+ "493": 83.0,
+ "494": 69.0,
+ "495": 50.0,
+ "496": 56.0,
+ "497": 97.0,
+ "498": 77.0,
+ "499": 75.0,
+ "500": 60.0,
+ "501": 64.0,
+ "502": 64.0,
+ "503": 71.0,
+ "504": 77.0,
+ "505": 68.0,
+ "506": 65.0,
+ "507": 80.0,
+ "508": 42.0,
+ "509": 63.0,
+ "510": 77.0,
+ "511": 81.0,
+ "512": 57.0,
+ "513": 61.0,
+ "514": 60.0,
+ "515": 71.0,
+ "516": 61.0,
+ "517": 85.0,
+ "518": 43.0,
+ "519": 72.0,
+ "520": 82.0,
+ "521": 50.0,
+ "522": 58.0,
+ "523": 74.0,
+ "524": 70.0,
+ "525": 82.0,
+ "526": 60.0,
+ "527": 71.0,
+ "528": 63.0,
+ "529": 66.0,
+ "530": 67.0,
+ "531": 69.0,
+ "532": 72.0,
+ "533": 81.0,
+ "534": 62.0,
+ "535": 66.0,
+ "536": 61.0,
+ "537": 60.0,
+ "538": 55.0,
+ "539": 62.0,
+ "540": 63.0,
+ "541": 61.0,
+ "542": 61.0,
+ "543": 55.0,
+ "544": 64.0,
+ "545": 73.0,
+ "546": 77.0,
+ "547": 69.0,
+ "548": 75.0,
+ "549": 61.0,
+ "550": 61.0,
+ "551": 63.0,
+ "552": 71.0,
+ "553": 78.0,
+ "554": 67.0,
+ "555": 65.0,
+ "556": 74.0,
+ "557": 61.0,
+ "558": 62.0,
+ "559": 62.0,
+ "560": 71.0,
+ "561": 56.0,
+ "562": 65.0,
+ "563": 77.0,
+ "564": 67.0,
+ "565": 55.0,
+ "566": 58.0,
+ "567": 42.0,
+ "568": 70.0,
+ "569": 56.0,
+ "570": 60.0,
+ "571": 58.0,
+ "572": 41.0,
+ "573": 71.0,
+ "574": 69.0,
+ "575": 85.0,
+ "576": 44.0,
+ "577": 50.0,
+ "578": 69.0,
+ "579": 62.0,
+ "580": 67.0,
+ "581": 59.0,
+ "582": 58.0,
+ "583": 55.0,
+ "584": 47.0,
+ "585": 60.0,
+ "586": 41.0,
+ "587": 47.0,
+ "588": 53.0,
+ "589": 55.0,
+ "590": 46.0,
+ "591": 69.0,
+ "592": 50.0,
+ "593": 52.0,
+ "594": 56.0,
+ "595": 47.0,
+ "596": 44.0,
+ "597": 33.0,
+ "598": 61.0,
+ "599": 50.0,
+ "600": 88.0,
+ "601": 55.0,
+ "602": 64.0,
+ "603": 60.0,
+ "604": 57.0,
+ "605": 57.0,
+ "606": 45.0,
+ "607": 54.0,
+ "608": 45.0,
+ "609": 40.0,
+ "610": 45.0,
+ "611": 53.0,
+ "612": 52.0,
+ "613": 73.0,
+ "614": 53.0,
+ "615": 52.0,
+ "616": 64.0,
+ "617": 44.0,
+ "618": 59.0,
+ "619": 50.0,
+ "620": 72.0,
+ "621": 50.0,
+ "622": 58.0,
+ "623": 57.0,
+ "624": 56.0,
+ "625": 56.0,
+ "626": 71.0,
+ "627": 50.0,
+ "628": 49.0,
+ "629": 50.0,
+ "630": 50.0,
+ "631": 40.0,
+ "632": 45.0,
+ "633": 42.0,
+ "634": 38.0,
+ "635": 51.0,
+ "636": 36.0,
+ "637": 55.0,
+ "638": 45.0,
+ "639": 63.0,
+ "640": 52.0,
+ "641": 51.0,
+ "642": 52.0,
+ "643": 49.0,
+ "644": 51.0,
+ "645": 57.0,
+ "646": 57.0,
+ "647": 69.0,
+ "648": 60.0,
+ "649": 49.0,
+ "650": 49.0,
+ "651": 66.0,
+ "652": 49.0,
+ "653": 59.0,
+ "654": 42.0,
+ "655": 42.0,
+ "656": 46.0,
+ "657": 49.0,
+ "658": 50.0,
+ "659": 44.0,
+ "660": 53.0,
+ "661": 46.0,
+ "662": 60.0,
+ "663": 43.0,
+ "664": 61.0,
+ "665": 37.0,
+ "666": 30.0,
+ "667": 42.0,
+ "668": 41.0,
+ "669": 44.0,
+ "670": 44.0,
+ "671": 59.0,
+ "672": 53.0,
+ "673": 47.0,
+ "674": 42.0,
+ "675": 54.0,
+ "676": 43.0,
+ "677": 68.0,
+ "678": 41.0,
+ "679": 38.0,
+ "680": 46.0,
+ "681": 50.0,
+ "682": 33.0,
+ "683": 38.0,
+ "684": 52.0,
+ "685": 40.0,
+ "686": 43.0,
+ "687": 61.0,
+ "688": 57.0,
+ "689": 51.0,
+ "690": 35.0,
+ "691": 45.0,
+ "692": 55.0,
+ "693": 36.0,
+ "694": 50.0,
+ "695": 50.0,
+ "696": 51.0,
+ "697": 41.0,
+ "698": 37.0,
+ "699": 47.0,
+ "700": 42.0,
+ "701": 37.0,
+ "702": 33.0,
+ "703": 39.0,
+ "704": 43.0,
+ "705": 45.0,
+ "706": 32.0,
+ "707": 38.0,
+ "708": 38.0,
+ "709": 46.0,
+ "710": 35.0,
+ "711": 48.0,
+ "712": 35.0,
+ "713": 48.0,
+ "714": 37.0,
+ "715": 48.0,
+ "716": 36.0,
+ "717": 34.0,
+ "718": 26.0,
+ "719": 36.0,
+ "720": 34.0,
+ "721": 36.0,
+ "722": 35.0,
+ "723": 29.0,
+ "724": 47.0,
+ "725": 32.0,
+ "726": 39.0,
+ "727": 40.0,
+ "728": 39.0,
+ "729": 47.0,
+ "730": 36.0,
+ "731": 48.0,
+ "732": 43.0,
+ "733": 39.0,
+ "734": 51.0,
+ "735": 40.0,
+ "736": 49.0,
+ "737": 44.0,
+ "738": 27.0,
+ "739": 46.0,
+ "740": 38.0,
+ "741": 38.0,
+ "742": 45.0,
+ "743": 44.0,
+ "744": 52.0,
+ "745": 48.0,
+ "746": 50.0,
+ "747": 53.0,
+ "748": 52.0,
+ "749": 48.0,
+ "750": 46.0,
+ "751": 40.0,
+ "752": 50.0,
+ "753": 44.0,
+ "754": 43.0,
+ "755": 48.0,
+ "756": 38.0,
+ "757": 45.0,
+ "758": 40.0,
+ "759": 56.0,
+ "760": 46.0,
+ "761": 44.0,
+ "762": 48.0,
+ "763": 54.0,
+ "764": 49.0,
+ "765": 42.0,
+ "766": 57.0,
+ "767": 45.0,
+ "768": 51.0,
+ "769": 60.0,
+ "770": 51.0,
+ "771": 31.0,
+ "772": 41.0,
+ "773": 60.0,
+ "774": 37.0,
+ "775": 43.0,
+ "776": 37.0,
+ "777": 34.0,
+ "778": 42.0,
+ "779": 37.0,
+ "780": 34.0,
+ "781": 41.0,
+ "782": 25.0,
+ "783": 30.0,
+ "784": 39.0,
+ "785": 34.0,
+ "786": 38.0,
+ "787": 47.0,
+ "788": 41.0,
+ "789": 50.0,
+ "790": 44.0,
+ "791": 34.0,
+ "792": 38.0,
+ "793": 53.0,
+ "794": 45.0,
+ "795": 52.0,
+ "796": 39.0,
+ "797": 41.0,
+ "798": 39.0,
+ "799": 44.0,
+ "800": 46.0,
+ "801": 44.0,
+ "802": 40.0,
+ "803": 47.0,
+ "804": 34.0,
+ "805": 45.0,
+ "806": 43.0,
+ "807": 46.0,
+ "808": 36.0,
+ "809": 35.0,
+ "810": 35.0,
+ "811": 44.0,
+ "812": 47.0,
+ "813": 41.0,
+ "814": 36.0,
+ "815": 41.0,
+ "816": 52.0,
+ "817": 43.0,
+ "818": 35.0,
+ "819": 52.0,
+ "820": 40.0,
+ "821": 29.0,
+ "822": 34.0,
+ "823": 44.0,
+ "824": 47.0,
+ "825": 36.0,
+ "826": 40.0,
+ "827": 29.0,
+ "828": 35.0,
+ "829": 32.0,
+ "830": 30.0,
+ "831": 36.0,
+ "832": 34.0,
+ "833": 39.0,
+ "834": 50.0,
+ "835": 38.0,
+ "836": 37.0,
+ "837": 50.0,
+ "838": 45.0,
+ "839": 52.0,
+ "840": 37.0,
+ "841": 35.0,
+ "842": 30.0,
+ "843": 50.0,
+ "844": 23.0,
+ "845": 45.0,
+ "846": 25.0,
+ "847": 32.0,
+ "848": 25.0,
+ "849": 34.0,
+ "850": 39.0,
+ "851": 46.0,
+ "852": 41.0,
+ "853": 43.0,
+ "854": 45.0,
+ "855": 27.0,
+ "856": 47.0,
+ "857": 47.0,
+ "858": 46.0,
+ "859": 35.0,
+ "860": 45.0,
+ "861": 30.0,
+ "862": 39.0,
+ "863": 21.0,
+ "864": 26.0,
+ "865": 46.0,
+ "866": 44.0,
+ "867": 48.0,
+ "868": 27.0,
+ "869": 42.0,
+ "870": 45.0,
+ "871": 33.0,
+ "872": 49.0,
+ "873": 32.0,
+ "874": 56.0,
+ "875": 38.0,
+ "876": 41.0,
+ "877": 40.0,
+ "878": 37.0,
+ "879": 22.0,
+ "880": 39.0,
+ "881": 40.0,
+ "882": 49.0,
+ "883": 39.0,
+ "884": 35.0,
+ "885": 32.0,
+ "886": 45.0,
+ "887": 41.0,
+ "888": 34.0,
+ "889": 35.0,
+ "890": 37.0,
+ "891": 41.0,
+ "892": 42.0,
+ "893": 42.0,
+ "894": 34.0,
+ "895": 38.0,
+ "896": 37.0,
+ "897": 41.0,
+ "898": 33.0,
+ "899": 35.0,
+ "900": 39.0,
+ "901": 37.0,
+ "902": 39.0,
+ "903": 42.0,
+ "904": 38.0,
+ "905": 32.0,
+ "906": 34.0,
+ "907": 38.0,
+ "908": 39.0,
+ "909": 52.0,
+ "910": 34.0,
+ "911": 26.0,
+ "912": 46.0,
+ "913": 40.0,
+ "914": 48.0,
+ "915": 25.0,
+ "916": 49.0,
+ "917": 36.0,
+ "918": 31.0,
+ "919": 26.0,
+ "920": 40.0,
+ "921": 34.0,
+ "922": 38.0,
+ "923": 41.0,
+ "924": 24.0,
+ "925": 27.0,
+ "926": 43.0,
+ "927": 31.0,
+ "928": 40.0,
+ "929": 32.0,
+ "930": 42.0,
+ "931": 33.0,
+ "932": 34.0,
+ "933": 38.0,
+ "934": 41.0,
+ "935": 26.0,
+ "936": 44.0,
+ "937": 36.0,
+ "938": 37.0,
+ "939": 28.0,
+ "940": 33.0,
+ "941": 34.0,
+ "942": 31.0,
+ "943": 26.0,
+ "944": 37.0,
+ "945": 29.0,
+ "946": 31.0,
+ "947": 34.0,
+ "948": 41.0,
+ "949": 31.0,
+ "950": 35.0,
+ "951": 31.0,
+ "952": 38.0,
+ "953": 47.0,
+ "954": 43.0,
+ "955": 46.0,
+ "956": 35.0,
+ "957": 40.0,
+ "958": 37.0,
+ "959": 52.0,
+ "960": 35.0,
+ "961": 38.0,
+ "962": 41.0,
+ "963": 45.0,
+ "964": 43.0,
+ "965": 51.0,
+ "966": 38.0,
+ "967": 31.0,
+ "968": 32.0,
+ "969": 35.0,
+ "970": 48.0,
+ "971": 38.0,
+ "972": 43.0,
+ "973": 38.0,
+ "974": 40.0,
+ "975": 43.0,
+ "976": 29.0,
+ "977": 44.0,
+ "978": 31.0,
+ "979": 43.0,
+ "980": 39.0,
+ "981": 33.0,
+ "982": 30.0,
+ "983": 54.0,
+ "984": 43.0,
+ "985": 48.0,
+ "986": 40.0,
+ "987": 30.0,
+ "988": 38.0,
+ "989": 38.0,
+ "990": 42.0,
+ "991": 36.0,
+ "992": 48.0,
+ "993": 47.0,
+ "994": 50.0,
+ "995": 35.0,
+ "996": 29.0,
+ "997": 51.0,
+ "998": 42.0,
+ "999": 35.0,
+ "1000": 28.0,
+ "1001": 23.0,
+ "1002": 35.0,
+ "1003": 39.0,
+ "1004": 46.0,
+ "1005": 42.0,
+ "1006": 27.0,
+ "1007": 44.0,
+ "1008": 32.0,
+ "1009": 34.0,
+ "1010": 29.0,
+ "1011": 31.0,
+ "1012": 28.0,
+ "1013": 37.0,
+ "1014": 29.0,
+ "1015": 39.0,
+ "1016": 31.0,
+ "1017": 37.0,
+ "1018": 46.0,
+ "1019": 26.0,
+ "1020": 34.0,
+ "1021": 30.0,
+ "1022": 46.0,
+ "1023": 38.0,
+ "1024": 49.0,
+ "1025": 41.0,
+ "1026": 55.0,
+ "1027": 37.0,
+ "1028": 29.0,
+ "1029": 38.0,
+ "1030": 35.0,
+ "1031": 41.0,
+ "1032": 42.0,
+ "1033": 27.0,
+ "1034": 29.0,
+ "1035": 32.0,
+ "1036": 25.0,
+ "1037": 34.0,
+ "1038": 32.0,
+ "1039": 31.0,
+ "1040": 30.0,
+ "1041": 24.0,
+ "1042": 20.0,
+ "1043": 26.0,
+ "1044": 44.0,
+ "1045": 37.0,
+ "1046": 34.0,
+ "1047": 27.0,
+ "1048": 36.0,
+ "1049": 42.0,
+ "1050": 37.0,
+ "1051": 40.0,
+ "1052": 40.0,
+ "1053": 32.0,
+ "1054": 37.0,
+ "1055": 31.0,
+ "1056": 36.0,
+ "1057": 37.0,
+ "1058": 37.0,
+ "1059": 35.0,
+ "1060": 32.0,
+ "1061": 37.0,
+ "1062": 45.0,
+ "1063": 38.0,
+ "1064": 42.0,
+ "1065": 35.0,
+ "1066": 36.0,
+ "1067": 29.0,
+ "1068": 30.0,
+ "1069": 30.0,
+ "1070": 39.0,
+ "1071": 33.0,
+ "1072": 36.0,
+ "1073": 41.0,
+ "1074": 47.0,
+ "1075": 36.0,
+ "1076": 39.0,
+ "1077": 45.0,
+ "1078": 32.0,
+ "1079": 46.0,
+ "1080": 43.0,
+ "1081": 40.0,
+ "1082": 42.0,
+ "1083": 42.0,
+ "1084": 42.0,
+ "1085": 38.0,
+ "1086": 42.0,
+ "1087": 36.0,
+ "1088": 31.0,
+ "1089": 42.0,
+ "1090": 28.0,
+ "1091": 36.0,
+ "1092": 35.0,
+ "1093": 36.0,
+ "1094": 41.0,
+ "1095": 37.0,
+ "1096": 48.0,
+ "1097": 33.0,
+ "1098": 24.0,
+ "1099": 43.0,
+ "1100": 41.0,
+ "1101": 38.0,
+ "1102": 39.0,
+ "1103": 29.0,
+ "1104": 33.0,
+ "1105": 38.0,
+ "1106": 37.0,
+ "1107": 30.0,
+ "1108": 41.0,
+ "1109": 41.0,
+ "1110": 42.0,
+ "1111": 43.0,
+ "1112": 25.0,
+ "1113": 40.0,
+ "1114": 32.0,
+ "1115": 34.0,
+ "1116": 45.0,
+ "1117": 40.0,
+ "1118": 39.0,
+ "1119": 31.0,
+ "1120": 28.0,
+ "1121": 28.0,
+ "1122": 28.0,
+ "1123": 43.0,
+ "1124": 34.0,
+ "1125": 26.0,
+ "1126": 33.0,
+ "1127": 31.0,
+ "1128": 33.0,
+ "1129": 43.0,
+ "1130": 43.0,
+ "1131": 40.0,
+ "1132": 42.0,
+ "1133": 34.0,
+ "1134": 32.0,
+ "1135": 29.0,
+ "1136": 36.0,
+ "1137": 42.0,
+ "1138": 34.0,
+ "1139": 31.0,
+ "1140": 38.0,
+ "1141": 37.0,
+ "1142": 38.0,
+ "1143": 44.0,
+ "1144": 40.0,
+ "1145": 39.0,
+ "1146": 42.0,
+ "1147": 35.0,
+ "1148": 29.0,
+ "1149": 40.0,
+ "1150": 34.0,
+ "1151": 27.0,
+ "1152": 22.0,
+ "1153": 36.0,
+ "1154": 31.0,
+ "1155": 41.0,
+ "1156": 26.0,
+ "1157": 33.0,
+ "1158": 35.0,
+ "1159": 36.0,
+ "1160": 41.0,
+ "1161": 40.0,
+ "1162": 48.0,
+ "1163": 37.0,
+ "1164": 43.0,
+ "1165": 34.0,
+ "1166": 30.0,
+ "1167": 34.0,
+ "1168": 31.0,
+ "1169": 41.0,
+ "1170": 27.0,
+ "1171": 40.0,
+ "1172": 34.0,
+ "1173": 23.0,
+ "1174": 40.0,
+ "1175": 30.0,
+ "1176": 50.0,
+ "1177": 39.0,
+ "1178": 33.0,
+ "1179": 42.0,
+ "1180": 31.0,
+ "1181": 30.0,
+ "1182": 38.0,
+ "1183": 37.0,
+ "1184": 35.0,
+ "1185": 31.0,
+ "1186": 29.0,
+ "1187": 39.0,
+ "1188": 34.0,
+ "1189": 48.0,
+ "1190": 32.0,
+ "1191": 41.0,
+ "1192": 45.0,
+ "1193": 28.0,
+ "1194": 46.0,
+ "1195": 34.0,
+ "1196": 38.0,
+ "1197": 51.0,
+ "1198": 36.0,
+ "1199": 40.0,
+ "1200": 29.0,
+ "1201": 37.0,
+ "1202": 32.0,
+ "1203": 35.0,
+ "1204": 37.0,
+ "1205": 56.0,
+ "1206": 40.0,
+ "1207": 36.0,
+ "1208": 41.0,
+ "1209": 31.0,
+ "1210": 39.0,
+ "1211": 46.0,
+ "1212": 45.0,
+ "1213": 57.0,
+ "1214": 31.0,
+ "1215": 33.0,
+ "1216": 31.0,
+ "1217": 34.0,
+ "1218": 42.0,
+ "1219": 45.0,
+ "1220": 37.0,
+ "1221": 44.0,
+ "1222": 32.0,
+ "1223": 35.0,
+ "1224": 34.0,
+ "1225": 45.0,
+ "1226": 28.0,
+ "1227": 34.0,
+ "1228": 27.0,
+ "1229": 23.0,
+ "1230": 25.0,
+ "1231": 14.0,
+ "1232": 36.0,
+ "1233": 39.0,
+ "1234": 37.0,
+ "1235": 32.0,
+ "1236": 41.0,
+ "1237": 30.0,
+ "1238": 36.0,
+ "1239": 37.0,
+ "1240": 48.0,
+ "1241": 31.0,
+ "1242": 34.0,
+ "1243": 35.0,
+ "1244": 29.0,
+ "1245": 28.0,
+ "1246": 36.0,
+ "1247": 31.0,
+ "1248": 38.0,
+ "1249": 27.0,
+ "1250": 40.0,
+ "1251": 26.0,
+ "1252": 42.0,
+ "1253": 32.0,
+ "1254": 39.0,
+ "1255": 46.0,
+ "1256": 41.0,
+ "1257": 30.0,
+ "1258": 44.0,
+ "1259": 32.0,
+ "1260": 25.0,
+ "1261": 42.0,
+ "1262": 36.0,
+ "1263": 34.0,
+ "1264": 32.0,
+ "1265": 35.0,
+ "1266": 34.0,
+ "1267": 38.0,
+ "1268": 43.0,
+ "1269": 30.0,
+ "1270": 28.0,
+ "1271": 42.0,
+ "1272": 32.0,
+ "1273": 40.0,
+ "1274": 44.0,
+ "1275": 38.0,
+ "1276": 31.0,
+ "1277": 54.0,
+ "1278": 46.0,
+ "1279": 44.0,
+ "1280": 34.0,
+ "1281": 26.0,
+ "1282": 37.0,
+ "1283": 32.0,
+ "1284": 43.0,
+ "1285": 43.0,
+ "1286": 36.0,
+ "1287": 46.0,
+ "1288": 33.0,
+ "1289": 43.0,
+ "1290": 37.0,
+ "1291": 42.0,
+ "1292": 38.0,
+ "1293": 43.0,
+ "1294": 30.0,
+ "1295": 34.0,
+ "1296": 31.0,
+ "1297": 26.0,
+ "1298": 38.0,
+ "1299": 40.0,
+ "1300": 32.0,
+ "1301": 43.0,
+ "1302": 35.0,
+ "1303": 35.0,
+ "1304": 41.0,
+ "1305": 30.0,
+ "1306": 28.0,
+ "1307": 34.0,
+ "1308": 32.0,
+ "1309": 36.0,
+ "1310": 29.0,
+ "1311": 43.0,
+ "1312": 32.0,
+ "1313": 37.0,
+ "1314": 35.0,
+ "1315": 33.0,
+ "1316": 37.0,
+ "1317": 33.0,
+ "1318": 41.0,
+ "1319": 28.0,
+ "1320": 42.0,
+ "1321": 30.0,
+ "1322": 21.0,
+ "1323": 28.0,
+ "1324": 40.0,
+ "1325": 36.0,
+ "1326": 43.0,
+ "1327": 32.0,
+ "1328": 35.0,
+ "1329": 33.0,
+ "1330": 27.0,
+ "1331": 30.0,
+ "1332": 36.0,
+ "1333": 45.0,
+ "1334": 32.0,
+ "1335": 41.0,
+ "1336": 38.0,
+ "1337": 37.0,
+ "1338": 38.0,
+ "1339": 27.0,
+ "1340": 33.0,
+ "1341": 47.0,
+ "1342": 24.0,
+ "1343": 27.0,
+ "1344": 34.0,
+ "1345": 34.0,
+ "1346": 21.0,
+ "1347": 33.0,
+ "1348": 33.0,
+ "1349": 42.0,
+ "1350": 30.0,
+ "1351": 39.0,
+ "1352": 26.0,
+ "1353": 36.0,
+ "1354": 40.0,
+ "1355": 31.0,
+ "1356": 46.0,
+ "1357": 46.0,
+ "1358": 29.0,
+ "1359": 29.0,
+ "1360": 30.0,
+ "1361": 35.0,
+ "1362": 40.0,
+ "1363": 33.0,
+ "1364": 36.0,
+ "1365": 34.0,
+ "1366": 47.0,
+ "1367": 31.0,
+ "1368": 37.0,
+ "1369": 28.0,
+ "1370": 41.0,
+ "1371": 30.0,
+ "1372": 42.0,
+ "1373": 44.0,
+ "1374": 34.0,
+ "1375": 22.0,
+ "1376": 47.0,
+ "1377": 29.0,
+ "1378": 39.0,
+ "1379": 49.0,
+ "1380": 44.0,
+ "1381": 30.0,
+ "1382": 45.0,
+ "1383": 44.0,
+ "1384": 31.0,
+ "1385": 35.0,
+ "1386": 31.0,
+ "1387": 31.0,
+ "1388": 22.0,
+ "1389": 32.0,
+ "1390": 38.0,
+ "1391": 42.0,
+ "1392": 34.0,
+ "1393": 43.0,
+ "1394": 33.0,
+ "1395": 39.0,
+ "1396": 37.0,
+ "1397": 27.0,
+ "1398": 33.0,
+ "1399": 29.0,
+ "1400": 36.0,
+ "1401": 28.0,
+ "1402": 27.0,
+ "1403": 23.0,
+ "1404": 28.0,
+ "1405": 36.0,
+ "1406": 29.0,
+ "1407": 36.0,
+ "1408": 43.0,
+ "1409": 37.0,
+ "1410": 37.0,
+ "1411": 38.0,
+ "1412": 28.0,
+ "1413": 48.0,
+ "1414": 34.0,
+ "1415": 42.0,
+ "1416": 35.0,
+ "1417": 34.0,
+ "1418": 43.0,
+ "1419": 38.0,
+ "1420": 33.0,
+ "1421": 33.0,
+ "1422": 53.0,
+ "1423": 22.0,
+ "1424": 35.0,
+ "1425": 43.0,
+ "1426": 36.0,
+ "1427": 43.0,
+ "1428": 31.0,
+ "1429": 30.0,
+ "1430": 36.0,
+ "1431": 29.0,
+ "1432": 37.0,
+ "1433": 32.0,
+ "1434": 47.0,
+ "1435": 38.0,
+ "1436": 40.0,
+ "1437": 47.0,
+ "1438": 28.0,
+ "1439": 33.0,
+ "1440": 25.0,
+ "1441": 35.0,
+ "1442": 38.0,
+ "1443": 42.0,
+ "1444": 28.0,
+ "1445": 34.0,
+ "1446": 28.0,
+ "1447": 39.0,
+ "1448": 45.0,
+ "1449": 41.0,
+ "1450": 25.0,
+ "1451": 38.0,
+ "1452": 27.0,
+ "1453": 28.0,
+ "1454": 28.0,
+ "1455": 32.0,
+ "1456": 40.0,
+ "1457": 33.0,
+ "1458": 37.0,
+ "1459": 41.0,
+ "1460": 31.0,
+ "1461": 34.0,
+ "1462": 23.0,
+ "1463": 33.0,
+ "1464": 42.0,
+ "1465": 42.0,
+ "1466": 29.0,
+ "1467": 27.0,
+ "1468": 41.0,
+ "1469": 30.0,
+ "1470": 35.0,
+ "1471": 32.0,
+ "1472": 44.0,
+ "1473": 53.0,
+ "1474": 28.0,
+ "1475": 25.0,
+ "1476": 47.0,
+ "1477": 40.0,
+ "1478": 26.0,
+ "1479": 33.0,
+ "1480": 33.0,
+ "1481": 33.0,
+ "1482": 33.0,
+ "1483": 31.0,
+ "1484": 31.0,
+ "1485": 45.0,
+ "1486": 37.0,
+ "1487": 32.0,
+ "1488": 26.0,
+ "1489": 45.0,
+ "1490": 40.0,
+ "1491": 44.0,
+ "1492": 44.0,
+ "1493": 44.0,
+ "1494": 33.0,
+ "1495": 42.0,
+ "1496": 32.0,
+ "1497": 39.0,
+ "1498": 32.0,
+ "1499": 42.0,
+ "1500": 42.0,
+ "1501": 46.0,
+ "1502": 46.0,
+ "1503": 39.0,
+ "1504": 31.0,
+ "1505": 47.0,
+ "1506": 41.0,
+ "1507": 35.0,
+ "1508": 39.0,
+ "1509": 32.0,
+ "1510": 37.0,
+ "1511": 52.0,
+ "1512": 29.0,
+ "1513": 46.0,
+ "1514": 40.0,
+ "1515": 41.0,
+ "1516": 31.0,
+ "1517": 39.0,
+ "1518": 40.0,
+ "1519": 32.0,
+ "1520": 34.0,
+ "1521": 44.0,
+ "1522": 53.0,
+ "1523": 40.0,
+ "1524": 39.0,
+ "1525": 30.0,
+ "1526": 34.0,
+ "1527": 19.0,
+ "1528": 40.0,
+ "1529": 30.0,
+ "1530": 38.0,
+ "1531": 28.0,
+ "1532": 30.0,
+ "1533": 43.0,
+ "1534": 34.0,
+ "1535": 35.0,
+ "1536": 34.0,
+ "1537": 33.0,
+ "1538": 36.0,
+ "1539": 32.0,
+ "1540": 38.0,
+ "1541": 35.0,
+ "1542": 50.0,
+ "1543": 50.0,
+ "1544": 38.0,
+ "1545": 38.0,
+ "1546": 35.0,
+ "1547": 31.0,
+ "1548": 39.0,
+ "1549": 36.0,
+ "1550": 30.0,
+ "1551": 42.0,
+ "1552": 49.0,
+ "1553": 46.0,
+ "1554": 41.0,
+ "1555": 25.0,
+ "1556": 33.0,
+ "1557": 46.0,
+ "1558": 43.0,
+ "1559": 36.0,
+ "1560": 30.0,
+ "1561": 48.0,
+ "1562": 30.0,
+ "1563": 38.0,
+ "1564": 40.0,
+ "1565": 30.0,
+ "1566": 34.0,
+ "1567": 36.0,
+ "1568": 43.0,
+ "1569": 35.0,
+ "1570": 43.0,
+ "1571": 32.0,
+ "1572": 34.0,
+ "1573": 35.0,
+ "1574": 31.0,
+ "1575": 39.0,
+ "1576": 30.0,
+ "1577": 41.0,
+ "1578": 46.0,
+ "1579": 35.0,
+ "1580": 39.0,
+ "1581": 43.0,
+ "1582": 30.0,
+ "1583": 43.0,
+ "1584": 36.0,
+ "1585": 37.0,
+ "1586": 44.0,
+ "1587": 37.0,
+ "1588": 43.0,
+ "1589": 41.0,
+ "1590": 46.0,
+ "1591": 32.0,
+ "1592": 37.0,
+ "1593": 32.0,
+ "1594": 36.0,
+ "1595": 27.0,
+ "1596": 40.0,
+ "1597": 36.0,
+ "1598": 36.0,
+ "1599": 32.0,
+ "1600": 41.0,
+ "1601": 34.0,
+ "1602": 38.0,
+ "1603": 48.0,
+ "1604": 29.0,
+ "1605": 42.0,
+ "1606": 33.0,
+ "1607": 41.0,
+ "1608": 40.0,
+ "1609": 42.0,
+ "1610": 37.0,
+ "1611": 35.0,
+ "1612": 37.0,
+ "1613": 39.0,
+ "1614": 51.0,
+ "1615": 38.0,
+ "1616": 33.0,
+ "1617": 45.0,
+ "1618": 43.0,
+ "1619": 32.0,
+ "1620": 43.0,
+ "1621": 47.0,
+ "1622": 36.0,
+ "1623": 50.0,
+ "1624": 40.0,
+ "1625": 33.0,
+ "1626": 39.0,
+ "1627": 34.0,
+ "1628": 40.0,
+ "1629": 30.0,
+ "1630": 34.0,
+ "1631": 45.0,
+ "1632": 39.0,
+ "1633": 40.0,
+ "1634": 30.0,
+ "1635": 53.0,
+ "1636": 31.0,
+ "1637": 35.0,
+ "1638": 39.0,
+ "1639": 42.0,
+ "1640": 37.0,
+ "1641": 43.0,
+ "1642": 30.0,
+ "1643": 43.0,
+ "1644": 36.0,
+ "1645": 37.0,
+ "1646": 61.0,
+ "1647": 34.0,
+ "1648": 41.0,
+ "1649": 39.0,
+ "1650": 42.0,
+ "1651": 33.0,
+ "1652": 45.0,
+ "1653": 25.0,
+ "1654": 36.0,
+ "1655": 29.0,
+ "1656": 45.0,
+ "1657": 37.0,
+ "1658": 46.0,
+ "1659": 38.0,
+ "1660": 46.0,
+ "1661": 41.0,
+ "1662": 35.0,
+ "1663": 35.0,
+ "1664": 37.0,
+ "1665": 30.0,
+ "1666": 44.0,
+ "1667": 45.0,
+ "1668": 40.0,
+ "1669": 35.0,
+ "1670": 35.0,
+ "1671": 37.0,
+ "1672": 32.0,
+ "1673": 48.0,
+ "1674": 41.0,
+ "1675": 40.0,
+ "1676": 49.0,
+ "1677": 35.0,
+ "1678": 30.0,
+ "1679": 45.0,
+ "1680": 40.0,
+ "1681": 32.0,
+ "1682": 32.0,
+ "1683": 42.0,
+ "1684": 44.0,
+ "1685": 47.0,
+ "1686": 30.0,
+ "1687": 31.0,
+ "1688": 31.0,
+ "1689": 40.0,
+ "1690": 43.0,
+ "1691": 36.0,
+ "1692": 31.0,
+ "1693": 31.0,
+ "1694": 35.0,
+ "1695": 41.0,
+ "1696": 32.0,
+ "1697": 27.0,
+ "1698": 39.0,
+ "1699": 41.0,
+ "1700": 31.0,
+ "1701": 35.0,
+ "1702": 31.0,
+ "1703": 40.0,
+ "1704": 36.0,
+ "1705": 36.0,
+ "1706": 46.0,
+ "1707": 26.0,
+ "1708": 37.0,
+ "1709": 37.0,
+ "1710": 39.0,
+ "1711": 32.0,
+ "1712": 46.0,
+ "1713": 44.0,
+ "1714": 45.0,
+ "1715": 43.0,
+ "1716": 30.0,
+ "1717": 41.0,
+ "1718": 43.0,
+ "1719": 28.0,
+ "1720": 36.0,
+ "1721": 26.0,
+ "1722": 42.0,
+ "1723": 42.0,
+ "1724": 39.0,
+ "1725": 28.0,
+ "1726": 46.0,
+ "1727": 43.0,
+ "1728": 40.0,
+ "1729": 44.0,
+ "1730": 38.0,
+ "1731": 26.0,
+ "1732": 39.0,
+ "1733": 44.0,
+ "1734": 39.0,
+ "1735": 34.0,
+ "1736": 46.0,
+ "1737": 46.0,
+ "1738": 34.0,
+ "1739": 47.0,
+ "1740": 44.0,
+ "1741": 31.0,
+ "1742": 46.0,
+ "1743": 43.0,
+ "1744": 46.0,
+ "1745": 53.0,
+ "1746": 42.0,
+ "1747": 37.0,
+ "1748": 37.0,
+ "1749": 47.0,
+ "1750": 46.0,
+ "1751": 43.0,
+ "1752": 35.0,
+ "1753": 41.0,
+ "1754": 40.0,
+ "1755": 32.0,
+ "1756": 36.0,
+ "1757": 48.0,
+ "1758": 34.0,
+ "1759": 49.0,
+ "1760": 46.0,
+ "1761": 36.0,
+ "1762": 34.0,
+ "1763": 36.0,
+ "1764": 39.0,
+ "1765": 24.0,
+ "1766": 46.0,
+ "1767": 46.0,
+ "1768": 36.0,
+ "1769": 56.0,
+ "1770": 28.0,
+ "1771": 42.0,
+ "1772": 52.0,
+ "1773": 45.0,
+ "1774": 37.0,
+ "1775": 33.0,
+ "1776": 43.0,
+ "1777": 54.0,
+ "1778": 39.0,
+ "1779": 33.0,
+ "1780": 39.0,
+ "1781": 45.0,
+ "1782": 35.0,
+ "1783": 43.0,
+ "1784": 53.0,
+ "1785": 36.0,
+ "1786": 38.0,
+ "1787": 43.0,
+ "1788": 45.0,
+ "1789": 33.0,
+ "1790": 42.0,
+ "1791": 44.0,
+ "1792": 34.0,
+ "1793": 30.0,
+ "1794": 40.0,
+ "1795": 55.0,
+ "1796": 33.0,
+ "1797": 30.0,
+ "1798": 41.0,
+ "1799": 37.0,
+ "1800": 41.0,
+ "1801": 40.0,
+ "1802": 30.0,
+ "1803": 36.0,
+ "1804": 41.0,
+ "1805": 34.0,
+ "1806": 39.0,
+ "1807": 36.0,
+ "1808": 43.0,
+ "1809": 45.0,
+ "1810": 41.0,
+ "1811": 28.0,
+ "1812": 33.0,
+ "1813": 30.0,
+ "1814": 36.0,
+ "1815": 35.0,
+ "1816": 35.0,
+ "1817": 35.0,
+ "1818": 42.0,
+ "1819": 25.0,
+ "1820": 38.0,
+ "1821": 48.0,
+ "1822": 38.0,
+ "1823": 38.0,
+ "1824": 49.0,
+ "1825": 46.0,
+ "1826": 32.0,
+ "1827": 47.0,
+ "1828": 30.0,
+ "1829": 50.0,
+ "1830": 43.0,
+ "1831": 36.0,
+ "1832": 47.0,
+ "1833": 42.0,
+ "1834": 41.0,
+ "1835": 39.0,
+ "1836": 39.0,
+ "1837": 34.0,
+ "1838": 50.0,
+ "1839": 35.0,
+ "1840": 41.0,
+ "1841": 30.0,
+ "1842": 34.0,
+ "1843": 44.0,
+ "1844": 38.0,
+ "1845": 41.0,
+ "1846": 32.0,
+ "1847": 32.0,
+ "1848": 36.0,
+ "1849": 45.0,
+ "1850": 40.0,
+ "1851": 36.0,
+ "1852": 41.0,
+ "1853": 29.0,
+ "1854": 35.0,
+ "1855": 45.0,
+ "1856": 39.0,
+ "1857": 33.0,
+ "1858": 40.0,
+ "1859": 40.0,
+ "1860": 48.0,
+ "1861": 37.0,
+ "1862": 46.0,
+ "1863": 47.0,
+ "1864": 48.0,
+ "1865": 38.0,
+ "1866": 51.0,
+ "1867": 34.0,
+ "1868": 40.0,
+ "1869": 42.0,
+ "1870": 38.0,
+ "1871": 36.0,
+ "1872": 42.0,
+ "1873": 42.0,
+ "1874": 38.0,
+ "1875": 51.0,
+ "1876": 39.0,
+ "1877": 41.0,
+ "1878": 26.0,
+ "1879": 33.0,
+ "1880": 41.0,
+ "1881": 50.0,
+ "1882": 37.0,
+ "1883": 45.0,
+ "1884": 39.0,
+ "1885": 37.0,
+ "1886": 32.0,
+ "1887": 36.0,
+ "1888": 28.0,
+ "1889": 38.0,
+ "1890": 37.0,
+ "1891": 51.0,
+ "1892": 44.0,
+ "1893": 50.0,
+ "1894": 44.0,
+ "1895": 35.0,
+ "1896": 34.0,
+ "1897": 35.0,
+ "1898": 31.0,
+ "1899": 39.0,
+ "1900": 40.0,
+ "1901": 52.0,
+ "1902": 31.0,
+ "1903": 44.0,
+ "1904": 45.0,
+ "1905": 32.0,
+ "1906": 49.0,
+ "1907": 34.0,
+ "1908": 33.0,
+ "1909": 34.0,
+ "1910": 45.0,
+ "1911": 41.0,
+ "1912": 46.0,
+ "1913": 46.0,
+ "1914": 51.0,
+ "1915": 35.0,
+ "1916": 42.0,
+ "1917": 40.0,
+ "1918": 32.0,
+ "1919": 54.0,
+ "1920": 41.0,
+ "1921": 40.0,
+ "1922": 36.0,
+ "1923": 34.0,
+ "1924": 43.0,
+ "1925": 47.0,
+ "1926": 42.0,
+ "1927": 37.0,
+ "1928": 40.0,
+ "1929": 40.0,
+ "1930": 39.0,
+ "1931": 37.0,
+ "1932": 40.0,
+ "1933": 46.0,
+ "1934": 30.0,
+ "1935": 50.0,
+ "1936": 51.0,
+ "1937": 34.0,
+ "1938": 38.0,
+ "1939": 44.0,
+ "1940": 35.0,
+ "1941": 39.0,
+ "1942": 59.0,
+ "1943": 42.0,
+ "1944": 46.0,
+ "1945": 36.0,
+ "1946": 43.0,
+ "1947": 39.0,
+ "1948": 39.0,
+ "1949": 31.0,
+ "1950": 36.0,
+ "1951": 41.0,
+ "1952": 37.0,
+ "1953": 26.0,
+ "1954": 43.0,
+ "1955": 33.0,
+ "1956": 37.0,
+ "1957": 48.0,
+ "1958": 35.0,
+ "1959": 44.0,
+ "1960": 35.0,
+ "1961": 28.0,
+ "1962": 51.0,
+ "1963": 47.0,
+ "1964": 33.0,
+ "1965": 56.0,
+ "1966": 46.0,
+ "1967": 33.0,
+ "1968": 53.0,
+ "1969": 36.0,
+ "1970": 47.0,
+ "1971": 35.0,
+ "1972": 34.0,
+ "1973": 38.0,
+ "1974": 46.0,
+ "1975": 32.0,
+ "1976": 43.0,
+ "1977": 38.0,
+ "1978": 43.0,
+ "1979": 49.0,
+ "1980": 32.0,
+ "1981": 30.0,
+ "1982": 55.0,
+ "1983": 41.0,
+ "1984": 62.0,
+ "1985": 41.0,
+ "1986": 48.0,
+ "1987": 48.0,
+ "1988": 41.0,
+ "1989": 50.0,
+ "1990": 53.0,
+ "1991": 45.0,
+ "1992": 46.0,
+ "1993": 60.0,
+ "1994": 30.0,
+ "1995": 41.0,
+ "1996": 51.0,
+ "1997": 41.0,
+ "1998": 45.0,
+ "1999": 32.0,
+ "2000": 43.0
+ }
+ },
+ "mem-allocated-bytes": {
+ "start_step": 1,
+ "end_step": 2000,
+ "step_interval": 1,
+ "values": {
+ "1": 302618112.0,
+ "2": 302618112.0,
+ "3": 302618112.0,
+ "4": 302618112.0,
+ "5": 302618112.0,
+ "6": 302618112.0,
+ "7": 302618112.0,
+ "8": 302618112.0,
+ "9": 302618112.0,
+ "10": 302618112.0,
+ "11": 302618112.0,
+ "12": 302618112.0,
+ "13": 302618112.0,
+ "14": 302618112.0,
+ "15": 302618112.0,
+ "16": 302618112.0,
+ "17": 302618112.0,
+ "18": 302618112.0,
+ "19": 302618112.0,
+ "20": 302618112.0,
+ "21": 302618112.0,
+ "22": 302618112.0,
+ "23": 302618112.0,
+ "24": 302618112.0,
+ "25": 302618112.0,
+ "26": 302618112.0,
+ "27": 302618112.0,
+ "28": 302618112.0,
+ "29": 302618112.0,
+ "30": 302618112.0,
+ "31": 302618112.0,
+ "32": 302618112.0,
+ "33": 302618112.0,
+ "34": 302618112.0,
+ "35": 302618112.0,
+ "36": 302618112.0,
+ "37": 302618112.0,
+ "38": 302618112.0,
+ "39": 302618112.0,
+ "40": 302618112.0,
+ "41": 302618112.0,
+ "42": 302618112.0,
+ "43": 302618112.0,
+ "44": 302618112.0,
+ "45": 302618112.0,
+ "46": 302618112.0,
+ "47": 302618112.0,
+ "48": 302618112.0,
+ "49": 302618112.0,
+ "50": 302618112.0,
+ "51": 302618112.0,
+ "52": 302618112.0,
+ "53": 302618112.0,
+ "54": 302618112.0,
+ "55": 302618112.0,
+ "56": 302618112.0,
+ "57": 302618112.0,
+ "58": 302618112.0,
+ "59": 302618112.0,
+ "60": 302618112.0,
+ "61": 302618112.0,
+ "62": 302618112.0,
+ "63": 302618112.0,
+ "64": 302618112.0,
+ "65": 302618112.0,
+ "66": 302618112.0,
+ "67": 302618112.0,
+ "68": 302618112.0,
+ "69": 302618112.0,
+ "70": 302618112.0,
+ "71": 302618112.0,
+ "72": 302618112.0,
+ "73": 302618112.0,
+ "74": 302618112.0,
+ "75": 302618112.0,
+ "76": 302618112.0,
+ "77": 302618112.0,
+ "78": 302618112.0,
+ "79": 302618112.0,
+ "80": 302618112.0,
+ "81": 302618112.0,
+ "82": 302618112.0,
+ "83": 302618112.0,
+ "84": 302618112.0,
+ "85": 302618112.0,
+ "86": 302618112.0,
+ "87": 302618112.0,
+ "88": 302618112.0,
+ "89": 302618112.0,
+ "90": 302618112.0,
+ "91": 302618112.0,
+ "92": 302618112.0,
+ "93": 302618112.0,
+ "94": 302618112.0,
+ "95": 302618112.0,
+ "96": 302618112.0,
+ "97": 302618112.0,
+ "98": 302618112.0,
+ "99": 302618112.0,
+ "100": 302618112.0,
+ "101": 302618112.0,
+ "102": 302618112.0,
+ "103": 302618112.0,
+ "104": 302618112.0,
+ "105": 302618112.0,
+ "106": 302618112.0,
+ "107": 302618112.0,
+ "108": 302618112.0,
+ "109": 302618112.0,
+ "110": 302618112.0,
+ "111": 302618112.0,
+ "112": 302618112.0,
+ "113": 302618112.0,
+ "114": 302618112.0,
+ "115": 302618112.0,
+ "116": 302618112.0,
+ "117": 302618112.0,
+ "118": 302618112.0,
+ "119": 302618112.0,
+ "120": 302618112.0,
+ "121": 302618112.0,
+ "122": 302618112.0,
+ "123": 302618112.0,
+ "124": 302618112.0,
+ "125": 302618112.0,
+ "126": 302618112.0,
+ "127": 302618112.0,
+ "128": 302618112.0,
+ "129": 302618112.0,
+ "130": 302618112.0,
+ "131": 302618112.0,
+ "132": 302618112.0,
+ "133": 302618112.0,
+ "134": 302618112.0,
+ "135": 302618112.0,
+ "136": 302618112.0,
+ "137": 302618112.0,
+ "138": 302618112.0,
+ "139": 302618112.0,
+ "140": 302618112.0,
+ "141": 302618112.0,
+ "142": 302618112.0,
+ "143": 302618112.0,
+ "144": 302618112.0,
+ "145": 302618112.0,
+ "146": 302618112.0,
+ "147": 302618112.0,
+ "148": 302618112.0,
+ "149": 302618112.0,
+ "150": 302618112.0,
+ "151": 302618112.0,
+ "152": 302618112.0,
+ "153": 302618112.0,
+ "154": 302618112.0,
+ "155": 302618112.0,
+ "156": 302618112.0,
+ "157": 302618112.0,
+ "158": 302618112.0,
+ "159": 302618112.0,
+ "160": 302618112.0,
+ "161": 302618112.0,
+ "162": 302618112.0,
+ "163": 302618112.0,
+ "164": 302618112.0,
+ "165": 302618112.0,
+ "166": 302618112.0,
+ "167": 302618112.0,
+ "168": 302618112.0,
+ "169": 302618112.0,
+ "170": 302618112.0,
+ "171": 302618112.0,
+ "172": 302618112.0,
+ "173": 302618112.0,
+ "174": 302618112.0,
+ "175": 302618112.0,
+ "176": 302618112.0,
+ "177": 302618112.0,
+ "178": 302618112.0,
+ "179": 302618112.0,
+ "180": 302618112.0,
+ "181": 302618112.0,
+ "182": 302618112.0,
+ "183": 302618112.0,
+ "184": 302618112.0,
+ "185": 302618112.0,
+ "186": 302618112.0,
+ "187": 302618112.0,
+ "188": 302618112.0,
+ "189": 302618112.0,
+ "190": 302618112.0,
+ "191": 302618112.0,
+ "192": 302618112.0,
+ "193": 302618112.0,
+ "194": 302618112.0,
+ "195": 302618112.0,
+ "196": 302618112.0,
+ "197": 302618112.0,
+ "198": 302618112.0,
+ "199": 302618112.0,
+ "200": 302618112.0,
+ "201": 302618112.0,
+ "202": 302618112.0,
+ "203": 302618112.0,
+ "204": 302618112.0,
+ "205": 302618112.0,
+ "206": 302618112.0,
+ "207": 302618112.0,
+ "208": 302618112.0,
+ "209": 302618112.0,
+ "210": 302618112.0,
+ "211": 302618112.0,
+ "212": 302618112.0,
+ "213": 302618112.0,
+ "214": 302618112.0,
+ "215": 302618112.0,
+ "216": 302618112.0,
+ "217": 302618112.0,
+ "218": 302618112.0,
+ "219": 302618112.0,
+ "220": 302618112.0,
+ "221": 302618112.0,
+ "222": 302618112.0,
+ "223": 302618112.0,
+ "224": 302618112.0,
+ "225": 302618112.0,
+ "226": 302618112.0,
+ "227": 302618112.0,
+ "228": 302618112.0,
+ "229": 302618112.0,
+ "230": 302618112.0,
+ "231": 302618112.0,
+ "232": 302618112.0,
+ "233": 302618112.0,
+ "234": 302618112.0,
+ "235": 302618112.0,
+ "236": 302618112.0,
+ "237": 302618112.0,
+ "238": 302618112.0,
+ "239": 302618112.0,
+ "240": 302618112.0,
+ "241": 302618112.0,
+ "242": 302618112.0,
+ "243": 302618112.0,
+ "244": 302618112.0,
+ "245": 302618112.0,
+ "246": 302618112.0,
+ "247": 302618112.0,
+ "248": 302618112.0,
+ "249": 302618112.0,
+ "250": 302618112.0,
+ "251": 302618112.0,
+ "252": 302618112.0,
+ "253": 302618112.0,
+ "254": 302618112.0,
+ "255": 302618112.0,
+ "256": 302618112.0,
+ "257": 302618112.0,
+ "258": 302618112.0,
+ "259": 302618112.0,
+ "260": 302618112.0,
+ "261": 302618112.0,
+ "262": 302618112.0,
+ "263": 302618112.0,
+ "264": 302618112.0,
+ "265": 302618112.0,
+ "266": 302618112.0,
+ "267": 302618112.0,
+ "268": 302618112.0,
+ "269": 302618112.0,
+ "270": 302618112.0,
+ "271": 302618112.0,
+ "272": 302618112.0,
+ "273": 302618112.0,
+ "274": 302618112.0,
+ "275": 302618112.0,
+ "276": 302618112.0,
+ "277": 302618112.0,
+ "278": 302618112.0,
+ "279": 302618112.0,
+ "280": 302618112.0,
+ "281": 302618112.0,
+ "282": 302618112.0,
+ "283": 302618112.0,
+ "284": 302618112.0,
+ "285": 302618112.0,
+ "286": 302618112.0,
+ "287": 302618112.0,
+ "288": 302618112.0,
+ "289": 302618112.0,
+ "290": 302618112.0,
+ "291": 302618112.0,
+ "292": 302618112.0,
+ "293": 302618112.0,
+ "294": 302618112.0,
+ "295": 302618112.0,
+ "296": 302618112.0,
+ "297": 302618112.0,
+ "298": 302618112.0,
+ "299": 302618112.0,
+ "300": 302618112.0,
+ "301": 302618112.0,
+ "302": 302618112.0,
+ "303": 302618112.0,
+ "304": 302618112.0,
+ "305": 302618112.0,
+ "306": 302618112.0,
+ "307": 302618112.0,
+ "308": 302618112.0,
+ "309": 302618112.0,
+ "310": 302618112.0,
+ "311": 302618112.0,
+ "312": 302618112.0,
+ "313": 302618112.0,
+ "314": 302618112.0,
+ "315": 302618112.0,
+ "316": 302618112.0,
+ "317": 302618112.0,
+ "318": 302618112.0,
+ "319": 302618112.0,
+ "320": 302618112.0,
+ "321": 302618112.0,
+ "322": 302618112.0,
+ "323": 302618112.0,
+ "324": 302618112.0,
+ "325": 302618112.0,
+ "326": 302618112.0,
+ "327": 302618112.0,
+ "328": 302618112.0,
+ "329": 302618112.0,
+ "330": 302618112.0,
+ "331": 302618112.0,
+ "332": 302618112.0,
+ "333": 302618112.0,
+ "334": 302618112.0,
+ "335": 302618112.0,
+ "336": 302618112.0,
+ "337": 302618112.0,
+ "338": 302618112.0,
+ "339": 302618112.0,
+ "340": 302618112.0,
+ "341": 302618112.0,
+ "342": 302618112.0,
+ "343": 302618112.0,
+ "344": 302618112.0,
+ "345": 302618112.0,
+ "346": 302618112.0,
+ "347": 302618112.0,
+ "348": 302618112.0,
+ "349": 302618112.0,
+ "350": 302618112.0,
+ "351": 302618112.0,
+ "352": 302618112.0,
+ "353": 302618112.0,
+ "354": 302618112.0,
+ "355": 302618112.0,
+ "356": 302618112.0,
+ "357": 302618112.0,
+ "358": 302618112.0,
+ "359": 302618112.0,
+ "360": 302618112.0,
+ "361": 302618112.0,
+ "362": 302618112.0,
+ "363": 302618112.0,
+ "364": 302618112.0,
+ "365": 302618112.0,
+ "366": 302618112.0,
+ "367": 302618112.0,
+ "368": 302618112.0,
+ "369": 302618112.0,
+ "370": 302618112.0,
+ "371": 302618112.0,
+ "372": 302618112.0,
+ "373": 302618112.0,
+ "374": 302618112.0,
+ "375": 302618112.0,
+ "376": 302618112.0,
+ "377": 302618112.0,
+ "378": 302618112.0,
+ "379": 302618112.0,
+ "380": 302618112.0,
+ "381": 302618112.0,
+ "382": 302618112.0,
+ "383": 302618112.0,
+ "384": 302618112.0,
+ "385": 302618112.0,
+ "386": 302618112.0,
+ "387": 302618112.0,
+ "388": 302618112.0,
+ "389": 302618112.0,
+ "390": 302618112.0,
+ "391": 302618112.0,
+ "392": 302618112.0,
+ "393": 302618112.0,
+ "394": 302618112.0,
+ "395": 302618112.0,
+ "396": 302618112.0,
+ "397": 302618112.0,
+ "398": 302618112.0,
+ "399": 302618112.0,
+ "400": 302618112.0,
+ "401": 302618112.0,
+ "402": 302618112.0,
+ "403": 302618112.0,
+ "404": 302618112.0,
+ "405": 302618112.0,
+ "406": 302618112.0,
+ "407": 302618112.0,
+ "408": 302618112.0,
+ "409": 302618112.0,
+ "410": 302618112.0,
+ "411": 302618112.0,
+ "412": 302618112.0,
+ "413": 302618112.0,
+ "414": 302618112.0,
+ "415": 302618112.0,
+ "416": 302618112.0,
+ "417": 302618112.0,
+ "418": 302618112.0,
+ "419": 302618112.0,
+ "420": 302618112.0,
+ "421": 302618112.0,
+ "422": 302618112.0,
+ "423": 302618112.0,
+ "424": 302618112.0,
+ "425": 302618112.0,
+ "426": 302618112.0,
+ "427": 302618112.0,
+ "428": 302618112.0,
+ "429": 302618112.0,
+ "430": 302618112.0,
+ "431": 302618112.0,
+ "432": 302618112.0,
+ "433": 302618112.0,
+ "434": 302618112.0,
+ "435": 302618112.0,
+ "436": 302618112.0,
+ "437": 302618112.0,
+ "438": 302618112.0,
+ "439": 302618112.0,
+ "440": 302618112.0,
+ "441": 302618112.0,
+ "442": 302618112.0,
+ "443": 302618112.0,
+ "444": 302618112.0,
+ "445": 302618112.0,
+ "446": 302618112.0,
+ "447": 302618112.0,
+ "448": 302618112.0,
+ "449": 302618112.0,
+ "450": 302618112.0,
+ "451": 302618112.0,
+ "452": 302618112.0,
+ "453": 302618112.0,
+ "454": 302618112.0,
+ "455": 302618112.0,
+ "456": 302618112.0,
+ "457": 302618112.0,
+ "458": 302618112.0,
+ "459": 302618112.0,
+ "460": 302618112.0,
+ "461": 302618112.0,
+ "462": 302618112.0,
+ "463": 302618112.0,
+ "464": 302618112.0,
+ "465": 302618112.0,
+ "466": 302618112.0,
+ "467": 302618112.0,
+ "468": 302618112.0,
+ "469": 302618112.0,
+ "470": 302618112.0,
+ "471": 302618112.0,
+ "472": 302618112.0,
+ "473": 302618112.0,
+ "474": 302618112.0,
+ "475": 302618112.0,
+ "476": 302618112.0,
+ "477": 302618112.0,
+ "478": 302618112.0,
+ "479": 302618112.0,
+ "480": 302618112.0,
+ "481": 302618112.0,
+ "482": 302618112.0,
+ "483": 302618112.0,
+ "484": 302618112.0,
+ "485": 302618112.0,
+ "486": 302618112.0,
+ "487": 302618112.0,
+ "488": 302618112.0,
+ "489": 302618112.0,
+ "490": 302618112.0,
+ "491": 302618112.0,
+ "492": 302618112.0,
+ "493": 302618112.0,
+ "494": 302618112.0,
+ "495": 302618112.0,
+ "496": 302618112.0,
+ "497": 302618112.0,
+ "498": 302618112.0,
+ "499": 302618112.0,
+ "500": 302618112.0,
+ "501": 302618112.0,
+ "502": 302618112.0,
+ "503": 302618112.0,
+ "504": 302618112.0,
+ "505": 302618112.0,
+ "506": 302618112.0,
+ "507": 302618112.0,
+ "508": 302618112.0,
+ "509": 302618112.0,
+ "510": 302618112.0,
+ "511": 302618112.0,
+ "512": 302618112.0,
+ "513": 302618112.0,
+ "514": 302618112.0,
+ "515": 302618112.0,
+ "516": 302618112.0,
+ "517": 302618112.0,
+ "518": 302618112.0,
+ "519": 302618112.0,
+ "520": 302618112.0,
+ "521": 302618112.0,
+ "522": 302618112.0,
+ "523": 302618112.0,
+ "524": 302618112.0,
+ "525": 302618112.0,
+ "526": 302618112.0,
+ "527": 302618112.0,
+ "528": 302618112.0,
+ "529": 302618112.0,
+ "530": 302618112.0,
+ "531": 302618112.0,
+ "532": 302618112.0,
+ "533": 302618112.0,
+ "534": 302618112.0,
+ "535": 302618112.0,
+ "536": 302618112.0,
+ "537": 302618112.0,
+ "538": 302618112.0,
+ "539": 302618112.0,
+ "540": 302618112.0,
+ "541": 302618112.0,
+ "542": 302618112.0,
+ "543": 302618112.0,
+ "544": 302618112.0,
+ "545": 302618112.0,
+ "546": 302618112.0,
+ "547": 302618112.0,
+ "548": 302618112.0,
+ "549": 302618112.0,
+ "550": 302618112.0,
+ "551": 302618112.0,
+ "552": 302618112.0,
+ "553": 302618112.0,
+ "554": 302618112.0,
+ "555": 302618112.0,
+ "556": 302618112.0,
+ "557": 302618112.0,
+ "558": 302618112.0,
+ "559": 302618112.0,
+ "560": 302618112.0,
+ "561": 302618112.0,
+ "562": 302618112.0,
+ "563": 302618112.0,
+ "564": 302618112.0,
+ "565": 302618112.0,
+ "566": 302618112.0,
+ "567": 302618112.0,
+ "568": 302618112.0,
+ "569": 302618112.0,
+ "570": 302618112.0,
+ "571": 302618112.0,
+ "572": 302618112.0,
+ "573": 302618112.0,
+ "574": 302618112.0,
+ "575": 302618112.0,
+ "576": 302618112.0,
+ "577": 302618112.0,
+ "578": 302618112.0,
+ "579": 302618112.0,
+ "580": 302618112.0,
+ "581": 302618112.0,
+ "582": 302618112.0,
+ "583": 302618112.0,
+ "584": 302618112.0,
+ "585": 302618112.0,
+ "586": 302618112.0,
+ "587": 302618112.0,
+ "588": 302618112.0,
+ "589": 302618112.0,
+ "590": 302618112.0,
+ "591": 302618112.0,
+ "592": 302618112.0,
+ "593": 302618112.0,
+ "594": 302618112.0,
+ "595": 302618112.0,
+ "596": 302618112.0,
+ "597": 302618112.0,
+ "598": 302618112.0,
+ "599": 302618112.0,
+ "600": 302618112.0,
+ "601": 302618112.0,
+ "602": 302618112.0,
+ "603": 302618112.0,
+ "604": 302618112.0,
+ "605": 302618112.0,
+ "606": 302618112.0,
+ "607": 302618112.0,
+ "608": 302618112.0,
+ "609": 302618112.0,
+ "610": 302618112.0,
+ "611": 302618112.0,
+ "612": 302618112.0,
+ "613": 302618112.0,
+ "614": 302618112.0,
+ "615": 302618112.0,
+ "616": 302618112.0,
+ "617": 302618112.0,
+ "618": 302618112.0,
+ "619": 302618112.0,
+ "620": 302618112.0,
+ "621": 302618112.0,
+ "622": 302618112.0,
+ "623": 302618112.0,
+ "624": 302618112.0,
+ "625": 302618112.0,
+ "626": 302618112.0,
+ "627": 302618112.0,
+ "628": 302618112.0,
+ "629": 302618112.0,
+ "630": 302618112.0,
+ "631": 302618112.0,
+ "632": 302618112.0,
+ "633": 302618112.0,
+ "634": 302618112.0,
+ "635": 302618112.0,
+ "636": 302618112.0,
+ "637": 302618112.0,
+ "638": 302618112.0,
+ "639": 302618112.0,
+ "640": 302618112.0,
+ "641": 302618112.0,
+ "642": 302618112.0,
+ "643": 302618112.0,
+ "644": 302618112.0,
+ "645": 302618112.0,
+ "646": 302618112.0,
+ "647": 302618112.0,
+ "648": 302618112.0,
+ "649": 302618112.0,
+ "650": 302618112.0,
+ "651": 302618112.0,
+ "652": 302618112.0,
+ "653": 302618112.0,
+ "654": 302618112.0,
+ "655": 302618112.0,
+ "656": 302618112.0,
+ "657": 302618112.0,
+ "658": 302618112.0,
+ "659": 302618112.0,
+ "660": 302618112.0,
+ "661": 302618112.0,
+ "662": 302618112.0,
+ "663": 302618112.0,
+ "664": 302618112.0,
+ "665": 302618112.0,
+ "666": 302618112.0,
+ "667": 302618112.0,
+ "668": 302618112.0,
+ "669": 302618112.0,
+ "670": 302618112.0,
+ "671": 302618112.0,
+ "672": 302618112.0,
+ "673": 302618112.0,
+ "674": 302618112.0,
+ "675": 302618112.0,
+ "676": 302618112.0,
+ "677": 302618112.0,
+ "678": 302618112.0,
+ "679": 302618112.0,
+ "680": 302618112.0,
+ "681": 302618112.0,
+ "682": 302618112.0,
+ "683": 302618112.0,
+ "684": 302618112.0,
+ "685": 302618112.0,
+ "686": 302618112.0,
+ "687": 302618112.0,
+ "688": 302618112.0,
+ "689": 302618112.0,
+ "690": 302618112.0,
+ "691": 302618112.0,
+ "692": 302618112.0,
+ "693": 302618112.0,
+ "694": 302618112.0,
+ "695": 302618112.0,
+ "696": 302618112.0,
+ "697": 302618112.0,
+ "698": 302618112.0,
+ "699": 302618112.0,
+ "700": 302618112.0,
+ "701": 302618112.0,
+ "702": 302618112.0,
+ "703": 302618112.0,
+ "704": 302618112.0,
+ "705": 302618112.0,
+ "706": 302618112.0,
+ "707": 302618112.0,
+ "708": 302618112.0,
+ "709": 302618112.0,
+ "710": 302618112.0,
+ "711": 302618112.0,
+ "712": 302618112.0,
+ "713": 302618112.0,
+ "714": 302618112.0,
+ "715": 302618112.0,
+ "716": 302618112.0,
+ "717": 302618112.0,
+ "718": 302618112.0,
+ "719": 302618112.0,
+ "720": 302618112.0,
+ "721": 302618112.0,
+ "722": 302618112.0,
+ "723": 302618112.0,
+ "724": 302618112.0,
+ "725": 302618112.0,
+ "726": 302618112.0,
+ "727": 302618112.0,
+ "728": 302618112.0,
+ "729": 302618112.0,
+ "730": 302618112.0,
+ "731": 302618112.0,
+ "732": 302618112.0,
+ "733": 302618112.0,
+ "734": 302618112.0,
+ "735": 302618112.0,
+ "736": 302618112.0,
+ "737": 302618112.0,
+ "738": 302618112.0,
+ "739": 302618112.0,
+ "740": 302618112.0,
+ "741": 302618112.0,
+ "742": 302618112.0,
+ "743": 302618112.0,
+ "744": 302618112.0,
+ "745": 302618112.0,
+ "746": 302618112.0,
+ "747": 302618112.0,
+ "748": 302618112.0,
+ "749": 302618112.0,
+ "750": 302618112.0,
+ "751": 302618112.0,
+ "752": 302618112.0,
+ "753": 302618112.0,
+ "754": 302618112.0,
+ "755": 302618112.0,
+ "756": 302618112.0,
+ "757": 302618112.0,
+ "758": 302618112.0,
+ "759": 302618112.0,
+ "760": 302618112.0,
+ "761": 302618112.0,
+ "762": 302618112.0,
+ "763": 302618112.0,
+ "764": 302618112.0,
+ "765": 302618112.0,
+ "766": 302618112.0,
+ "767": 302618112.0,
+ "768": 302618112.0,
+ "769": 302618112.0,
+ "770": 302618112.0,
+ "771": 302618112.0,
+ "772": 302618112.0,
+ "773": 302618112.0,
+ "774": 302618112.0,
+ "775": 302618112.0,
+ "776": 302618112.0,
+ "777": 302618112.0,
+ "778": 302618112.0,
+ "779": 302618112.0,
+ "780": 302618112.0,
+ "781": 302618112.0,
+ "782": 302618112.0,
+ "783": 302618112.0,
+ "784": 302618112.0,
+ "785": 302618112.0,
+ "786": 302618112.0,
+ "787": 302618112.0,
+ "788": 302618112.0,
+ "789": 302618112.0,
+ "790": 302618112.0,
+ "791": 302618112.0,
+ "792": 302618112.0,
+ "793": 302618112.0,
+ "794": 302618112.0,
+ "795": 302618112.0,
+ "796": 302618112.0,
+ "797": 302618112.0,
+ "798": 302618112.0,
+ "799": 302618112.0,
+ "800": 302618112.0,
+ "801": 302618112.0,
+ "802": 302618112.0,
+ "803": 302618112.0,
+ "804": 302618112.0,
+ "805": 302618112.0,
+ "806": 302618112.0,
+ "807": 302618112.0,
+ "808": 302618112.0,
+ "809": 302618112.0,
+ "810": 302618112.0,
+ "811": 302618112.0,
+ "812": 302618112.0,
+ "813": 302618112.0,
+ "814": 302618112.0,
+ "815": 302618112.0,
+ "816": 302618112.0,
+ "817": 302618112.0,
+ "818": 302618112.0,
+ "819": 302618112.0,
+ "820": 302618112.0,
+ "821": 302618112.0,
+ "822": 302618112.0,
+ "823": 302618112.0,
+ "824": 302618112.0,
+ "825": 302618112.0,
+ "826": 302618112.0,
+ "827": 302618112.0,
+ "828": 302618112.0,
+ "829": 302618112.0,
+ "830": 302618112.0,
+ "831": 302618112.0,
+ "832": 302618112.0,
+ "833": 302618112.0,
+ "834": 302618112.0,
+ "835": 302618112.0,
+ "836": 302618112.0,
+ "837": 302618112.0,
+ "838": 302618112.0,
+ "839": 302618112.0,
+ "840": 302618112.0,
+ "841": 302618112.0,
+ "842": 302618112.0,
+ "843": 302618112.0,
+ "844": 302618112.0,
+ "845": 302618112.0,
+ "846": 302618112.0,
+ "847": 302618112.0,
+ "848": 302618112.0,
+ "849": 302618112.0,
+ "850": 302618112.0,
+ "851": 302618112.0,
+ "852": 302618112.0,
+ "853": 302618112.0,
+ "854": 302618112.0,
+ "855": 302618112.0,
+ "856": 302618112.0,
+ "857": 302618112.0,
+ "858": 302618112.0,
+ "859": 302618112.0,
+ "860": 302618112.0,
+ "861": 302618112.0,
+ "862": 302618112.0,
+ "863": 302618112.0,
+ "864": 302618112.0,
+ "865": 302618112.0,
+ "866": 302618112.0,
+ "867": 302618112.0,
+ "868": 302618112.0,
+ "869": 302618112.0,
+ "870": 302618112.0,
+ "871": 302618112.0,
+ "872": 302618112.0,
+ "873": 302618112.0,
+ "874": 302618112.0,
+ "875": 302618112.0,
+ "876": 302618112.0,
+ "877": 302618112.0,
+ "878": 302618112.0,
+ "879": 302618112.0,
+ "880": 302618112.0,
+ "881": 302618112.0,
+ "882": 302618112.0,
+ "883": 302618112.0,
+ "884": 302618112.0,
+ "885": 302618112.0,
+ "886": 302618112.0,
+ "887": 302618112.0,
+ "888": 302618112.0,
+ "889": 302618112.0,
+ "890": 302618112.0,
+ "891": 302618112.0,
+ "892": 302618112.0,
+ "893": 302618112.0,
+ "894": 302618112.0,
+ "895": 302618112.0,
+ "896": 302618112.0,
+ "897": 302618112.0,
+ "898": 302618112.0,
+ "899": 302618112.0,
+ "900": 302618112.0,
+ "901": 302618112.0,
+ "902": 302618112.0,
+ "903": 302618112.0,
+ "904": 302618112.0,
+ "905": 302618112.0,
+ "906": 302618112.0,
+ "907": 302618112.0,
+ "908": 302618112.0,
+ "909": 302618112.0,
+ "910": 302618112.0,
+ "911": 302618112.0,
+ "912": 302618112.0,
+ "913": 302618112.0,
+ "914": 302618112.0,
+ "915": 302618112.0,
+ "916": 302618112.0,
+ "917": 302618112.0,
+ "918": 302618112.0,
+ "919": 302618112.0,
+ "920": 302618112.0,
+ "921": 302618112.0,
+ "922": 302618112.0,
+ "923": 302618112.0,
+ "924": 302618112.0,
+ "925": 302618112.0,
+ "926": 302618112.0,
+ "927": 302618112.0,
+ "928": 302618112.0,
+ "929": 302618112.0,
+ "930": 302618112.0,
+ "931": 302618112.0,
+ "932": 302618112.0,
+ "933": 302618112.0,
+ "934": 302618112.0,
+ "935": 302618112.0,
+ "936": 302618112.0,
+ "937": 302618112.0,
+ "938": 302618112.0,
+ "939": 302618112.0,
+ "940": 302618112.0,
+ "941": 302618112.0,
+ "942": 302618112.0,
+ "943": 302618112.0,
+ "944": 302618112.0,
+ "945": 302618112.0,
+ "946": 302618112.0,
+ "947": 302618112.0,
+ "948": 302618112.0,
+ "949": 302618112.0,
+ "950": 302618112.0,
+ "951": 302618112.0,
+ "952": 302618112.0,
+ "953": 302618112.0,
+ "954": 302618112.0,
+ "955": 302618112.0,
+ "956": 302618112.0,
+ "957": 302618112.0,
+ "958": 302618112.0,
+ "959": 302618112.0,
+ "960": 302618112.0,
+ "961": 302618112.0,
+ "962": 302618112.0,
+ "963": 302618112.0,
+ "964": 302618112.0,
+ "965": 302618112.0,
+ "966": 302618112.0,
+ "967": 302618112.0,
+ "968": 302618112.0,
+ "969": 302618112.0,
+ "970": 302618112.0,
+ "971": 302618112.0,
+ "972": 302618112.0,
+ "973": 302618112.0,
+ "974": 302618112.0,
+ "975": 302618112.0,
+ "976": 302618112.0,
+ "977": 302618112.0,
+ "978": 302618112.0,
+ "979": 302618112.0,
+ "980": 302618112.0,
+ "981": 302618112.0,
+ "982": 302618112.0,
+ "983": 302618112.0,
+ "984": 302618112.0,
+ "985": 302618112.0,
+ "986": 302618112.0,
+ "987": 302618112.0,
+ "988": 302618112.0,
+ "989": 302618112.0,
+ "990": 302618112.0,
+ "991": 302618112.0,
+ "992": 302618112.0,
+ "993": 302618112.0,
+ "994": 302618112.0,
+ "995": 302618112.0,
+ "996": 302618112.0,
+ "997": 302618112.0,
+ "998": 302618112.0,
+ "999": 302618112.0,
+ "1000": 302618112.0,
+ "1001": 302618112.0,
+ "1002": 302618112.0,
+ "1003": 302618112.0,
+ "1004": 302618112.0,
+ "1005": 302618112.0,
+ "1006": 302618112.0,
+ "1007": 302618112.0,
+ "1008": 302618112.0,
+ "1009": 302618112.0,
+ "1010": 302618112.0,
+ "1011": 302618112.0,
+ "1012": 302618112.0,
+ "1013": 302618112.0,
+ "1014": 302618112.0,
+ "1015": 302618112.0,
+ "1016": 302618112.0,
+ "1017": 302618112.0,
+ "1018": 302618112.0,
+ "1019": 302618112.0,
+ "1020": 302618112.0,
+ "1021": 302618112.0,
+ "1022": 302618112.0,
+ "1023": 302618112.0,
+ "1024": 302618112.0,
+ "1025": 302618112.0,
+ "1026": 302618112.0,
+ "1027": 302618112.0,
+ "1028": 302618112.0,
+ "1029": 302618112.0,
+ "1030": 302618112.0,
+ "1031": 302618112.0,
+ "1032": 302618112.0,
+ "1033": 302618112.0,
+ "1034": 302618112.0,
+ "1035": 302618112.0,
+ "1036": 302618112.0,
+ "1037": 302618112.0,
+ "1038": 302618112.0,
+ "1039": 302618112.0,
+ "1040": 302618112.0,
+ "1041": 302618112.0,
+ "1042": 302618112.0,
+ "1043": 302618112.0,
+ "1044": 302618112.0,
+ "1045": 302618112.0,
+ "1046": 302618112.0,
+ "1047": 302618112.0,
+ "1048": 302618112.0,
+ "1049": 302618112.0,
+ "1050": 302618112.0,
+ "1051": 302618112.0,
+ "1052": 302618112.0,
+ "1053": 302618112.0,
+ "1054": 302618112.0,
+ "1055": 302618112.0,
+ "1056": 302618112.0,
+ "1057": 302618112.0,
+ "1058": 302618112.0,
+ "1059": 302618112.0,
+ "1060": 302618112.0,
+ "1061": 302618112.0,
+ "1062": 302618112.0,
+ "1063": 302618112.0,
+ "1064": 302618112.0,
+ "1065": 302618112.0,
+ "1066": 302618112.0,
+ "1067": 302618112.0,
+ "1068": 302618112.0,
+ "1069": 302618112.0,
+ "1070": 302618112.0,
+ "1071": 302618112.0,
+ "1072": 302618112.0,
+ "1073": 302618112.0,
+ "1074": 302618112.0,
+ "1075": 302618112.0,
+ "1076": 302618112.0,
+ "1077": 302618112.0,
+ "1078": 302618112.0,
+ "1079": 302618112.0,
+ "1080": 302618112.0,
+ "1081": 302618112.0,
+ "1082": 302618112.0,
+ "1083": 302618112.0,
+ "1084": 302618112.0,
+ "1085": 302618112.0,
+ "1086": 302618112.0,
+ "1087": 302618112.0,
+ "1088": 302618112.0,
+ "1089": 302618112.0,
+ "1090": 302618112.0,
+ "1091": 302618112.0,
+ "1092": 302618112.0,
+ "1093": 302618112.0,
+ "1094": 302618112.0,
+ "1095": 302618112.0,
+ "1096": 302618112.0,
+ "1097": 302618112.0,
+ "1098": 302618112.0,
+ "1099": 302618112.0,
+ "1100": 302618112.0,
+ "1101": 302618112.0,
+ "1102": 302618112.0,
+ "1103": 302618112.0,
+ "1104": 302618112.0,
+ "1105": 302618112.0,
+ "1106": 302618112.0,
+ "1107": 302618112.0,
+ "1108": 302618112.0,
+ "1109": 302618112.0,
+ "1110": 302618112.0,
+ "1111": 302618112.0,
+ "1112": 302618112.0,
+ "1113": 302618112.0,
+ "1114": 302618112.0,
+ "1115": 302618112.0,
+ "1116": 302618112.0,
+ "1117": 302618112.0,
+ "1118": 302618112.0,
+ "1119": 302618112.0,
+ "1120": 302618112.0,
+ "1121": 302618112.0,
+ "1122": 302618112.0,
+ "1123": 302618112.0,
+ "1124": 302618112.0,
+ "1125": 302618112.0,
+ "1126": 302618112.0,
+ "1127": 302618112.0,
+ "1128": 302618112.0,
+ "1129": 302618112.0,
+ "1130": 302618112.0,
+ "1131": 302618112.0,
+ "1132": 302618112.0,
+ "1133": 302618112.0,
+ "1134": 302618112.0,
+ "1135": 302618112.0,
+ "1136": 302618112.0,
+ "1137": 302618112.0,
+ "1138": 302618112.0,
+ "1139": 302618112.0,
+ "1140": 302618112.0,
+ "1141": 302618112.0,
+ "1142": 302618112.0,
+ "1143": 302618112.0,
+ "1144": 302618112.0,
+ "1145": 302618112.0,
+ "1146": 302618112.0,
+ "1147": 302618112.0,
+ "1148": 302618112.0,
+ "1149": 302618112.0,
+ "1150": 302618112.0,
+ "1151": 302618112.0,
+ "1152": 302618112.0,
+ "1153": 302618112.0,
+ "1154": 302618112.0,
+ "1155": 302618112.0,
+ "1156": 302618112.0,
+ "1157": 302618112.0,
+ "1158": 302618112.0,
+ "1159": 302618112.0,
+ "1160": 302618112.0,
+ "1161": 302618112.0,
+ "1162": 302618112.0,
+ "1163": 302618112.0,
+ "1164": 302618112.0,
+ "1165": 302618112.0,
+ "1166": 302618112.0,
+ "1167": 302618112.0,
+ "1168": 302618112.0,
+ "1169": 302618112.0,
+ "1170": 302618112.0,
+ "1171": 302618112.0,
+ "1172": 302618112.0,
+ "1173": 302618112.0,
+ "1174": 302618112.0,
+ "1175": 302618112.0,
+ "1176": 302618112.0,
+ "1177": 302618112.0,
+ "1178": 302618112.0,
+ "1179": 302618112.0,
+ "1180": 302618112.0,
+ "1181": 302618112.0,
+ "1182": 302618112.0,
+ "1183": 302618112.0,
+ "1184": 302618112.0,
+ "1185": 302618112.0,
+ "1186": 302618112.0,
+ "1187": 302618112.0,
+ "1188": 302618112.0,
+ "1189": 302618112.0,
+ "1190": 302618112.0,
+ "1191": 302618112.0,
+ "1192": 302618112.0,
+ "1193": 302618112.0,
+ "1194": 302618112.0,
+ "1195": 302618112.0,
+ "1196": 302618112.0,
+ "1197": 302618112.0,
+ "1198": 302618112.0,
+ "1199": 302618112.0,
+ "1200": 302618112.0,
+ "1201": 302618112.0,
+ "1202": 302618112.0,
+ "1203": 302618112.0,
+ "1204": 302618112.0,
+ "1205": 302618112.0,
+ "1206": 302618112.0,
+ "1207": 302618112.0,
+ "1208": 302618112.0,
+ "1209": 302618112.0,
+ "1210": 302618112.0,
+ "1211": 302618112.0,
+ "1212": 302618112.0,
+ "1213": 302618112.0,
+ "1214": 302618112.0,
+ "1215": 302618112.0,
+ "1216": 302618112.0,
+ "1217": 302618112.0,
+ "1218": 302618112.0,
+ "1219": 302618112.0,
+ "1220": 302618112.0,
+ "1221": 302618112.0,
+ "1222": 302618112.0,
+ "1223": 302618112.0,
+ "1224": 302618112.0,
+ "1225": 302618112.0,
+ "1226": 302618112.0,
+ "1227": 302618112.0,
+ "1228": 302618112.0,
+ "1229": 302618112.0,
+ "1230": 302618112.0,
+ "1231": 302618112.0,
+ "1232": 302618112.0,
+ "1233": 302618112.0,
+ "1234": 302618112.0,
+ "1235": 302618112.0,
+ "1236": 302618112.0,
+ "1237": 302618112.0,
+ "1238": 302618112.0,
+ "1239": 302618112.0,
+ "1240": 302618112.0,
+ "1241": 302618112.0,
+ "1242": 302618112.0,
+ "1243": 302618112.0,
+ "1244": 302618112.0,
+ "1245": 302618112.0,
+ "1246": 302618112.0,
+ "1247": 302618112.0,
+ "1248": 302618112.0,
+ "1249": 302618112.0,
+ "1250": 302618112.0,
+ "1251": 302618112.0,
+ "1252": 302618112.0,
+ "1253": 302618112.0,
+ "1254": 302618112.0,
+ "1255": 302618112.0,
+ "1256": 302618112.0,
+ "1257": 302618112.0,
+ "1258": 302618112.0,
+ "1259": 302618112.0,
+ "1260": 302618112.0,
+ "1261": 302618112.0,
+ "1262": 302618112.0,
+ "1263": 302618112.0,
+ "1264": 302618112.0,
+ "1265": 302618112.0,
+ "1266": 302618112.0,
+ "1267": 302618112.0,
+ "1268": 302618112.0,
+ "1269": 302618112.0,
+ "1270": 302618112.0,
+ "1271": 302618112.0,
+ "1272": 302618112.0,
+ "1273": 302618112.0,
+ "1274": 302618112.0,
+ "1275": 302618112.0,
+ "1276": 302618112.0,
+ "1277": 302618112.0,
+ "1278": 302618112.0,
+ "1279": 302618112.0,
+ "1280": 302618112.0,
+ "1281": 302618112.0,
+ "1282": 302618112.0,
+ "1283": 302618112.0,
+ "1284": 302618112.0,
+ "1285": 302618112.0,
+ "1286": 302618112.0,
+ "1287": 302618112.0,
+ "1288": 302618112.0,
+ "1289": 302618112.0,
+ "1290": 302618112.0,
+ "1291": 302618112.0,
+ "1292": 302618112.0,
+ "1293": 302618112.0,
+ "1294": 302618112.0,
+ "1295": 302618112.0,
+ "1296": 302618112.0,
+ "1297": 302618112.0,
+ "1298": 302618112.0,
+ "1299": 302618112.0,
+ "1300": 302618112.0,
+ "1301": 302618112.0,
+ "1302": 302618112.0,
+ "1303": 302618112.0,
+ "1304": 302618112.0,
+ "1305": 302618112.0,
+ "1306": 302618112.0,
+ "1307": 302618112.0,
+ "1308": 302618112.0,
+ "1309": 302618112.0,
+ "1310": 302618112.0,
+ "1311": 302618112.0,
+ "1312": 302618112.0,
+ "1313": 302618112.0,
+ "1314": 302618112.0,
+ "1315": 302618112.0,
+ "1316": 302618112.0,
+ "1317": 302618112.0,
+ "1318": 302618112.0,
+ "1319": 302618112.0,
+ "1320": 302618112.0,
+ "1321": 302618112.0,
+ "1322": 302618112.0,
+ "1323": 302618112.0,
+ "1324": 302618112.0,
+ "1325": 302618112.0,
+ "1326": 302618112.0,
+ "1327": 302618112.0,
+ "1328": 302618112.0,
+ "1329": 302618112.0,
+ "1330": 302618112.0,
+ "1331": 302618112.0,
+ "1332": 302618112.0,
+ "1333": 302618112.0,
+ "1334": 302618112.0,
+ "1335": 302618112.0,
+ "1336": 302618112.0,
+ "1337": 302618112.0,
+ "1338": 302618112.0,
+ "1339": 302618112.0,
+ "1340": 302618112.0,
+ "1341": 302618112.0,
+ "1342": 302618112.0,
+ "1343": 302618112.0,
+ "1344": 302618112.0,
+ "1345": 302618112.0,
+ "1346": 302618112.0,
+ "1347": 302618112.0,
+ "1348": 302618112.0,
+ "1349": 302618112.0,
+ "1350": 302618112.0,
+ "1351": 302618112.0,
+ "1352": 302618112.0,
+ "1353": 302618112.0,
+ "1354": 302618112.0,
+ "1355": 302618112.0,
+ "1356": 302618112.0,
+ "1357": 302618112.0,
+ "1358": 302618112.0,
+ "1359": 302618112.0,
+ "1360": 302618112.0,
+ "1361": 302618112.0,
+ "1362": 302618112.0,
+ "1363": 302618112.0,
+ "1364": 302618112.0,
+ "1365": 302618112.0,
+ "1366": 302618112.0,
+ "1367": 302618112.0,
+ "1368": 302618112.0,
+ "1369": 302618112.0,
+ "1370": 302618112.0,
+ "1371": 302618112.0,
+ "1372": 302618112.0,
+ "1373": 302618112.0,
+ "1374": 302618112.0,
+ "1375": 302618112.0,
+ "1376": 302618112.0,
+ "1377": 302618112.0,
+ "1378": 302618112.0,
+ "1379": 302618112.0,
+ "1380": 302618112.0,
+ "1381": 302618112.0,
+ "1382": 302618112.0,
+ "1383": 302618112.0,
+ "1384": 302618112.0,
+ "1385": 302618112.0,
+ "1386": 302618112.0,
+ "1387": 302618112.0,
+ "1388": 302618112.0,
+ "1389": 302618112.0,
+ "1390": 302618112.0,
+ "1391": 302618112.0,
+ "1392": 302618112.0,
+ "1393": 302618112.0,
+ "1394": 302618112.0,
+ "1395": 302618112.0,
+ "1396": 302618112.0,
+ "1397": 302618112.0,
+ "1398": 302618112.0,
+ "1399": 302618112.0,
+ "1400": 302618112.0,
+ "1401": 302618112.0,
+ "1402": 302618112.0,
+ "1403": 302618112.0,
+ "1404": 302618112.0,
+ "1405": 302618112.0,
+ "1406": 302618112.0,
+ "1407": 302618112.0,
+ "1408": 302618112.0,
+ "1409": 302618112.0,
+ "1410": 302618112.0,
+ "1411": 302618112.0,
+ "1412": 302618112.0,
+ "1413": 302618112.0,
+ "1414": 302618112.0,
+ "1415": 302618112.0,
+ "1416": 302618112.0,
+ "1417": 302618112.0,
+ "1418": 302618112.0,
+ "1419": 302618112.0,
+ "1420": 302618112.0,
+ "1421": 302618112.0,
+ "1422": 302618112.0,
+ "1423": 302618112.0,
+ "1424": 302618112.0,
+ "1425": 302618112.0,
+ "1426": 302618112.0,
+ "1427": 302618112.0,
+ "1428": 302618112.0,
+ "1429": 302618112.0,
+ "1430": 302618112.0,
+ "1431": 302618112.0,
+ "1432": 302618112.0,
+ "1433": 302618112.0,
+ "1434": 302618112.0,
+ "1435": 302618112.0,
+ "1436": 302618112.0,
+ "1437": 302618112.0,
+ "1438": 302618112.0,
+ "1439": 302618112.0,
+ "1440": 302618112.0,
+ "1441": 302618112.0,
+ "1442": 302618112.0,
+ "1443": 302618112.0,
+ "1444": 302618112.0,
+ "1445": 302618112.0,
+ "1446": 302618112.0,
+ "1447": 302618112.0,
+ "1448": 302618112.0,
+ "1449": 302618112.0,
+ "1450": 302618112.0,
+ "1451": 302618112.0,
+ "1452": 302618112.0,
+ "1453": 302618112.0,
+ "1454": 302618112.0,
+ "1455": 302618112.0,
+ "1456": 302618112.0,
+ "1457": 302618112.0,
+ "1458": 302618112.0,
+ "1459": 302618112.0,
+ "1460": 302618112.0,
+ "1461": 302618112.0,
+ "1462": 302618112.0,
+ "1463": 302618112.0,
+ "1464": 302618112.0,
+ "1465": 302618112.0,
+ "1466": 302618112.0,
+ "1467": 302618112.0,
+ "1468": 302618112.0,
+ "1469": 302618112.0,
+ "1470": 302618112.0,
+ "1471": 302618112.0,
+ "1472": 302618112.0,
+ "1473": 302618112.0,
+ "1474": 302618112.0,
+ "1475": 302618112.0,
+ "1476": 302618112.0,
+ "1477": 302618112.0,
+ "1478": 302618112.0,
+ "1479": 302618112.0,
+ "1480": 302618112.0,
+ "1481": 302618112.0,
+ "1482": 302618112.0,
+ "1483": 302618112.0,
+ "1484": 302618112.0,
+ "1485": 302618112.0,
+ "1486": 302618112.0,
+ "1487": 302618112.0,
+ "1488": 302618112.0,
+ "1489": 302618112.0,
+ "1490": 302618112.0,
+ "1491": 302618112.0,
+ "1492": 302618112.0,
+ "1493": 302618112.0,
+ "1494": 302618112.0,
+ "1495": 302618112.0,
+ "1496": 302618112.0,
+ "1497": 302618112.0,
+ "1498": 302618112.0,
+ "1499": 302618112.0,
+ "1500": 302618112.0,
+ "1501": 302618112.0,
+ "1502": 302618112.0,
+ "1503": 302618112.0,
+ "1504": 302618112.0,
+ "1505": 302618112.0,
+ "1506": 302618112.0,
+ "1507": 302618112.0,
+ "1508": 302618112.0,
+ "1509": 302618112.0,
+ "1510": 302618112.0,
+ "1511": 302618112.0,
+ "1512": 302618112.0,
+ "1513": 302618112.0,
+ "1514": 302618112.0,
+ "1515": 302618112.0,
+ "1516": 302618112.0,
+ "1517": 302618112.0,
+ "1518": 302618112.0,
+ "1519": 302618112.0,
+ "1520": 302618112.0,
+ "1521": 302618112.0,
+ "1522": 302618112.0,
+ "1523": 302618112.0,
+ "1524": 302618112.0,
+ "1525": 302618112.0,
+ "1526": 302618112.0,
+ "1527": 302618112.0,
+ "1528": 302618112.0,
+ "1529": 302618112.0,
+ "1530": 302618112.0,
+ "1531": 302618112.0,
+ "1532": 302618112.0,
+ "1533": 302618112.0,
+ "1534": 302618112.0,
+ "1535": 302618112.0,
+ "1536": 302618112.0,
+ "1537": 302618112.0,
+ "1538": 302618112.0,
+ "1539": 302618112.0,
+ "1540": 302618112.0,
+ "1541": 302618112.0,
+ "1542": 302618112.0,
+ "1543": 302618112.0,
+ "1544": 302618112.0,
+ "1545": 302618112.0,
+ "1546": 302618112.0,
+ "1547": 302618112.0,
+ "1548": 302618112.0,
+ "1549": 302618112.0,
+ "1550": 302618112.0,
+ "1551": 302618112.0,
+ "1552": 302618112.0,
+ "1553": 302618112.0,
+ "1554": 302618112.0,
+ "1555": 302618112.0,
+ "1556": 302618112.0,
+ "1557": 302618112.0,
+ "1558": 302618112.0,
+ "1559": 302618112.0,
+ "1560": 302618112.0,
+ "1561": 302618112.0,
+ "1562": 302618112.0,
+ "1563": 302618112.0,
+ "1564": 302618112.0,
+ "1565": 302618112.0,
+ "1566": 302618112.0,
+ "1567": 302618112.0,
+ "1568": 302618112.0,
+ "1569": 302618112.0,
+ "1570": 302618112.0,
+ "1571": 302618112.0,
+ "1572": 302618112.0,
+ "1573": 302618112.0,
+ "1574": 302618112.0,
+ "1575": 302618112.0,
+ "1576": 302618112.0,
+ "1577": 302618112.0,
+ "1578": 302618112.0,
+ "1579": 302618112.0,
+ "1580": 302618112.0,
+ "1581": 302618112.0,
+ "1582": 302618112.0,
+ "1583": 302618112.0,
+ "1584": 302618112.0,
+ "1585": 302618112.0,
+ "1586": 302618112.0,
+ "1587": 302618112.0,
+ "1588": 302618112.0,
+ "1589": 302618112.0,
+ "1590": 302618112.0,
+ "1591": 302618112.0,
+ "1592": 302618112.0,
+ "1593": 302618112.0,
+ "1594": 302618112.0,
+ "1595": 302618112.0,
+ "1596": 302618112.0,
+ "1597": 302618112.0,
+ "1598": 302618112.0,
+ "1599": 302618112.0,
+ "1600": 302618112.0,
+ "1601": 302618112.0,
+ "1602": 302618112.0,
+ "1603": 302618112.0,
+ "1604": 302618112.0,
+ "1605": 302618112.0,
+ "1606": 302618112.0,
+ "1607": 302618112.0,
+ "1608": 302618112.0,
+ "1609": 302618112.0,
+ "1610": 302618112.0,
+ "1611": 302618112.0,
+ "1612": 302618112.0,
+ "1613": 302618112.0,
+ "1614": 302618112.0,
+ "1615": 302618112.0,
+ "1616": 302618112.0,
+ "1617": 302618112.0,
+ "1618": 302618112.0,
+ "1619": 302618112.0,
+ "1620": 302618112.0,
+ "1621": 302618112.0,
+ "1622": 302618112.0,
+ "1623": 302618112.0,
+ "1624": 302618112.0,
+ "1625": 302618112.0,
+ "1626": 302618112.0,
+ "1627": 302618112.0,
+ "1628": 302618112.0,
+ "1629": 302618112.0,
+ "1630": 302618112.0,
+ "1631": 302618112.0,
+ "1632": 302618112.0,
+ "1633": 302618112.0,
+ "1634": 302618112.0,
+ "1635": 302618112.0,
+ "1636": 302618112.0,
+ "1637": 302618112.0,
+ "1638": 302618112.0,
+ "1639": 302618112.0,
+ "1640": 302618112.0,
+ "1641": 302618112.0,
+ "1642": 302618112.0,
+ "1643": 302618112.0,
+ "1644": 302618112.0,
+ "1645": 302618112.0,
+ "1646": 302618112.0,
+ "1647": 302618112.0,
+ "1648": 302618112.0,
+ "1649": 302618112.0,
+ "1650": 302618112.0,
+ "1651": 302618112.0,
+ "1652": 302618112.0,
+ "1653": 302618112.0,
+ "1654": 302618112.0,
+ "1655": 302618112.0,
+ "1656": 302618112.0,
+ "1657": 302618112.0,
+ "1658": 302618112.0,
+ "1659": 302618112.0,
+ "1660": 302618112.0,
+ "1661": 302618112.0,
+ "1662": 302618112.0,
+ "1663": 302618112.0,
+ "1664": 302618112.0,
+ "1665": 302618112.0,
+ "1666": 302618112.0,
+ "1667": 302618112.0,
+ "1668": 302618112.0,
+ "1669": 302618112.0,
+ "1670": 302618112.0,
+ "1671": 302618112.0,
+ "1672": 302618112.0,
+ "1673": 302618112.0,
+ "1674": 302618112.0,
+ "1675": 302618112.0,
+ "1676": 302618112.0,
+ "1677": 302618112.0,
+ "1678": 302618112.0,
+ "1679": 302618112.0,
+ "1680": 302618112.0,
+ "1681": 302618112.0,
+ "1682": 302618112.0,
+ "1683": 302618112.0,
+ "1684": 302618112.0,
+ "1685": 302618112.0,
+ "1686": 302618112.0,
+ "1687": 302618112.0,
+ "1688": 302618112.0,
+ "1689": 302618112.0,
+ "1690": 302618112.0,
+ "1691": 302618112.0,
+ "1692": 302618112.0,
+ "1693": 302618112.0,
+ "1694": 302618112.0,
+ "1695": 302618112.0,
+ "1696": 302618112.0,
+ "1697": 302618112.0,
+ "1698": 302618112.0,
+ "1699": 302618112.0,
+ "1700": 302618112.0,
+ "1701": 302618112.0,
+ "1702": 302618112.0,
+ "1703": 302618112.0,
+ "1704": 302618112.0,
+ "1705": 302618112.0,
+ "1706": 302618112.0,
+ "1707": 302618112.0,
+ "1708": 302618112.0,
+ "1709": 302618112.0,
+ "1710": 302618112.0,
+ "1711": 302618112.0,
+ "1712": 302618112.0,
+ "1713": 302618112.0,
+ "1714": 302618112.0,
+ "1715": 302618112.0,
+ "1716": 302618112.0,
+ "1717": 302618112.0,
+ "1718": 302618112.0,
+ "1719": 302618112.0,
+ "1720": 302618112.0,
+ "1721": 302618112.0,
+ "1722": 302618112.0,
+ "1723": 302618112.0,
+ "1724": 302618112.0,
+ "1725": 302618112.0,
+ "1726": 302618112.0,
+ "1727": 302618112.0,
+ "1728": 302618112.0,
+ "1729": 302618112.0,
+ "1730": 302618112.0,
+ "1731": 302618112.0,
+ "1732": 302618112.0,
+ "1733": 302618112.0,
+ "1734": 302618112.0,
+ "1735": 302618112.0,
+ "1736": 302618112.0,
+ "1737": 302618112.0,
+ "1738": 302618112.0,
+ "1739": 302618112.0,
+ "1740": 302618112.0,
+ "1741": 302618112.0,
+ "1742": 302618112.0,
+ "1743": 302618112.0,
+ "1744": 302618112.0,
+ "1745": 302618112.0,
+ "1746": 302618112.0,
+ "1747": 302618112.0,
+ "1748": 302618112.0,
+ "1749": 302618112.0,
+ "1750": 302618112.0,
+ "1751": 302618112.0,
+ "1752": 302618112.0,
+ "1753": 302618112.0,
+ "1754": 302618112.0,
+ "1755": 302618112.0,
+ "1756": 302618112.0,
+ "1757": 302618112.0,
+ "1758": 302618112.0,
+ "1759": 302618112.0,
+ "1760": 302618112.0,
+ "1761": 302618112.0,
+ "1762": 302618112.0,
+ "1763": 302618112.0,
+ "1764": 302618112.0,
+ "1765": 302618112.0,
+ "1766": 302618112.0,
+ "1767": 302618112.0,
+ "1768": 302618112.0,
+ "1769": 302618112.0,
+ "1770": 302618112.0,
+ "1771": 302618112.0,
+ "1772": 302618112.0,
+ "1773": 302618112.0,
+ "1774": 302618112.0,
+ "1775": 302618112.0,
+ "1776": 302618112.0,
+ "1777": 302618112.0,
+ "1778": 302618112.0,
+ "1779": 302618112.0,
+ "1780": 302618112.0,
+ "1781": 302618112.0,
+ "1782": 302618112.0,
+ "1783": 302618112.0,
+ "1784": 302618112.0,
+ "1785": 302618112.0,
+ "1786": 302618112.0,
+ "1787": 302618112.0,
+ "1788": 302618112.0,
+ "1789": 302618112.0,
+ "1790": 302618112.0,
+ "1791": 302618112.0,
+ "1792": 302618112.0,
+ "1793": 302618112.0,
+ "1794": 302618112.0,
+ "1795": 302618112.0,
+ "1796": 302618112.0,
+ "1797": 302618112.0,
+ "1798": 302618112.0,
+ "1799": 302618112.0,
+ "1800": 302618112.0,
+ "1801": 302618112.0,
+ "1802": 302618112.0,
+ "1803": 302618112.0,
+ "1804": 302618112.0,
+ "1805": 302618112.0,
+ "1806": 302618112.0,
+ "1807": 302618112.0,
+ "1808": 302618112.0,
+ "1809": 302618112.0,
+ "1810": 302618112.0,
+ "1811": 302618112.0,
+ "1812": 302618112.0,
+ "1813": 302618112.0,
+ "1814": 302618112.0,
+ "1815": 302618112.0,
+ "1816": 302618112.0,
+ "1817": 302618112.0,
+ "1818": 302618112.0,
+ "1819": 302618112.0,
+ "1820": 302618112.0,
+ "1821": 302618112.0,
+ "1822": 302618112.0,
+ "1823": 302618112.0,
+ "1824": 302618112.0,
+ "1825": 302618112.0,
+ "1826": 302618112.0,
+ "1827": 302618112.0,
+ "1828": 302618112.0,
+ "1829": 302618112.0,
+ "1830": 302618112.0,
+ "1831": 302618112.0,
+ "1832": 302618112.0,
+ "1833": 302618112.0,
+ "1834": 302618112.0,
+ "1835": 302618112.0,
+ "1836": 302618112.0,
+ "1837": 302618112.0,
+ "1838": 302618112.0,
+ "1839": 302618112.0,
+ "1840": 302618112.0,
+ "1841": 302618112.0,
+ "1842": 302618112.0,
+ "1843": 302618112.0,
+ "1844": 302618112.0,
+ "1845": 302618112.0,
+ "1846": 302618112.0,
+ "1847": 302618112.0,
+ "1848": 302618112.0,
+ "1849": 302618112.0,
+ "1850": 302618112.0,
+ "1851": 302618112.0,
+ "1852": 302618112.0,
+ "1853": 302618112.0,
+ "1854": 302618112.0,
+ "1855": 302618112.0,
+ "1856": 302618112.0,
+ "1857": 302618112.0,
+ "1858": 302618112.0,
+ "1859": 302618112.0,
+ "1860": 302618112.0,
+ "1861": 302618112.0,
+ "1862": 302618112.0,
+ "1863": 302618112.0,
+ "1864": 302618112.0,
+ "1865": 302618112.0,
+ "1866": 302618112.0,
+ "1867": 302618112.0,
+ "1868": 302618112.0,
+ "1869": 302618112.0,
+ "1870": 302618112.0,
+ "1871": 302618112.0,
+ "1872": 302618112.0,
+ "1873": 302618112.0,
+ "1874": 302618112.0,
+ "1875": 302618112.0,
+ "1876": 302618112.0,
+ "1877": 302618112.0,
+ "1878": 302618112.0,
+ "1879": 302618112.0,
+ "1880": 302618112.0,
+ "1881": 302618112.0,
+ "1882": 302618112.0,
+ "1883": 302618112.0,
+ "1884": 302618112.0,
+ "1885": 302618112.0,
+ "1886": 302618112.0,
+ "1887": 302618112.0,
+ "1888": 302618112.0,
+ "1889": 302618112.0,
+ "1890": 302618112.0,
+ "1891": 302618112.0,
+ "1892": 302618112.0,
+ "1893": 302618112.0,
+ "1894": 302618112.0,
+ "1895": 302618112.0,
+ "1896": 302618112.0,
+ "1897": 302618112.0,
+ "1898": 302618112.0,
+ "1899": 302618112.0,
+ "1900": 302618112.0,
+ "1901": 302618112.0,
+ "1902": 302618112.0,
+ "1903": 302618112.0,
+ "1904": 302618112.0,
+ "1905": 302618112.0,
+ "1906": 302618112.0,
+ "1907": 302618112.0,
+ "1908": 302618112.0,
+ "1909": 302618112.0,
+ "1910": 302618112.0,
+ "1911": 302618112.0,
+ "1912": 302618112.0,
+ "1913": 302618112.0,
+ "1914": 302618112.0,
+ "1915": 302618112.0,
+ "1916": 302618112.0,
+ "1917": 302618112.0,
+ "1918": 302618112.0,
+ "1919": 302618112.0,
+ "1920": 302618112.0,
+ "1921": 302618112.0,
+ "1922": 302618112.0,
+ "1923": 302618112.0,
+ "1924": 302618112.0,
+ "1925": 302618112.0,
+ "1926": 302618112.0,
+ "1927": 302618112.0,
+ "1928": 302618112.0,
+ "1929": 302618112.0,
+ "1930": 302618112.0,
+ "1931": 302618112.0,
+ "1932": 302618112.0,
+ "1933": 302618112.0,
+ "1934": 302618112.0,
+ "1935": 302618112.0,
+ "1936": 302618112.0,
+ "1937": 302618112.0,
+ "1938": 302618112.0,
+ "1939": 302618112.0,
+ "1940": 302618112.0,
+ "1941": 302618112.0,
+ "1942": 302618112.0,
+ "1943": 302618112.0,
+ "1944": 302618112.0,
+ "1945": 302618112.0,
+ "1946": 302618112.0,
+ "1947": 302618112.0,
+ "1948": 302618112.0,
+ "1949": 302618112.0,
+ "1950": 302618112.0,
+ "1951": 302618112.0,
+ "1952": 302618112.0,
+ "1953": 302618112.0,
+ "1954": 302618112.0,
+ "1955": 302618112.0,
+ "1956": 302618112.0,
+ "1957": 302618112.0,
+ "1958": 302618112.0,
+ "1959": 302618112.0,
+ "1960": 302618112.0,
+ "1961": 302618112.0,
+ "1962": 302618112.0,
+ "1963": 302618112.0,
+ "1964": 302618112.0,
+ "1965": 302618112.0,
+ "1966": 302618112.0,
+ "1967": 302618112.0,
+ "1968": 302618112.0,
+ "1969": 302618112.0,
+ "1970": 302618112.0,
+ "1971": 302618112.0,
+ "1972": 302618112.0,
+ "1973": 302618112.0,
+ "1974": 302618112.0,
+ "1975": 302618112.0,
+ "1976": 302618112.0,
+ "1977": 302618112.0,
+ "1978": 302618112.0,
+ "1979": 302618112.0,
+ "1980": 302618112.0,
+ "1981": 302618112.0,
+ "1982": 302618112.0,
+ "1983": 302618112.0,
+ "1984": 302618112.0,
+ "1985": 302618112.0,
+ "1986": 302618112.0,
+ "1987": 302618112.0,
+ "1988": 302618112.0,
+ "1989": 302618112.0,
+ "1990": 302618112.0,
+ "1991": 302618112.0,
+ "1992": 302618112.0,
+ "1993": 302618112.0,
+ "1994": 302618112.0,
+ "1995": 302618112.0,
+ "1996": 302618112.0,
+ "1997": 302618112.0,
+ "1998": 302618112.0,
+ "1999": 302618112.0,
+ "2000": 302618112.0
+ }
+ },
+ "mem-max-allocated-bytes": {
+ "start_step": 1,
+ "end_step": 2000,
+ "step_interval": 1,
+ "values": {
+ "1": 362060288.0,
+ "2": 428612096.0,
+ "3": 428612096.0,
+ "4": 428612096.0,
+ "5": 428612096.0,
+ "6": 428612096.0,
+ "7": 428612096.0,
+ "8": 428612096.0,
+ "9": 428612096.0,
+ "10": 428612096.0,
+ "11": 428612096.0,
+ "12": 428612096.0,
+ "13": 428612096.0,
+ "14": 428612096.0,
+ "15": 428612096.0,
+ "16": 428612096.0,
+ "17": 428612096.0,
+ "18": 428612096.0,
+ "19": 428612096.0,
+ "20": 428612096.0,
+ "21": 428612096.0,
+ "22": 428612096.0,
+ "23": 428612096.0,
+ "24": 428612096.0,
+ "25": 428612096.0,
+ "26": 428612096.0,
+ "27": 428612096.0,
+ "28": 428612096.0,
+ "29": 428612096.0,
+ "30": 428612096.0,
+ "31": 428612096.0,
+ "32": 428612096.0,
+ "33": 428612096.0,
+ "34": 428612096.0,
+ "35": 428612096.0,
+ "36": 428612096.0,
+ "37": 428612096.0,
+ "38": 428612096.0,
+ "39": 428612096.0,
+ "40": 428612096.0,
+ "41": 428612096.0,
+ "42": 428612096.0,
+ "43": 428612096.0,
+ "44": 428612096.0,
+ "45": 428612096.0,
+ "46": 428612096.0,
+ "47": 428612096.0,
+ "48": 428612096.0,
+ "49": 428612096.0,
+ "50": 428612096.0,
+ "51": 428612096.0,
+ "52": 428612096.0,
+ "53": 428612096.0,
+ "54": 428612096.0,
+ "55": 428612096.0,
+ "56": 428612096.0,
+ "57": 428612096.0,
+ "58": 428612096.0,
+ "59": 428612096.0,
+ "60": 428612096.0,
+ "61": 428612096.0,
+ "62": 428612096.0,
+ "63": 428612096.0,
+ "64": 428612096.0,
+ "65": 428612096.0,
+ "66": 428612096.0,
+ "67": 428612096.0,
+ "68": 428612096.0,
+ "69": 428612096.0,
+ "70": 428612096.0,
+ "71": 428612096.0,
+ "72": 428612096.0,
+ "73": 428612096.0,
+ "74": 428612096.0,
+ "75": 428612096.0,
+ "76": 428612096.0,
+ "77": 428612096.0,
+ "78": 428612096.0,
+ "79": 428612096.0,
+ "80": 428612096.0,
+ "81": 428612096.0,
+ "82": 428612096.0,
+ "83": 428612096.0,
+ "84": 428612096.0,
+ "85": 428612096.0,
+ "86": 428612096.0,
+ "87": 428612096.0,
+ "88": 428612096.0,
+ "89": 428612096.0,
+ "90": 428612096.0,
+ "91": 428612096.0,
+ "92": 428612096.0,
+ "93": 428612096.0,
+ "94": 428612096.0,
+ "95": 428612096.0,
+ "96": 428612096.0,
+ "97": 428612096.0,
+ "98": 428612096.0,
+ "99": 428612096.0,
+ "100": 428612096.0,
+ "101": 428612096.0,
+ "102": 428612096.0,
+ "103": 428612096.0,
+ "104": 428612096.0,
+ "105": 428612096.0,
+ "106": 428612096.0,
+ "107": 428612096.0,
+ "108": 428612096.0,
+ "109": 428612096.0,
+ "110": 428612096.0,
+ "111": 428612096.0,
+ "112": 428612096.0,
+ "113": 428612096.0,
+ "114": 428612096.0,
+ "115": 428612096.0,
+ "116": 428612096.0,
+ "117": 428612096.0,
+ "118": 428612096.0,
+ "119": 428612096.0,
+ "120": 428612096.0,
+ "121": 428612096.0,
+ "122": 428612096.0,
+ "123": 428612096.0,
+ "124": 428612096.0,
+ "125": 428612096.0,
+ "126": 428612096.0,
+ "127": 428612096.0,
+ "128": 428612096.0,
+ "129": 428612096.0,
+ "130": 428612096.0,
+ "131": 428612096.0,
+ "132": 428612096.0,
+ "133": 428612096.0,
+ "134": 428612096.0,
+ "135": 428612096.0,
+ "136": 428612096.0,
+ "137": 428612096.0,
+ "138": 428612096.0,
+ "139": 428612096.0,
+ "140": 428612096.0,
+ "141": 428612096.0,
+ "142": 428612096.0,
+ "143": 428612096.0,
+ "144": 428612096.0,
+ "145": 428612096.0,
+ "146": 428612096.0,
+ "147": 428612096.0,
+ "148": 428612096.0,
+ "149": 428612096.0,
+ "150": 428612096.0,
+ "151": 428612096.0,
+ "152": 428612096.0,
+ "153": 428612096.0,
+ "154": 428612096.0,
+ "155": 428612096.0,
+ "156": 428612096.0,
+ "157": 428612096.0,
+ "158": 428612096.0,
+ "159": 428612096.0,
+ "160": 428612096.0,
+ "161": 428612096.0,
+ "162": 428612096.0,
+ "163": 428612096.0,
+ "164": 428612096.0,
+ "165": 428612096.0,
+ "166": 428612096.0,
+ "167": 428612096.0,
+ "168": 428612096.0,
+ "169": 428612096.0,
+ "170": 428612096.0,
+ "171": 428612096.0,
+ "172": 428612096.0,
+ "173": 428612096.0,
+ "174": 428612096.0,
+ "175": 428612096.0,
+ "176": 428612096.0,
+ "177": 428612096.0,
+ "178": 428612096.0,
+ "179": 428612096.0,
+ "180": 428612096.0,
+ "181": 428612096.0,
+ "182": 428612096.0,
+ "183": 428612096.0,
+ "184": 428612096.0,
+ "185": 428612096.0,
+ "186": 428612096.0,
+ "187": 428612096.0,
+ "188": 428612096.0,
+ "189": 428612096.0,
+ "190": 428612096.0,
+ "191": 428612096.0,
+ "192": 428612096.0,
+ "193": 428612096.0,
+ "194": 428612096.0,
+ "195": 428612096.0,
+ "196": 428612096.0,
+ "197": 428612096.0,
+ "198": 428612096.0,
+ "199": 428612096.0,
+ "200": 428612096.0,
+ "201": 428612096.0,
+ "202": 428612096.0,
+ "203": 428612096.0,
+ "204": 428612096.0,
+ "205": 428612096.0,
+ "206": 428612096.0,
+ "207": 428612096.0,
+ "208": 428612096.0,
+ "209": 428612096.0,
+ "210": 428612096.0,
+ "211": 428612096.0,
+ "212": 428612096.0,
+ "213": 428612096.0,
+ "214": 428612096.0,
+ "215": 428612096.0,
+ "216": 428612096.0,
+ "217": 428612096.0,
+ "218": 428612096.0,
+ "219": 428612096.0,
+ "220": 428612096.0,
+ "221": 428612096.0,
+ "222": 428612096.0,
+ "223": 428612096.0,
+ "224": 428612096.0,
+ "225": 428612096.0,
+ "226": 428612096.0,
+ "227": 428612096.0,
+ "228": 428612096.0,
+ "229": 428612096.0,
+ "230": 428612096.0,
+ "231": 428612096.0,
+ "232": 428612096.0,
+ "233": 428612096.0,
+ "234": 428612096.0,
+ "235": 428612096.0,
+ "236": 428612096.0,
+ "237": 428612096.0,
+ "238": 428612096.0,
+ "239": 428612096.0,
+ "240": 428612096.0,
+ "241": 428612096.0,
+ "242": 428612096.0,
+ "243": 428612096.0,
+ "244": 428612096.0,
+ "245": 428612096.0,
+ "246": 428612096.0,
+ "247": 428612096.0,
+ "248": 428612096.0,
+ "249": 428612096.0,
+ "250": 428612096.0,
+ "251": 428612096.0,
+ "252": 428612096.0,
+ "253": 428612096.0,
+ "254": 428612096.0,
+ "255": 428612096.0,
+ "256": 428612096.0,
+ "257": 428612096.0,
+ "258": 428612096.0,
+ "259": 428612096.0,
+ "260": 428612096.0,
+ "261": 428612096.0,
+ "262": 428612096.0,
+ "263": 428612096.0,
+ "264": 428612096.0,
+ "265": 428612096.0,
+ "266": 428612096.0,
+ "267": 428612096.0,
+ "268": 428612096.0,
+ "269": 428612096.0,
+ "270": 428612096.0,
+ "271": 428612096.0,
+ "272": 428612096.0,
+ "273": 428612096.0,
+ "274": 428612096.0,
+ "275": 428612096.0,
+ "276": 428612096.0,
+ "277": 428612096.0,
+ "278": 428612096.0,
+ "279": 428612096.0,
+ "280": 428612096.0,
+ "281": 428612096.0,
+ "282": 428612096.0,
+ "283": 428612096.0,
+ "284": 428612096.0,
+ "285": 428612096.0,
+ "286": 428612096.0,
+ "287": 428612096.0,
+ "288": 428612096.0,
+ "289": 428612096.0,
+ "290": 428612096.0,
+ "291": 428612096.0,
+ "292": 428612096.0,
+ "293": 428612096.0,
+ "294": 428612096.0,
+ "295": 428612096.0,
+ "296": 428612096.0,
+ "297": 428612096.0,
+ "298": 428612096.0,
+ "299": 428612096.0,
+ "300": 428612096.0,
+ "301": 428612096.0,
+ "302": 428612096.0,
+ "303": 428612096.0,
+ "304": 428612096.0,
+ "305": 428612096.0,
+ "306": 428612096.0,
+ "307": 428612096.0,
+ "308": 428612096.0,
+ "309": 428612096.0,
+ "310": 428612096.0,
+ "311": 428612096.0,
+ "312": 428612096.0,
+ "313": 428612096.0,
+ "314": 428612096.0,
+ "315": 428612096.0,
+ "316": 428612096.0,
+ "317": 428612096.0,
+ "318": 428612096.0,
+ "319": 428612096.0,
+ "320": 428612096.0,
+ "321": 428612096.0,
+ "322": 428612096.0,
+ "323": 428612096.0,
+ "324": 428612096.0,
+ "325": 428612096.0,
+ "326": 428612096.0,
+ "327": 428612096.0,
+ "328": 428612096.0,
+ "329": 428612096.0,
+ "330": 428612096.0,
+ "331": 428612096.0,
+ "332": 428612096.0,
+ "333": 428612096.0,
+ "334": 428612096.0,
+ "335": 428612096.0,
+ "336": 428612096.0,
+ "337": 428612096.0,
+ "338": 428612096.0,
+ "339": 428612096.0,
+ "340": 428612096.0,
+ "341": 428612096.0,
+ "342": 428612096.0,
+ "343": 428612096.0,
+ "344": 428612096.0,
+ "345": 428612096.0,
+ "346": 428612096.0,
+ "347": 428612096.0,
+ "348": 428612096.0,
+ "349": 428612096.0,
+ "350": 428612096.0,
+ "351": 428612096.0,
+ "352": 428612096.0,
+ "353": 428612096.0,
+ "354": 428612096.0,
+ "355": 428612096.0,
+ "356": 428612096.0,
+ "357": 428612096.0,
+ "358": 428612096.0,
+ "359": 428612096.0,
+ "360": 428612096.0,
+ "361": 428612096.0,
+ "362": 428612096.0,
+ "363": 428612096.0,
+ "364": 428612096.0,
+ "365": 428612096.0,
+ "366": 428612096.0,
+ "367": 428612096.0,
+ "368": 428612096.0,
+ "369": 428612096.0,
+ "370": 428612096.0,
+ "371": 428612096.0,
+ "372": 428612096.0,
+ "373": 428612096.0,
+ "374": 428612096.0,
+ "375": 428612096.0,
+ "376": 428612096.0,
+ "377": 428612096.0,
+ "378": 428612096.0,
+ "379": 428612096.0,
+ "380": 428612096.0,
+ "381": 428612096.0,
+ "382": 428612096.0,
+ "383": 428612096.0,
+ "384": 428612096.0,
+ "385": 428612096.0,
+ "386": 428612096.0,
+ "387": 428612096.0,
+ "388": 428612096.0,
+ "389": 428612096.0,
+ "390": 428612096.0,
+ "391": 428612096.0,
+ "392": 428612096.0,
+ "393": 428612096.0,
+ "394": 428612096.0,
+ "395": 428612096.0,
+ "396": 428612096.0,
+ "397": 428612096.0,
+ "398": 428612096.0,
+ "399": 428612096.0,
+ "400": 428612096.0,
+ "401": 428612096.0,
+ "402": 428612096.0,
+ "403": 428612096.0,
+ "404": 428612096.0,
+ "405": 428612096.0,
+ "406": 428612096.0,
+ "407": 428612096.0,
+ "408": 428612096.0,
+ "409": 428612096.0,
+ "410": 428612096.0,
+ "411": 428612096.0,
+ "412": 428612096.0,
+ "413": 428612096.0,
+ "414": 428612096.0,
+ "415": 428612096.0,
+ "416": 428612096.0,
+ "417": 428612096.0,
+ "418": 428612096.0,
+ "419": 428612096.0,
+ "420": 428612096.0,
+ "421": 428612096.0,
+ "422": 428612096.0,
+ "423": 428612096.0,
+ "424": 428612096.0,
+ "425": 428612096.0,
+ "426": 428612096.0,
+ "427": 428612096.0,
+ "428": 428612096.0,
+ "429": 428612096.0,
+ "430": 428612096.0,
+ "431": 428612096.0,
+ "432": 428612096.0,
+ "433": 428612096.0,
+ "434": 428612096.0,
+ "435": 428612096.0,
+ "436": 428612096.0,
+ "437": 428612096.0,
+ "438": 428612096.0,
+ "439": 428612096.0,
+ "440": 428612096.0,
+ "441": 428612096.0,
+ "442": 428612096.0,
+ "443": 428612096.0,
+ "444": 428612096.0,
+ "445": 428612096.0,
+ "446": 428612096.0,
+ "447": 428612096.0,
+ "448": 428612096.0,
+ "449": 428612096.0,
+ "450": 428612096.0,
+ "451": 428612096.0,
+ "452": 428612096.0,
+ "453": 428612096.0,
+ "454": 428612096.0,
+ "455": 428612096.0,
+ "456": 428612096.0,
+ "457": 428612096.0,
+ "458": 428612096.0,
+ "459": 428612096.0,
+ "460": 428612096.0,
+ "461": 428612096.0,
+ "462": 428612096.0,
+ "463": 428612096.0,
+ "464": 428612096.0,
+ "465": 428612096.0,
+ "466": 428612096.0,
+ "467": 428612096.0,
+ "468": 428612096.0,
+ "469": 428612096.0,
+ "470": 428612096.0,
+ "471": 428612096.0,
+ "472": 428612096.0,
+ "473": 428612096.0,
+ "474": 428612096.0,
+ "475": 428612096.0,
+ "476": 428612096.0,
+ "477": 428612096.0,
+ "478": 428612096.0,
+ "479": 428612096.0,
+ "480": 428612096.0,
+ "481": 428612096.0,
+ "482": 428612096.0,
+ "483": 428612096.0,
+ "484": 428612096.0,
+ "485": 428612096.0,
+ "486": 428612096.0,
+ "487": 428612096.0,
+ "488": 428612096.0,
+ "489": 428612096.0,
+ "490": 428612096.0,
+ "491": 428612096.0,
+ "492": 428612096.0,
+ "493": 428612096.0,
+ "494": 428612096.0,
+ "495": 428612096.0,
+ "496": 428612096.0,
+ "497": 428612096.0,
+ "498": 428612096.0,
+ "499": 428612096.0,
+ "500": 428612096.0,
+ "501": 428612096.0,
+ "502": 428612096.0,
+ "503": 428612096.0,
+ "504": 428612096.0,
+ "505": 428612096.0,
+ "506": 428612096.0,
+ "507": 428612096.0,
+ "508": 428612096.0,
+ "509": 428612096.0,
+ "510": 428612096.0,
+ "511": 428612096.0,
+ "512": 428612096.0,
+ "513": 428612096.0,
+ "514": 428612096.0,
+ "515": 428612096.0,
+ "516": 428612096.0,
+ "517": 428612096.0,
+ "518": 428612096.0,
+ "519": 428612096.0,
+ "520": 428612096.0,
+ "521": 428612096.0,
+ "522": 428612096.0,
+ "523": 428612096.0,
+ "524": 428612096.0,
+ "525": 428612096.0,
+ "526": 428612096.0,
+ "527": 428612096.0,
+ "528": 428612096.0,
+ "529": 428612096.0,
+ "530": 428612096.0,
+ "531": 428612096.0,
+ "532": 428612096.0,
+ "533": 428612096.0,
+ "534": 428612096.0,
+ "535": 428612096.0,
+ "536": 428612096.0,
+ "537": 428612096.0,
+ "538": 428612096.0,
+ "539": 428612096.0,
+ "540": 428612096.0,
+ "541": 428612096.0,
+ "542": 428612096.0,
+ "543": 428612096.0,
+ "544": 428612096.0,
+ "545": 428612096.0,
+ "546": 428612096.0,
+ "547": 428612096.0,
+ "548": 428612096.0,
+ "549": 428612096.0,
+ "550": 428612096.0,
+ "551": 428612096.0,
+ "552": 428612096.0,
+ "553": 428612096.0,
+ "554": 428612096.0,
+ "555": 428612096.0,
+ "556": 428612096.0,
+ "557": 428612096.0,
+ "558": 428612096.0,
+ "559": 428612096.0,
+ "560": 428612096.0,
+ "561": 428612096.0,
+ "562": 428612096.0,
+ "563": 428612096.0,
+ "564": 428612096.0,
+ "565": 428612096.0,
+ "566": 428612096.0,
+ "567": 428612096.0,
+ "568": 428612096.0,
+ "569": 428612096.0,
+ "570": 428612096.0,
+ "571": 428612096.0,
+ "572": 428612096.0,
+ "573": 428612096.0,
+ "574": 428612096.0,
+ "575": 428612096.0,
+ "576": 428612096.0,
+ "577": 428612096.0,
+ "578": 428612096.0,
+ "579": 428612096.0,
+ "580": 428612096.0,
+ "581": 428612096.0,
+ "582": 428612096.0,
+ "583": 428612096.0,
+ "584": 428612096.0,
+ "585": 428612096.0,
+ "586": 428612096.0,
+ "587": 428612096.0,
+ "588": 428612096.0,
+ "589": 428612096.0,
+ "590": 428612096.0,
+ "591": 428612096.0,
+ "592": 428612096.0,
+ "593": 428612096.0,
+ "594": 428612096.0,
+ "595": 428612096.0,
+ "596": 428612096.0,
+ "597": 428612096.0,
+ "598": 428612096.0,
+ "599": 428612096.0,
+ "600": 428612096.0,
+ "601": 428612096.0,
+ "602": 428612096.0,
+ "603": 428612096.0,
+ "604": 428612096.0,
+ "605": 428612096.0,
+ "606": 428612096.0,
+ "607": 428612096.0,
+ "608": 428612096.0,
+ "609": 428612096.0,
+ "610": 428612096.0,
+ "611": 428612096.0,
+ "612": 428612096.0,
+ "613": 428612096.0,
+ "614": 428612096.0,
+ "615": 428612096.0,
+ "616": 428612096.0,
+ "617": 428612096.0,
+ "618": 428612096.0,
+ "619": 428612096.0,
+ "620": 428612096.0,
+ "621": 428612096.0,
+ "622": 428612096.0,
+ "623": 428612096.0,
+ "624": 428612096.0,
+ "625": 428612096.0,
+ "626": 428612096.0,
+ "627": 428612096.0,
+ "628": 428612096.0,
+ "629": 428612096.0,
+ "630": 428612096.0,
+ "631": 428612096.0,
+ "632": 428612096.0,
+ "633": 428612096.0,
+ "634": 428612096.0,
+ "635": 428612096.0,
+ "636": 428612096.0,
+ "637": 428612096.0,
+ "638": 428612096.0,
+ "639": 428612096.0,
+ "640": 428612096.0,
+ "641": 428612096.0,
+ "642": 428612096.0,
+ "643": 428612096.0,
+ "644": 428612096.0,
+ "645": 428612096.0,
+ "646": 428612096.0,
+ "647": 428612096.0,
+ "648": 428612096.0,
+ "649": 428612096.0,
+ "650": 428612096.0,
+ "651": 428612096.0,
+ "652": 428612096.0,
+ "653": 428612096.0,
+ "654": 428612096.0,
+ "655": 428612096.0,
+ "656": 428612096.0,
+ "657": 428612096.0,
+ "658": 428612096.0,
+ "659": 428612096.0,
+ "660": 428612096.0,
+ "661": 428612096.0,
+ "662": 428612096.0,
+ "663": 428612096.0,
+ "664": 428612096.0,
+ "665": 428612096.0,
+ "666": 428612096.0,
+ "667": 428612096.0,
+ "668": 428612096.0,
+ "669": 428612096.0,
+ "670": 428612096.0,
+ "671": 428612096.0,
+ "672": 428612096.0,
+ "673": 428612096.0,
+ "674": 428612096.0,
+ "675": 428612096.0,
+ "676": 428612096.0,
+ "677": 428612096.0,
+ "678": 428612096.0,
+ "679": 428612096.0,
+ "680": 428612096.0,
+ "681": 428612096.0,
+ "682": 428612096.0,
+ "683": 428612096.0,
+ "684": 428612096.0,
+ "685": 428612096.0,
+ "686": 428612096.0,
+ "687": 428612096.0,
+ "688": 428612096.0,
+ "689": 428612096.0,
+ "690": 428612096.0,
+ "691": 428612096.0,
+ "692": 428612096.0,
+ "693": 428612096.0,
+ "694": 428612096.0,
+ "695": 428612096.0,
+ "696": 428612096.0,
+ "697": 428612096.0,
+ "698": 428612096.0,
+ "699": 428612096.0,
+ "700": 428612096.0,
+ "701": 428612096.0,
+ "702": 428612096.0,
+ "703": 428612096.0,
+ "704": 428612096.0,
+ "705": 428612096.0,
+ "706": 428612096.0,
+ "707": 428612096.0,
+ "708": 428612096.0,
+ "709": 428612096.0,
+ "710": 428612096.0,
+ "711": 428612096.0,
+ "712": 428612096.0,
+ "713": 428612096.0,
+ "714": 428612096.0,
+ "715": 428612096.0,
+ "716": 428612096.0,
+ "717": 428612096.0,
+ "718": 428612096.0,
+ "719": 428612096.0,
+ "720": 428612096.0,
+ "721": 428612096.0,
+ "722": 428612096.0,
+ "723": 428612096.0,
+ "724": 428612096.0,
+ "725": 428612096.0,
+ "726": 428612096.0,
+ "727": 428612096.0,
+ "728": 428612096.0,
+ "729": 428612096.0,
+ "730": 428612096.0,
+ "731": 428612096.0,
+ "732": 428612096.0,
+ "733": 428612096.0,
+ "734": 428612096.0,
+ "735": 428612096.0,
+ "736": 428612096.0,
+ "737": 428612096.0,
+ "738": 428612096.0,
+ "739": 428612096.0,
+ "740": 428612096.0,
+ "741": 428612096.0,
+ "742": 428612096.0,
+ "743": 428612096.0,
+ "744": 428612096.0,
+ "745": 428612096.0,
+ "746": 428612096.0,
+ "747": 428612096.0,
+ "748": 428612096.0,
+ "749": 428612096.0,
+ "750": 428612096.0,
+ "751": 428612096.0,
+ "752": 428612096.0,
+ "753": 428612096.0,
+ "754": 428612096.0,
+ "755": 428612096.0,
+ "756": 428612096.0,
+ "757": 428612096.0,
+ "758": 428612096.0,
+ "759": 428612096.0,
+ "760": 428612096.0,
+ "761": 428612096.0,
+ "762": 428612096.0,
+ "763": 428612096.0,
+ "764": 428612096.0,
+ "765": 428612096.0,
+ "766": 428612096.0,
+ "767": 428612096.0,
+ "768": 428612096.0,
+ "769": 428612096.0,
+ "770": 428612096.0,
+ "771": 428612096.0,
+ "772": 428612096.0,
+ "773": 428612096.0,
+ "774": 428612096.0,
+ "775": 428612096.0,
+ "776": 428612096.0,
+ "777": 428612096.0,
+ "778": 428612096.0,
+ "779": 428612096.0,
+ "780": 428612096.0,
+ "781": 428612096.0,
+ "782": 428612096.0,
+ "783": 428612096.0,
+ "784": 428612096.0,
+ "785": 428612096.0,
+ "786": 428612096.0,
+ "787": 428612096.0,
+ "788": 428612096.0,
+ "789": 428612096.0,
+ "790": 428612096.0,
+ "791": 428612096.0,
+ "792": 428612096.0,
+ "793": 428612096.0,
+ "794": 428612096.0,
+ "795": 428612096.0,
+ "796": 428612096.0,
+ "797": 428612096.0,
+ "798": 428612096.0,
+ "799": 428612096.0,
+ "800": 428612096.0,
+ "801": 428612096.0,
+ "802": 428612096.0,
+ "803": 428612096.0,
+ "804": 428612096.0,
+ "805": 428612096.0,
+ "806": 428612096.0,
+ "807": 428612096.0,
+ "808": 428612096.0,
+ "809": 428612096.0,
+ "810": 428612096.0,
+ "811": 428612096.0,
+ "812": 428612096.0,
+ "813": 428612096.0,
+ "814": 428612096.0,
+ "815": 428612096.0,
+ "816": 428612096.0,
+ "817": 428612096.0,
+ "818": 428612096.0,
+ "819": 428612096.0,
+ "820": 428612096.0,
+ "821": 428612096.0,
+ "822": 428612096.0,
+ "823": 428612096.0,
+ "824": 428612096.0,
+ "825": 428612096.0,
+ "826": 428612096.0,
+ "827": 428612096.0,
+ "828": 428612096.0,
+ "829": 428612096.0,
+ "830": 428612096.0,
+ "831": 428612096.0,
+ "832": 428612096.0,
+ "833": 428612096.0,
+ "834": 428612096.0,
+ "835": 428612096.0,
+ "836": 428612096.0,
+ "837": 428612096.0,
+ "838": 428612096.0,
+ "839": 428612096.0,
+ "840": 428612096.0,
+ "841": 428612096.0,
+ "842": 428612096.0,
+ "843": 428612096.0,
+ "844": 428612096.0,
+ "845": 428612096.0,
+ "846": 428612096.0,
+ "847": 428612096.0,
+ "848": 428612096.0,
+ "849": 428612096.0,
+ "850": 428612096.0,
+ "851": 428612096.0,
+ "852": 428612096.0,
+ "853": 428612096.0,
+ "854": 428612096.0,
+ "855": 428612096.0,
+ "856": 428612096.0,
+ "857": 428612096.0,
+ "858": 428612096.0,
+ "859": 428612096.0,
+ "860": 428612096.0,
+ "861": 428612096.0,
+ "862": 428612096.0,
+ "863": 428612096.0,
+ "864": 428612096.0,
+ "865": 428612096.0,
+ "866": 428612096.0,
+ "867": 428612096.0,
+ "868": 428612096.0,
+ "869": 428612096.0,
+ "870": 428612096.0,
+ "871": 428612096.0,
+ "872": 428612096.0,
+ "873": 428612096.0,
+ "874": 428612096.0,
+ "875": 428612096.0,
+ "876": 428612096.0,
+ "877": 428612096.0,
+ "878": 428612096.0,
+ "879": 428612096.0,
+ "880": 428612096.0,
+ "881": 428612096.0,
+ "882": 428612096.0,
+ "883": 428612096.0,
+ "884": 428612096.0,
+ "885": 428612096.0,
+ "886": 428612096.0,
+ "887": 428612096.0,
+ "888": 428612096.0,
+ "889": 428612096.0,
+ "890": 428612096.0,
+ "891": 428612096.0,
+ "892": 428612096.0,
+ "893": 428612096.0,
+ "894": 428612096.0,
+ "895": 428612096.0,
+ "896": 428612096.0,
+ "897": 428612096.0,
+ "898": 428612096.0,
+ "899": 428612096.0,
+ "900": 428612096.0,
+ "901": 428612096.0,
+ "902": 428612096.0,
+ "903": 428612096.0,
+ "904": 428612096.0,
+ "905": 428612096.0,
+ "906": 428612096.0,
+ "907": 428612096.0,
+ "908": 428612096.0,
+ "909": 428612096.0,
+ "910": 428612096.0,
+ "911": 428612096.0,
+ "912": 428612096.0,
+ "913": 428612096.0,
+ "914": 428612096.0,
+ "915": 428612096.0,
+ "916": 428612096.0,
+ "917": 428612096.0,
+ "918": 428612096.0,
+ "919": 428612096.0,
+ "920": 428612096.0,
+ "921": 428612096.0,
+ "922": 428612096.0,
+ "923": 428612096.0,
+ "924": 428612096.0,
+ "925": 428612096.0,
+ "926": 428612096.0,
+ "927": 428612096.0,
+ "928": 428612096.0,
+ "929": 428612096.0,
+ "930": 428612096.0,
+ "931": 428612096.0,
+ "932": 428612096.0,
+ "933": 428612096.0,
+ "934": 428612096.0,
+ "935": 428612096.0,
+ "936": 428612096.0,
+ "937": 428612096.0,
+ "938": 428612096.0,
+ "939": 428612096.0,
+ "940": 428612096.0,
+ "941": 428612096.0,
+ "942": 428612096.0,
+ "943": 428612096.0,
+ "944": 428612096.0,
+ "945": 428612096.0,
+ "946": 428612096.0,
+ "947": 428612096.0,
+ "948": 428612096.0,
+ "949": 428612096.0,
+ "950": 428612096.0,
+ "951": 428612096.0,
+ "952": 428612096.0,
+ "953": 428612096.0,
+ "954": 428612096.0,
+ "955": 428612096.0,
+ "956": 428612096.0,
+ "957": 428612096.0,
+ "958": 428612096.0,
+ "959": 428612096.0,
+ "960": 428612096.0,
+ "961": 428612096.0,
+ "962": 428612096.0,
+ "963": 428612096.0,
+ "964": 428612096.0,
+ "965": 428612096.0,
+ "966": 428612096.0,
+ "967": 428612096.0,
+ "968": 428612096.0,
+ "969": 428612096.0,
+ "970": 428612096.0,
+ "971": 428612096.0,
+ "972": 428612096.0,
+ "973": 428612096.0,
+ "974": 428612096.0,
+ "975": 428612096.0,
+ "976": 428612096.0,
+ "977": 428612096.0,
+ "978": 428612096.0,
+ "979": 428612096.0,
+ "980": 428612096.0,
+ "981": 428612096.0,
+ "982": 428612096.0,
+ "983": 428612096.0,
+ "984": 428612096.0,
+ "985": 428612096.0,
+ "986": 428612096.0,
+ "987": 428612096.0,
+ "988": 428612096.0,
+ "989": 428612096.0,
+ "990": 428612096.0,
+ "991": 428612096.0,
+ "992": 428612096.0,
+ "993": 428612096.0,
+ "994": 428612096.0,
+ "995": 428612096.0,
+ "996": 428612096.0,
+ "997": 428612096.0,
+ "998": 428612096.0,
+ "999": 428612096.0,
+ "1000": 428612096.0,
+ "1001": 428612096.0,
+ "1002": 428612096.0,
+ "1003": 428612096.0,
+ "1004": 428612096.0,
+ "1005": 428612096.0,
+ "1006": 428612096.0,
+ "1007": 428612096.0,
+ "1008": 428612096.0,
+ "1009": 428612096.0,
+ "1010": 428612096.0,
+ "1011": 428612096.0,
+ "1012": 428612096.0,
+ "1013": 428612096.0,
+ "1014": 428612096.0,
+ "1015": 428612096.0,
+ "1016": 428612096.0,
+ "1017": 428612096.0,
+ "1018": 428612096.0,
+ "1019": 428612096.0,
+ "1020": 428612096.0,
+ "1021": 428612096.0,
+ "1022": 428612096.0,
+ "1023": 428612096.0,
+ "1024": 428612096.0,
+ "1025": 428612096.0,
+ "1026": 428612096.0,
+ "1027": 428612096.0,
+ "1028": 428612096.0,
+ "1029": 428612096.0,
+ "1030": 428612096.0,
+ "1031": 428612096.0,
+ "1032": 428612096.0,
+ "1033": 428612096.0,
+ "1034": 428612096.0,
+ "1035": 428612096.0,
+ "1036": 428612096.0,
+ "1037": 428612096.0,
+ "1038": 428612096.0,
+ "1039": 428612096.0,
+ "1040": 428612096.0,
+ "1041": 428612096.0,
+ "1042": 428612096.0,
+ "1043": 428612096.0,
+ "1044": 428612096.0,
+ "1045": 428612096.0,
+ "1046": 428612096.0,
+ "1047": 428612096.0,
+ "1048": 428612096.0,
+ "1049": 428612096.0,
+ "1050": 428612096.0,
+ "1051": 428612096.0,
+ "1052": 428612096.0,
+ "1053": 428612096.0,
+ "1054": 428612096.0,
+ "1055": 428612096.0,
+ "1056": 428612096.0,
+ "1057": 428612096.0,
+ "1058": 428612096.0,
+ "1059": 428612096.0,
+ "1060": 428612096.0,
+ "1061": 428612096.0,
+ "1062": 428612096.0,
+ "1063": 428612096.0,
+ "1064": 428612096.0,
+ "1065": 428612096.0,
+ "1066": 428612096.0,
+ "1067": 428612096.0,
+ "1068": 428612096.0,
+ "1069": 428612096.0,
+ "1070": 428612096.0,
+ "1071": 428612096.0,
+ "1072": 428612096.0,
+ "1073": 428612096.0,
+ "1074": 428612096.0,
+ "1075": 428612096.0,
+ "1076": 428612096.0,
+ "1077": 428612096.0,
+ "1078": 428612096.0,
+ "1079": 428612096.0,
+ "1080": 428612096.0,
+ "1081": 428612096.0,
+ "1082": 428612096.0,
+ "1083": 428612096.0,
+ "1084": 428612096.0,
+ "1085": 428612096.0,
+ "1086": 428612096.0,
+ "1087": 428612096.0,
+ "1088": 428612096.0,
+ "1089": 428612096.0,
+ "1090": 428612096.0,
+ "1091": 428612096.0,
+ "1092": 428612096.0,
+ "1093": 428612096.0,
+ "1094": 428612096.0,
+ "1095": 428612096.0,
+ "1096": 428612096.0,
+ "1097": 428612096.0,
+ "1098": 428612096.0,
+ "1099": 428612096.0,
+ "1100": 428612096.0,
+ "1101": 428612096.0,
+ "1102": 428612096.0,
+ "1103": 428612096.0,
+ "1104": 428612096.0,
+ "1105": 428612096.0,
+ "1106": 428612096.0,
+ "1107": 428612096.0,
+ "1108": 428612096.0,
+ "1109": 428612096.0,
+ "1110": 428612096.0,
+ "1111": 428612096.0,
+ "1112": 428612096.0,
+ "1113": 428612096.0,
+ "1114": 428612096.0,
+ "1115": 428612096.0,
+ "1116": 428612096.0,
+ "1117": 428612096.0,
+ "1118": 428612096.0,
+ "1119": 428612096.0,
+ "1120": 428612096.0,
+ "1121": 428612096.0,
+ "1122": 428612096.0,
+ "1123": 428612096.0,
+ "1124": 428612096.0,
+ "1125": 428612096.0,
+ "1126": 428612096.0,
+ "1127": 428612096.0,
+ "1128": 428612096.0,
+ "1129": 428612096.0,
+ "1130": 428612096.0,
+ "1131": 428612096.0,
+ "1132": 428612096.0,
+ "1133": 428612096.0,
+ "1134": 428612096.0,
+ "1135": 428612096.0,
+ "1136": 428612096.0,
+ "1137": 428612096.0,
+ "1138": 428612096.0,
+ "1139": 428612096.0,
+ "1140": 428612096.0,
+ "1141": 428612096.0,
+ "1142": 428612096.0,
+ "1143": 428612096.0,
+ "1144": 428612096.0,
+ "1145": 428612096.0,
+ "1146": 428612096.0,
+ "1147": 428612096.0,
+ "1148": 428612096.0,
+ "1149": 428612096.0,
+ "1150": 428612096.0,
+ "1151": 428612096.0,
+ "1152": 428612096.0,
+ "1153": 428612096.0,
+ "1154": 428612096.0,
+ "1155": 428612096.0,
+ "1156": 428612096.0,
+ "1157": 428612096.0,
+ "1158": 428612096.0,
+ "1159": 428612096.0,
+ "1160": 428612096.0,
+ "1161": 428612096.0,
+ "1162": 428612096.0,
+ "1163": 428612096.0,
+ "1164": 428612096.0,
+ "1165": 428612096.0,
+ "1166": 428612096.0,
+ "1167": 428612096.0,
+ "1168": 428612096.0,
+ "1169": 428612096.0,
+ "1170": 428612096.0,
+ "1171": 428612096.0,
+ "1172": 428612096.0,
+ "1173": 428612096.0,
+ "1174": 428612096.0,
+ "1175": 428612096.0,
+ "1176": 428612096.0,
+ "1177": 428612096.0,
+ "1178": 428612096.0,
+ "1179": 428612096.0,
+ "1180": 428612096.0,
+ "1181": 428612096.0,
+ "1182": 428612096.0,
+ "1183": 428612096.0,
+ "1184": 428612096.0,
+ "1185": 428612096.0,
+ "1186": 428612096.0,
+ "1187": 428612096.0,
+ "1188": 428612096.0,
+ "1189": 428612096.0,
+ "1190": 428612096.0,
+ "1191": 428612096.0,
+ "1192": 428612096.0,
+ "1193": 428612096.0,
+ "1194": 428612096.0,
+ "1195": 428612096.0,
+ "1196": 428612096.0,
+ "1197": 428612096.0,
+ "1198": 428612096.0,
+ "1199": 428612096.0,
+ "1200": 428612096.0,
+ "1201": 428612096.0,
+ "1202": 428612096.0,
+ "1203": 428612096.0,
+ "1204": 428612096.0,
+ "1205": 428612096.0,
+ "1206": 428612096.0,
+ "1207": 428612096.0,
+ "1208": 428612096.0,
+ "1209": 428612096.0,
+ "1210": 428612096.0,
+ "1211": 428612096.0,
+ "1212": 428612096.0,
+ "1213": 428612096.0,
+ "1214": 428612096.0,
+ "1215": 428612096.0,
+ "1216": 428612096.0,
+ "1217": 428612096.0,
+ "1218": 428612096.0,
+ "1219": 428612096.0,
+ "1220": 428612096.0,
+ "1221": 428612096.0,
+ "1222": 428612096.0,
+ "1223": 428612096.0,
+ "1224": 428612096.0,
+ "1225": 428612096.0,
+ "1226": 428612096.0,
+ "1227": 428612096.0,
+ "1228": 428612096.0,
+ "1229": 428612096.0,
+ "1230": 428612096.0,
+ "1231": 428612096.0,
+ "1232": 428612096.0,
+ "1233": 428612096.0,
+ "1234": 428612096.0,
+ "1235": 428612096.0,
+ "1236": 428612096.0,
+ "1237": 428612096.0,
+ "1238": 428612096.0,
+ "1239": 428612096.0,
+ "1240": 428612096.0,
+ "1241": 428612096.0,
+ "1242": 428612096.0,
+ "1243": 428612096.0,
+ "1244": 428612096.0,
+ "1245": 428612096.0,
+ "1246": 428612096.0,
+ "1247": 428612096.0,
+ "1248": 428612096.0,
+ "1249": 428612096.0,
+ "1250": 428612096.0,
+ "1251": 428612096.0,
+ "1252": 428612096.0,
+ "1253": 428612096.0,
+ "1254": 428612096.0,
+ "1255": 428612096.0,
+ "1256": 428612096.0,
+ "1257": 428612096.0,
+ "1258": 428612096.0,
+ "1259": 428612096.0,
+ "1260": 428612096.0,
+ "1261": 428612096.0,
+ "1262": 428612096.0,
+ "1263": 428612096.0,
+ "1264": 428612096.0,
+ "1265": 428612096.0,
+ "1266": 428612096.0,
+ "1267": 428612096.0,
+ "1268": 428612096.0,
+ "1269": 428612096.0,
+ "1270": 428612096.0,
+ "1271": 428612096.0,
+ "1272": 428612096.0,
+ "1273": 428612096.0,
+ "1274": 428612096.0,
+ "1275": 428612096.0,
+ "1276": 428612096.0,
+ "1277": 428612096.0,
+ "1278": 428612096.0,
+ "1279": 428612096.0,
+ "1280": 428612096.0,
+ "1281": 428612096.0,
+ "1282": 428612096.0,
+ "1283": 428612096.0,
+ "1284": 428612096.0,
+ "1285": 428612096.0,
+ "1286": 428612096.0,
+ "1287": 428612096.0,
+ "1288": 428612096.0,
+ "1289": 428612096.0,
+ "1290": 428612096.0,
+ "1291": 428612096.0,
+ "1292": 428612096.0,
+ "1293": 428612096.0,
+ "1294": 428612096.0,
+ "1295": 428612096.0,
+ "1296": 428612096.0,
+ "1297": 428612096.0,
+ "1298": 428612096.0,
+ "1299": 428612096.0,
+ "1300": 428612096.0,
+ "1301": 428612096.0,
+ "1302": 428612096.0,
+ "1303": 428612096.0,
+ "1304": 428612096.0,
+ "1305": 428612096.0,
+ "1306": 428612096.0,
+ "1307": 428612096.0,
+ "1308": 428612096.0,
+ "1309": 428612096.0,
+ "1310": 428612096.0,
+ "1311": 428612096.0,
+ "1312": 428612096.0,
+ "1313": 428612096.0,
+ "1314": 428612096.0,
+ "1315": 428612096.0,
+ "1316": 428612096.0,
+ "1317": 428612096.0,
+ "1318": 428612096.0,
+ "1319": 428612096.0,
+ "1320": 428612096.0,
+ "1321": 428612096.0,
+ "1322": 428612096.0,
+ "1323": 428612096.0,
+ "1324": 428612096.0,
+ "1325": 428612096.0,
+ "1326": 428612096.0,
+ "1327": 428612096.0,
+ "1328": 428612096.0,
+ "1329": 428612096.0,
+ "1330": 428612096.0,
+ "1331": 428612096.0,
+ "1332": 428612096.0,
+ "1333": 428612096.0,
+ "1334": 428612096.0,
+ "1335": 428612096.0,
+ "1336": 428612096.0,
+ "1337": 428612096.0,
+ "1338": 428612096.0,
+ "1339": 428612096.0,
+ "1340": 428612096.0,
+ "1341": 428612096.0,
+ "1342": 428612096.0,
+ "1343": 428612096.0,
+ "1344": 428612096.0,
+ "1345": 428612096.0,
+ "1346": 428612096.0,
+ "1347": 428612096.0,
+ "1348": 428612096.0,
+ "1349": 428612096.0,
+ "1350": 428612096.0,
+ "1351": 428612096.0,
+ "1352": 428612096.0,
+ "1353": 428612096.0,
+ "1354": 428612096.0,
+ "1355": 428612096.0,
+ "1356": 428612096.0,
+ "1357": 428612096.0,
+ "1358": 428612096.0,
+ "1359": 428612096.0,
+ "1360": 428612096.0,
+ "1361": 428612096.0,
+ "1362": 428612096.0,
+ "1363": 428612096.0,
+ "1364": 428612096.0,
+ "1365": 428612096.0,
+ "1366": 428612096.0,
+ "1367": 428612096.0,
+ "1368": 428612096.0,
+ "1369": 428612096.0,
+ "1370": 428612096.0,
+ "1371": 428612096.0,
+ "1372": 428612096.0,
+ "1373": 428612096.0,
+ "1374": 428612096.0,
+ "1375": 428612096.0,
+ "1376": 428612096.0,
+ "1377": 428612096.0,
+ "1378": 428612096.0,
+ "1379": 428612096.0,
+ "1380": 428612096.0,
+ "1381": 428612096.0,
+ "1382": 428612096.0,
+ "1383": 428612096.0,
+ "1384": 428612096.0,
+ "1385": 428612096.0,
+ "1386": 428612096.0,
+ "1387": 428612096.0,
+ "1388": 428612096.0,
+ "1389": 428612096.0,
+ "1390": 428612096.0,
+ "1391": 428612096.0,
+ "1392": 428612096.0,
+ "1393": 428612096.0,
+ "1394": 428612096.0,
+ "1395": 428612096.0,
+ "1396": 428612096.0,
+ "1397": 428612096.0,
+ "1398": 428612096.0,
+ "1399": 428612096.0,
+ "1400": 428612096.0,
+ "1401": 428612096.0,
+ "1402": 428612096.0,
+ "1403": 428612096.0,
+ "1404": 428612096.0,
+ "1405": 428612096.0,
+ "1406": 428612096.0,
+ "1407": 428612096.0,
+ "1408": 428612096.0,
+ "1409": 428612096.0,
+ "1410": 428612096.0,
+ "1411": 428612096.0,
+ "1412": 428612096.0,
+ "1413": 428612096.0,
+ "1414": 428612096.0,
+ "1415": 428612096.0,
+ "1416": 428612096.0,
+ "1417": 428612096.0,
+ "1418": 428612096.0,
+ "1419": 428612096.0,
+ "1420": 428612096.0,
+ "1421": 428612096.0,
+ "1422": 428612096.0,
+ "1423": 428612096.0,
+ "1424": 428612096.0,
+ "1425": 428612096.0,
+ "1426": 428612096.0,
+ "1427": 428612096.0,
+ "1428": 428612096.0,
+ "1429": 428612096.0,
+ "1430": 428612096.0,
+ "1431": 428612096.0,
+ "1432": 428612096.0,
+ "1433": 428612096.0,
+ "1434": 428612096.0,
+ "1435": 428612096.0,
+ "1436": 428612096.0,
+ "1437": 428612096.0,
+ "1438": 428612096.0,
+ "1439": 428612096.0,
+ "1440": 428612096.0,
+ "1441": 428612096.0,
+ "1442": 428612096.0,
+ "1443": 428612096.0,
+ "1444": 428612096.0,
+ "1445": 428612096.0,
+ "1446": 428612096.0,
+ "1447": 428612096.0,
+ "1448": 428612096.0,
+ "1449": 428612096.0,
+ "1450": 428612096.0,
+ "1451": 428612096.0,
+ "1452": 428612096.0,
+ "1453": 428612096.0,
+ "1454": 428612096.0,
+ "1455": 428612096.0,
+ "1456": 428612096.0,
+ "1457": 428612096.0,
+ "1458": 428612096.0,
+ "1459": 428612096.0,
+ "1460": 428612096.0,
+ "1461": 428612096.0,
+ "1462": 428612096.0,
+ "1463": 428612096.0,
+ "1464": 428612096.0,
+ "1465": 428612096.0,
+ "1466": 428612096.0,
+ "1467": 428612096.0,
+ "1468": 428612096.0,
+ "1469": 428612096.0,
+ "1470": 428612096.0,
+ "1471": 428612096.0,
+ "1472": 428612096.0,
+ "1473": 428612096.0,
+ "1474": 428612096.0,
+ "1475": 428612096.0,
+ "1476": 428612096.0,
+ "1477": 428612096.0,
+ "1478": 428612096.0,
+ "1479": 428612096.0,
+ "1480": 428612096.0,
+ "1481": 428612096.0,
+ "1482": 428612096.0,
+ "1483": 428612096.0,
+ "1484": 428612096.0,
+ "1485": 428612096.0,
+ "1486": 428612096.0,
+ "1487": 428612096.0,
+ "1488": 428612096.0,
+ "1489": 428612096.0,
+ "1490": 428612096.0,
+ "1491": 428612096.0,
+ "1492": 428612096.0,
+ "1493": 428612096.0,
+ "1494": 428612096.0,
+ "1495": 428612096.0,
+ "1496": 428612096.0,
+ "1497": 428612096.0,
+ "1498": 428612096.0,
+ "1499": 428612096.0,
+ "1500": 428612096.0,
+ "1501": 428612096.0,
+ "1502": 428612096.0,
+ "1503": 428612096.0,
+ "1504": 428612096.0,
+ "1505": 428612096.0,
+ "1506": 428612096.0,
+ "1507": 428612096.0,
+ "1508": 428612096.0,
+ "1509": 428612096.0,
+ "1510": 428612096.0,
+ "1511": 428612096.0,
+ "1512": 428612096.0,
+ "1513": 428612096.0,
+ "1514": 428612096.0,
+ "1515": 428612096.0,
+ "1516": 428612096.0,
+ "1517": 428612096.0,
+ "1518": 428612096.0,
+ "1519": 428612096.0,
+ "1520": 428612096.0,
+ "1521": 428612096.0,
+ "1522": 428612096.0,
+ "1523": 428612096.0,
+ "1524": 428612096.0,
+ "1525": 428612096.0,
+ "1526": 428612096.0,
+ "1527": 428612096.0,
+ "1528": 428612096.0,
+ "1529": 428612096.0,
+ "1530": 428612096.0,
+ "1531": 428612096.0,
+ "1532": 428612096.0,
+ "1533": 428612096.0,
+ "1534": 428612096.0,
+ "1535": 428612096.0,
+ "1536": 428612096.0,
+ "1537": 428612096.0,
+ "1538": 428612096.0,
+ "1539": 428612096.0,
+ "1540": 428612096.0,
+ "1541": 428612096.0,
+ "1542": 428612096.0,
+ "1543": 428612096.0,
+ "1544": 428612096.0,
+ "1545": 428612096.0,
+ "1546": 428612096.0,
+ "1547": 428612096.0,
+ "1548": 428612096.0,
+ "1549": 428612096.0,
+ "1550": 428612096.0,
+ "1551": 428612096.0,
+ "1552": 428612096.0,
+ "1553": 428612096.0,
+ "1554": 428612096.0,
+ "1555": 428612096.0,
+ "1556": 428612096.0,
+ "1557": 428612096.0,
+ "1558": 428612096.0,
+ "1559": 428612096.0,
+ "1560": 428612096.0,
+ "1561": 428612096.0,
+ "1562": 428612096.0,
+ "1563": 428612096.0,
+ "1564": 428612096.0,
+ "1565": 428612096.0,
+ "1566": 428612096.0,
+ "1567": 428612096.0,
+ "1568": 428612096.0,
+ "1569": 428612096.0,
+ "1570": 428612096.0,
+ "1571": 428612096.0,
+ "1572": 428612096.0,
+ "1573": 428612096.0,
+ "1574": 428612096.0,
+ "1575": 428612096.0,
+ "1576": 428612096.0,
+ "1577": 428612096.0,
+ "1578": 428612096.0,
+ "1579": 428612096.0,
+ "1580": 428612096.0,
+ "1581": 428612096.0,
+ "1582": 428612096.0,
+ "1583": 428612096.0,
+ "1584": 428612096.0,
+ "1585": 428612096.0,
+ "1586": 428612096.0,
+ "1587": 428612096.0,
+ "1588": 428612096.0,
+ "1589": 428612096.0,
+ "1590": 428612096.0,
+ "1591": 428612096.0,
+ "1592": 428612096.0,
+ "1593": 428612096.0,
+ "1594": 428612096.0,
+ "1595": 428612096.0,
+ "1596": 428612096.0,
+ "1597": 428612096.0,
+ "1598": 428612096.0,
+ "1599": 428612096.0,
+ "1600": 428612096.0,
+ "1601": 428612096.0,
+ "1602": 428612096.0,
+ "1603": 428612096.0,
+ "1604": 428612096.0,
+ "1605": 428612096.0,
+ "1606": 428612096.0,
+ "1607": 428612096.0,
+ "1608": 428612096.0,
+ "1609": 428612096.0,
+ "1610": 428612096.0,
+ "1611": 428612096.0,
+ "1612": 428612096.0,
+ "1613": 428612096.0,
+ "1614": 428612096.0,
+ "1615": 428612096.0,
+ "1616": 428612096.0,
+ "1617": 428612096.0,
+ "1618": 428612096.0,
+ "1619": 428612096.0,
+ "1620": 428612096.0,
+ "1621": 428612096.0,
+ "1622": 428612096.0,
+ "1623": 428612096.0,
+ "1624": 428612096.0,
+ "1625": 428612096.0,
+ "1626": 428612096.0,
+ "1627": 428612096.0,
+ "1628": 428612096.0,
+ "1629": 428612096.0,
+ "1630": 428612096.0,
+ "1631": 428612096.0,
+ "1632": 428612096.0,
+ "1633": 428612096.0,
+ "1634": 428612096.0,
+ "1635": 428612096.0,
+ "1636": 428612096.0,
+ "1637": 428612096.0,
+ "1638": 428612096.0,
+ "1639": 428612096.0,
+ "1640": 428612096.0,
+ "1641": 428612096.0,
+ "1642": 428612096.0,
+ "1643": 428612096.0,
+ "1644": 428612096.0,
+ "1645": 428612096.0,
+ "1646": 428612096.0,
+ "1647": 428612096.0,
+ "1648": 428612096.0,
+ "1649": 428612096.0,
+ "1650": 428612096.0,
+ "1651": 428612096.0,
+ "1652": 428612096.0,
+ "1653": 428612096.0,
+ "1654": 428612096.0,
+ "1655": 428612096.0,
+ "1656": 428612096.0,
+ "1657": 428612096.0,
+ "1658": 428612096.0,
+ "1659": 428612096.0,
+ "1660": 428612096.0,
+ "1661": 428612096.0,
+ "1662": 428612096.0,
+ "1663": 428612096.0,
+ "1664": 428612096.0,
+ "1665": 428612096.0,
+ "1666": 428612096.0,
+ "1667": 428612096.0,
+ "1668": 428612096.0,
+ "1669": 428612096.0,
+ "1670": 428612096.0,
+ "1671": 428612096.0,
+ "1672": 428612096.0,
+ "1673": 428612096.0,
+ "1674": 428612096.0,
+ "1675": 428612096.0,
+ "1676": 428612096.0,
+ "1677": 428612096.0,
+ "1678": 428612096.0,
+ "1679": 428612096.0,
+ "1680": 428612096.0,
+ "1681": 428612096.0,
+ "1682": 428612096.0,
+ "1683": 428612096.0,
+ "1684": 428612096.0,
+ "1685": 428612096.0,
+ "1686": 428612096.0,
+ "1687": 428612096.0,
+ "1688": 428612096.0,
+ "1689": 428612096.0,
+ "1690": 428612096.0,
+ "1691": 428612096.0,
+ "1692": 428612096.0,
+ "1693": 428612096.0,
+ "1694": 428612096.0,
+ "1695": 428612096.0,
+ "1696": 428612096.0,
+ "1697": 428612096.0,
+ "1698": 428612096.0,
+ "1699": 428612096.0,
+ "1700": 428612096.0,
+ "1701": 428612096.0,
+ "1702": 428612096.0,
+ "1703": 428612096.0,
+ "1704": 428612096.0,
+ "1705": 428612096.0,
+ "1706": 428612096.0,
+ "1707": 428612096.0,
+ "1708": 428612096.0,
+ "1709": 428612096.0,
+ "1710": 428612096.0,
+ "1711": 428612096.0,
+ "1712": 428612096.0,
+ "1713": 428612096.0,
+ "1714": 428612096.0,
+ "1715": 428612096.0,
+ "1716": 428612096.0,
+ "1717": 428612096.0,
+ "1718": 428612096.0,
+ "1719": 428612096.0,
+ "1720": 428612096.0,
+ "1721": 428612096.0,
+ "1722": 428612096.0,
+ "1723": 428612096.0,
+ "1724": 428612096.0,
+ "1725": 428612096.0,
+ "1726": 428612096.0,
+ "1727": 428612096.0,
+ "1728": 428612096.0,
+ "1729": 428612096.0,
+ "1730": 428612096.0,
+ "1731": 428612096.0,
+ "1732": 428612096.0,
+ "1733": 428612096.0,
+ "1734": 428612096.0,
+ "1735": 428612096.0,
+ "1736": 428612096.0,
+ "1737": 428612096.0,
+ "1738": 428612096.0,
+ "1739": 428612096.0,
+ "1740": 428612096.0,
+ "1741": 428612096.0,
+ "1742": 428612096.0,
+ "1743": 428612096.0,
+ "1744": 428612096.0,
+ "1745": 428612096.0,
+ "1746": 428612096.0,
+ "1747": 428612096.0,
+ "1748": 428612096.0,
+ "1749": 428612096.0,
+ "1750": 428612096.0,
+ "1751": 428612096.0,
+ "1752": 428612096.0,
+ "1753": 428612096.0,
+ "1754": 428612096.0,
+ "1755": 428612096.0,
+ "1756": 428612096.0,
+ "1757": 428612096.0,
+ "1758": 428612096.0,
+ "1759": 428612096.0,
+ "1760": 428612096.0,
+ "1761": 428612096.0,
+ "1762": 428612096.0,
+ "1763": 428612096.0,
+ "1764": 428612096.0,
+ "1765": 428612096.0,
+ "1766": 428612096.0,
+ "1767": 428612096.0,
+ "1768": 428612096.0,
+ "1769": 428612096.0,
+ "1770": 428612096.0,
+ "1771": 428612096.0,
+ "1772": 428612096.0,
+ "1773": 428612096.0,
+ "1774": 428612096.0,
+ "1775": 428612096.0,
+ "1776": 428612096.0,
+ "1777": 428612096.0,
+ "1778": 428612096.0,
+ "1779": 428612096.0,
+ "1780": 428612096.0,
+ "1781": 428612096.0,
+ "1782": 428612096.0,
+ "1783": 428612096.0,
+ "1784": 428612096.0,
+ "1785": 428612096.0,
+ "1786": 428612096.0,
+ "1787": 428612096.0,
+ "1788": 428612096.0,
+ "1789": 428612096.0,
+ "1790": 428612096.0,
+ "1791": 428612096.0,
+ "1792": 428612096.0,
+ "1793": 428612096.0,
+ "1794": 428612096.0,
+ "1795": 428612096.0,
+ "1796": 428612096.0,
+ "1797": 428612096.0,
+ "1798": 428612096.0,
+ "1799": 428612096.0,
+ "1800": 428612096.0,
+ "1801": 428612096.0,
+ "1802": 428612096.0,
+ "1803": 428612096.0,
+ "1804": 428612096.0,
+ "1805": 428612096.0,
+ "1806": 428612096.0,
+ "1807": 428612096.0,
+ "1808": 428612096.0,
+ "1809": 428612096.0,
+ "1810": 428612096.0,
+ "1811": 428612096.0,
+ "1812": 428612096.0,
+ "1813": 428612096.0,
+ "1814": 428612096.0,
+ "1815": 428612096.0,
+ "1816": 428612096.0,
+ "1817": 428612096.0,
+ "1818": 428612096.0,
+ "1819": 428612096.0,
+ "1820": 428612096.0,
+ "1821": 428612096.0,
+ "1822": 428612096.0,
+ "1823": 428612096.0,
+ "1824": 428612096.0,
+ "1825": 428612096.0,
+ "1826": 428612096.0,
+ "1827": 428612096.0,
+ "1828": 428612096.0,
+ "1829": 428612096.0,
+ "1830": 428612096.0,
+ "1831": 428612096.0,
+ "1832": 428612096.0,
+ "1833": 428612096.0,
+ "1834": 428612096.0,
+ "1835": 428612096.0,
+ "1836": 428612096.0,
+ "1837": 428612096.0,
+ "1838": 428612096.0,
+ "1839": 428612096.0,
+ "1840": 428612096.0,
+ "1841": 428612096.0,
+ "1842": 428612096.0,
+ "1843": 428612096.0,
+ "1844": 428612096.0,
+ "1845": 428612096.0,
+ "1846": 428612096.0,
+ "1847": 428612096.0,
+ "1848": 428612096.0,
+ "1849": 428612096.0,
+ "1850": 428612096.0,
+ "1851": 428612096.0,
+ "1852": 428612096.0,
+ "1853": 428612096.0,
+ "1854": 428612096.0,
+ "1855": 428612096.0,
+ "1856": 428612096.0,
+ "1857": 428612096.0,
+ "1858": 428612096.0,
+ "1859": 428612096.0,
+ "1860": 428612096.0,
+ "1861": 428612096.0,
+ "1862": 428612096.0,
+ "1863": 428612096.0,
+ "1864": 428612096.0,
+ "1865": 428612096.0,
+ "1866": 428612096.0,
+ "1867": 428612096.0,
+ "1868": 428612096.0,
+ "1869": 428612096.0,
+ "1870": 428612096.0,
+ "1871": 428612096.0,
+ "1872": 428612096.0,
+ "1873": 428612096.0,
+ "1874": 428612096.0,
+ "1875": 428612096.0,
+ "1876": 428612096.0,
+ "1877": 428612096.0,
+ "1878": 428612096.0,
+ "1879": 428612096.0,
+ "1880": 428612096.0,
+ "1881": 428612096.0,
+ "1882": 428612096.0,
+ "1883": 428612096.0,
+ "1884": 428612096.0,
+ "1885": 428612096.0,
+ "1886": 428612096.0,
+ "1887": 428612096.0,
+ "1888": 428612096.0,
+ "1889": 428612096.0,
+ "1890": 428612096.0,
+ "1891": 428612096.0,
+ "1892": 428612096.0,
+ "1893": 428612096.0,
+ "1894": 428612096.0,
+ "1895": 428612096.0,
+ "1896": 428612096.0,
+ "1897": 428612096.0,
+ "1898": 428612096.0,
+ "1899": 428612096.0,
+ "1900": 428612096.0,
+ "1901": 428612096.0,
+ "1902": 428612096.0,
+ "1903": 428612096.0,
+ "1904": 428612096.0,
+ "1905": 428612096.0,
+ "1906": 428612096.0,
+ "1907": 428612096.0,
+ "1908": 428612096.0,
+ "1909": 428612096.0,
+ "1910": 428612096.0,
+ "1911": 428612096.0,
+ "1912": 428612096.0,
+ "1913": 428612096.0,
+ "1914": 428612096.0,
+ "1915": 428612096.0,
+ "1916": 428612096.0,
+ "1917": 428612096.0,
+ "1918": 428612096.0,
+ "1919": 428612096.0,
+ "1920": 428612096.0,
+ "1921": 428612096.0,
+ "1922": 428612096.0,
+ "1923": 428612096.0,
+ "1924": 428612096.0,
+ "1925": 428612096.0,
+ "1926": 428612096.0,
+ "1927": 428612096.0,
+ "1928": 428612096.0,
+ "1929": 428612096.0,
+ "1930": 428612096.0,
+ "1931": 428612096.0,
+ "1932": 428612096.0,
+ "1933": 428612096.0,
+ "1934": 428612096.0,
+ "1935": 428612096.0,
+ "1936": 428612096.0,
+ "1937": 428612096.0,
+ "1938": 428612096.0,
+ "1939": 428612096.0,
+ "1940": 428612096.0,
+ "1941": 428612096.0,
+ "1942": 428612096.0,
+ "1943": 428612096.0,
+ "1944": 428612096.0,
+ "1945": 428612096.0,
+ "1946": 428612096.0,
+ "1947": 428612096.0,
+ "1948": 428612096.0,
+ "1949": 428612096.0,
+ "1950": 428612096.0,
+ "1951": 428612096.0,
+ "1952": 428612096.0,
+ "1953": 428612096.0,
+ "1954": 428612096.0,
+ "1955": 428612096.0,
+ "1956": 428612096.0,
+ "1957": 428612096.0,
+ "1958": 428612096.0,
+ "1959": 428612096.0,
+ "1960": 428612096.0,
+ "1961": 428612096.0,
+ "1962": 428612096.0,
+ "1963": 428612096.0,
+ "1964": 428612096.0,
+ "1965": 428612096.0,
+ "1966": 428612096.0,
+ "1967": 428612096.0,
+ "1968": 428612096.0,
+ "1969": 428612096.0,
+ "1970": 428612096.0,
+ "1971": 428612096.0,
+ "1972": 428612096.0,
+ "1973": 428612096.0,
+ "1974": 428612096.0,
+ "1975": 428612096.0,
+ "1976": 428612096.0,
+ "1977": 428612096.0,
+ "1978": 428612096.0,
+ "1979": 428612096.0,
+ "1980": 428612096.0,
+ "1981": 428612096.0,
+ "1982": 428612096.0,
+ "1983": 428612096.0,
+ "1984": 428612096.0,
+ "1985": 428612096.0,
+ "1986": 428612096.0,
+ "1987": 428612096.0,
+ "1988": 428612096.0,
+ "1989": 428612096.0,
+ "1990": 428612096.0,
+ "1991": 428612096.0,
+ "1992": 428612096.0,
+ "1993": 428612096.0,
+ "1994": 428612096.0,
+ "1995": 428612096.0,
+ "1996": 428612096.0,
+ "1997": 428612096.0,
+ "1998": 428612096.0,
+ "1999": 428612096.0,
+ "2000": 428612096.0
+ }
+ },
+ "iteration-time": {
+ "start_step": 1,
+ "end_step": 2000,
+ "step_interval": 1,
+ "values": {
+ "1": 22.43653,
+ "2": 5.05,
+ "3": 4.99632,
+ "4": 5.00941,
+ "5": 5.30047,
+ "6": 5.00529,
+ "7": 4.98693,
+ "8": 5.03236,
+ "9": 5.04733,
+ "10": 5.0355,
+ "11": 5.05504,
+ "12": 5.02789,
+ "13": 5.05026,
+ "14": 5.03817,
+ "15": 5.03065,
+ "16": 5.04414,
+ "17": 5.00251,
+ "18": 4.9928,
+ "19": 4.99792,
+ "20": 4.99648,
+ "21": 5.01668,
+ "22": 4.97973,
+ "23": 5.06379,
+ "24": 5.01631,
+ "25": 4.96187,
+ "26": 4.95004,
+ "27": 4.95649,
+ "28": 4.93702,
+ "29": 4.93675,
+ "30": 4.92101,
+ "31": 4.93325,
+ "32": 4.92626,
+ "33": 4.93256,
+ "34": 4.93518,
+ "35": 4.95011,
+ "36": 4.959,
+ "37": 5.41549,
+ "38": 5.7108,
+ "39": 4.96475,
+ "40": 4.95756,
+ "41": 5.03533,
+ "42": 4.94591,
+ "43": 5.30856,
+ "44": 4.93166,
+ "45": 5.29533,
+ "46": 6.02838,
+ "47": 4.99271,
+ "48": 4.93548,
+ "49": 4.93262,
+ "50": 4.93589,
+ "51": 4.93457,
+ "52": 4.9402,
+ "53": 4.93593,
+ "54": 4.93266,
+ "55": 4.93457,
+ "56": 4.926,
+ "57": 4.94015,
+ "58": 4.93606,
+ "59": 4.92819,
+ "60": 4.92679,
+ "61": 4.92853,
+ "62": 4.93744,
+ "63": 4.93014,
+ "64": 4.92895,
+ "65": 4.92774,
+ "66": 4.9263,
+ "67": 4.92483,
+ "68": 4.91654,
+ "69": 4.95386,
+ "70": 4.95969,
+ "71": 4.97371,
+ "72": 4.96736,
+ "73": 4.98575,
+ "74": 4.968,
+ "75": 5.68071,
+ "76": 4.98487,
+ "77": 4.98651,
+ "78": 4.97441,
+ "79": 4.97854,
+ "80": 4.97886,
+ "81": 4.98163,
+ "82": 4.97647,
+ "83": 5.33849,
+ "84": 4.98394,
+ "85": 4.98,
+ "86": 4.96888,
+ "87": 4.9685,
+ "88": 5.33167,
+ "89": 5.40565,
+ "90": 4.97724,
+ "91": 6.05451,
+ "92": 4.9699,
+ "93": 4.96947,
+ "94": 4.97853,
+ "95": 5.03234,
+ "96": 4.9703,
+ "97": 4.9766,
+ "98": 4.96386,
+ "99": 4.97968,
+ "100": 4.96583,
+ "101": 4.956,
+ "102": 4.94425,
+ "103": 4.96789,
+ "104": 4.96252,
+ "105": 4.97853,
+ "106": 4.98313,
+ "107": 4.98,
+ "108": 4.97528,
+ "109": 4.98226,
+ "110": 4.98532,
+ "111": 4.95791,
+ "112": 4.95409,
+ "113": 5.66529,
+ "114": 4.96347,
+ "115": 4.99625,
+ "116": 4.99199,
+ "117": 4.98823,
+ "118": 4.98114,
+ "119": 4.97652,
+ "120": 4.98449,
+ "121": 4.98578,
+ "122": 4.98423,
+ "123": 4.9824,
+ "124": 4.98111,
+ "125": 4.98291,
+ "126": 4.98215,
+ "127": 4.98484,
+ "128": 5.35151,
+ "129": 4.9912,
+ "130": 4.99188,
+ "131": 4.98662,
+ "132": 5.34041,
+ "133": 4.98063,
+ "134": 5.33235,
+ "135": 5.69907,
+ "136": 5.33587,
+ "137": 4.98509,
+ "138": 5.61624,
+ "139": 5.23864,
+ "140": 4.92839,
+ "141": 4.95868,
+ "142": 4.93611,
+ "143": 4.9473,
+ "144": 4.9282,
+ "145": 4.93563,
+ "146": 4.92822,
+ "147": 4.94205,
+ "148": 4.94037,
+ "149": 4.93429,
+ "150": 5.62642,
+ "151": 4.93794,
+ "152": 4.9323,
+ "153": 4.93391,
+ "154": 4.93581,
+ "155": 4.93177,
+ "156": 4.93719,
+ "157": 4.93775,
+ "158": 4.93223,
+ "159": 4.9449,
+ "160": 4.93898,
+ "161": 4.94198,
+ "162": 4.9436,
+ "163": 4.9355,
+ "164": 4.93432,
+ "165": 4.9382,
+ "166": 4.94332,
+ "167": 4.93425,
+ "168": 4.93189,
+ "169": 4.92717,
+ "170": 4.94393,
+ "171": 4.94517,
+ "172": 4.92976,
+ "173": 5.303,
+ "174": 4.92818,
+ "175": 4.92924,
+ "176": 4.9385,
+ "177": 5.27801,
+ "178": 4.93182,
+ "179": 5.28092,
+ "180": 5.99722,
+ "181": 4.92656,
+ "182": 4.92594,
+ "183": 4.92947,
+ "184": 4.93087,
+ "185": 4.92967,
+ "186": 4.93088,
+ "187": 5.62908,
+ "188": 4.93498,
+ "189": 4.9476,
+ "190": 4.93843,
+ "191": 4.94101,
+ "192": 4.93265,
+ "193": 4.93046,
+ "194": 4.93133,
+ "195": 4.94044,
+ "196": 4.93997,
+ "197": 4.93336,
+ "198": 6.32096,
+ "199": 4.95042,
+ "200": 4.91888,
+ "201": 4.91803,
+ "202": 4.92212,
+ "203": 4.91738,
+ "204": 4.93431,
+ "205": 4.93078,
+ "206": 4.9288,
+ "207": 4.9431,
+ "208": 4.93288,
+ "209": 4.93152,
+ "210": 4.92297,
+ "211": 4.92152,
+ "212": 4.92078,
+ "213": 4.93382,
+ "214": 4.92203,
+ "215": 4.92628,
+ "216": 4.92759,
+ "217": 4.91972,
+ "218": 4.93018,
+ "219": 5.30587,
+ "220": 4.92639,
+ "221": 4.92815,
+ "222": 5.28345,
+ "223": 4.93513,
+ "224": 5.62954,
+ "225": 6.35198,
+ "226": 4.94108,
+ "227": 4.94033,
+ "228": 4.94077,
+ "229": 4.9445,
+ "230": 4.95277,
+ "231": 4.93684,
+ "232": 4.94258,
+ "233": 4.9386,
+ "234": 4.94149,
+ "235": 4.94872,
+ "236": 4.95361,
+ "237": 4.94924,
+ "238": 4.93722,
+ "239": 4.94342,
+ "240": 4.95029,
+ "241": 4.94512,
+ "242": 4.9423,
+ "243": 4.93861,
+ "244": 4.93578,
+ "245": 4.93502,
+ "246": 4.94519,
+ "247": 4.93658,
+ "248": 4.93761,
+ "249": 4.94583,
+ "250": 4.94414,
+ "251": 4.94331,
+ "252": 4.94044,
+ "253": 4.94317,
+ "254": 4.94161,
+ "255": 4.95295,
+ "256": 4.95044,
+ "257": 4.94816,
+ "258": 4.94006,
+ "259": 4.94409,
+ "260": 4.9408,
+ "261": 4.94791,
+ "262": 5.63079,
+ "263": 4.95361,
+ "264": 5.3219,
+ "265": 4.96046,
+ "266": 4.95564,
+ "267": 5.30372,
+ "268": 5.30618,
+ "269": 4.94954,
+ "270": 6.01622,
+ "271": 4.9509,
+ "272": 4.9579,
+ "273": 4.9529,
+ "274": 4.95339,
+ "275": 4.94721,
+ "276": 4.95053,
+ "277": 4.9434,
+ "278": 4.9389,
+ "279": 4.94021,
+ "280": 4.93862,
+ "281": 4.93834,
+ "282": 4.93985,
+ "283": 4.94183,
+ "284": 4.93716,
+ "285": 4.9443,
+ "286": 4.94305,
+ "287": 4.93467,
+ "288": 4.93816,
+ "289": 4.93749,
+ "290": 4.9349,
+ "291": 4.939,
+ "292": 4.93482,
+ "293": 4.94665,
+ "294": 4.93648,
+ "295": 4.93823,
+ "296": 4.93522,
+ "297": 4.93472,
+ "298": 4.93288,
+ "299": 5.61551,
+ "300": 4.95418,
+ "301": 4.95347,
+ "302": 4.95005,
+ "303": 4.95224,
+ "304": 5.01672,
+ "305": 4.94451,
+ "306": 4.9469,
+ "307": 4.94674,
+ "308": 4.95506,
+ "309": 5.3147,
+ "310": 4.97913,
+ "311": 5.29357,
+ "312": 4.94239,
+ "313": 5.28356,
+ "314": 5.66502,
+ "315": 5.29945,
+ "316": 4.94213,
+ "317": 4.93439,
+ "318": 4.94085,
+ "319": 4.93452,
+ "320": 4.94083,
+ "321": 4.93407,
+ "322": 4.93596,
+ "323": 4.9411,
+ "324": 4.94091,
+ "325": 4.93723,
+ "326": 4.93682,
+ "327": 4.93712,
+ "328": 4.99643,
+ "329": 4.94011,
+ "330": 4.93777,
+ "331": 4.93553,
+ "332": 4.938,
+ "333": 4.94101,
+ "334": 4.93199,
+ "335": 4.93179,
+ "336": 5.28612,
+ "337": 5.30266,
+ "338": 4.96477,
+ "339": 4.97585,
+ "340": 4.95959,
+ "341": 4.95912,
+ "342": 4.96594,
+ "343": 4.96105,
+ "344": 4.96501,
+ "345": 4.96175,
+ "346": 4.96452,
+ "347": 4.9603,
+ "348": 4.95434,
+ "349": 4.95658,
+ "350": 4.95773,
+ "351": 4.96723,
+ "352": 5.02353,
+ "353": 4.95487,
+ "354": 5.32227,
+ "355": 4.95601,
+ "356": 5.29598,
+ "357": 4.95819,
+ "358": 5.29935,
+ "359": 6.01593,
+ "360": 4.96832,
+ "361": 4.95302,
+ "362": 4.95944,
+ "363": 4.95167,
+ "364": 4.9483,
+ "365": 4.94951,
+ "366": 4.9525,
+ "367": 4.95364,
+ "368": 4.94948,
+ "369": 4.95258,
+ "370": 4.94974,
+ "371": 4.96357,
+ "372": 4.94701,
+ "373": 4.94584,
+ "374": 5.27688,
+ "375": 5.29329,
+ "376": 4.93553,
+ "377": 4.93296,
+ "378": 4.93431,
+ "379": 4.94158,
+ "380": 4.98441,
+ "381": 4.99657,
+ "382": 4.97634,
+ "383": 4.98015,
+ "384": 4.98178,
+ "385": 4.97595,
+ "386": 4.97431,
+ "387": 4.97965,
+ "388": 4.91884,
+ "389": 4.92436,
+ "390": 4.9179,
+ "391": 4.91999,
+ "392": 4.92113,
+ "393": 4.92231,
+ "394": 4.91815,
+ "395": 4.92381,
+ "396": 4.91848,
+ "397": 4.92412,
+ "398": 4.91541,
+ "399": 4.91455,
+ "400": 5.29982,
+ "401": 5.26416,
+ "402": 5.2612,
+ "403": 4.91795,
+ "404": 5.63316,
+ "405": 5.27153,
+ "406": 4.90744,
+ "407": 4.9142,
+ "408": 4.90831,
+ "409": 4.90838,
+ "410": 4.92063,
+ "411": 5.25377,
+ "412": 5.26322,
+ "413": 4.91895,
+ "414": 4.92378,
+ "415": 4.91866,
+ "416": 4.91955,
+ "417": 4.92152,
+ "418": 4.91929,
+ "419": 4.9201,
+ "420": 4.91526,
+ "421": 4.91974,
+ "422": 4.92503,
+ "423": 4.92579,
+ "424": 4.91791,
+ "425": 4.92253,
+ "426": 4.92114,
+ "427": 4.91774,
+ "428": 4.91171,
+ "429": 4.9125,
+ "430": 4.91411,
+ "431": 4.90802,
+ "432": 4.9164,
+ "433": 4.90723,
+ "434": 4.92382,
+ "435": 4.9069,
+ "436": 4.91154,
+ "437": 4.90512,
+ "438": 4.9175,
+ "439": 4.91782,
+ "440": 4.91028,
+ "441": 4.91048,
+ "442": 4.90894,
+ "443": 4.88817,
+ "444": 4.88126,
+ "445": 5.24853,
+ "446": 4.87836,
+ "447": 5.24263,
+ "448": 5.25398,
+ "449": 6.28763,
+ "450": 4.88338,
+ "451": 4.89491,
+ "452": 4.88709,
+ "453": 4.89008,
+ "454": 4.90322,
+ "455": 4.90113,
+ "456": 4.90439,
+ "457": 4.90223,
+ "458": 4.90641,
+ "459": 4.90851,
+ "460": 4.9009,
+ "461": 4.89968,
+ "462": 4.89662,
+ "463": 4.9081,
+ "464": 4.88866,
+ "465": 4.90253,
+ "466": 4.90724,
+ "467": 4.89875,
+ "468": 4.90067,
+ "469": 4.90495,
+ "470": 4.89887,
+ "471": 4.89965,
+ "472": 4.90145,
+ "473": 4.88549,
+ "474": 4.87833,
+ "475": 4.88274,
+ "476": 4.87937,
+ "477": 4.88019,
+ "478": 4.87808,
+ "479": 4.88269,
+ "480": 4.87591,
+ "481": 4.88072,
+ "482": 4.87452,
+ "483": 4.8839,
+ "484": 4.87834,
+ "485": 5.21963,
+ "486": 4.8887,
+ "487": 5.22473,
+ "488": 4.88748,
+ "489": 4.89663,
+ "490": 5.6108,
+ "491": 5.24875,
+ "492": 4.88583,
+ "493": 5.24488,
+ "494": 5.59516,
+ "495": 4.89058,
+ "496": 4.91601,
+ "497": 4.88752,
+ "498": 4.88645,
+ "499": 4.89008,
+ "500": 4.89271,
+ "501": 4.8913,
+ "502": 4.89039,
+ "503": 4.8906,
+ "504": 4.88603,
+ "505": 4.92691,
+ "506": 4.91793,
+ "507": 4.92158,
+ "508": 4.91981,
+ "509": 4.92795,
+ "510": 4.91413,
+ "511": 4.91073,
+ "512": 4.90909,
+ "513": 4.91434,
+ "514": 4.91509,
+ "515": 4.91002,
+ "516": 4.9115,
+ "517": 4.91722,
+ "518": 4.91514,
+ "519": 4.91283,
+ "520": 4.91403,
+ "521": 4.91077,
+ "522": 4.91167,
+ "523": 5.26088,
+ "524": 5.27803,
+ "525": 4.92516,
+ "526": 4.93143,
+ "527": 4.9217,
+ "528": 4.92344,
+ "529": 4.91786,
+ "530": 4.9193,
+ "531": 4.881,
+ "532": 4.87697,
+ "533": 4.88329,
+ "534": 5.23628,
+ "535": 5.26149,
+ "536": 4.88132,
+ "537": 5.23366,
+ "538": 5.92272,
+ "539": 4.8822,
+ "540": 4.87645,
+ "541": 4.87941,
+ "542": 4.8726,
+ "543": 4.87977,
+ "544": 4.88572,
+ "545": 4.97915,
+ "546": 4.94014,
+ "547": 4.9447,
+ "548": 4.94585,
+ "549": 4.93712,
+ "550": 4.95428,
+ "551": 4.9405,
+ "552": 4.94013,
+ "553": 4.94514,
+ "554": 4.94542,
+ "555": 4.94729,
+ "556": 4.93818,
+ "557": 4.94632,
+ "558": 4.95928,
+ "559": 4.94439,
+ "560": 5.29538,
+ "561": 5.29912,
+ "562": 4.95591,
+ "563": 4.94545,
+ "564": 4.9589,
+ "565": 4.9486,
+ "566": 4.94487,
+ "567": 4.94563,
+ "568": 4.96795,
+ "569": 4.96332,
+ "570": 4.95731,
+ "571": 4.95751,
+ "572": 4.94401,
+ "573": 4.94623,
+ "574": 4.9438,
+ "575": 4.9342,
+ "576": 4.93847,
+ "577": 4.94215,
+ "578": 4.94036,
+ "579": 4.95135,
+ "580": 5.28996,
+ "581": 5.66625,
+ "582": 4.93892,
+ "583": 5.64719,
+ "584": 5.28091,
+ "585": 4.95827,
+ "586": 4.95725,
+ "587": 4.96107,
+ "588": 4.95092,
+ "589": 4.95514,
+ "590": 4.94845,
+ "591": 4.94342,
+ "592": 4.9488,
+ "593": 4.93576,
+ "594": 4.93657,
+ "595": 4.93545,
+ "596": 4.93595,
+ "597": 5.29319,
+ "598": 5.28921,
+ "599": 4.95347,
+ "600": 4.94896,
+ "601": 4.94543,
+ "602": 4.95405,
+ "603": 4.94996,
+ "604": 4.94726,
+ "605": 4.94394,
+ "606": 4.9443,
+ "607": 4.99448,
+ "608": 4.93032,
+ "609": 4.96191,
+ "610": 4.95086,
+ "611": 4.94486,
+ "612": 4.94403,
+ "613": 4.94194,
+ "614": 4.94624,
+ "615": 4.94461,
+ "616": 4.96458,
+ "617": 4.94658,
+ "618": 4.94254,
+ "619": 4.93901,
+ "620": 4.94138,
+ "621": 4.94747,
+ "622": 4.95796,
+ "623": 4.94579,
+ "624": 5.30372,
+ "625": 4.94082,
+ "626": 5.66834,
+ "627": 4.93994,
+ "628": 5.97473,
+ "629": 4.94152,
+ "630": 4.94328,
+ "631": 4.9385,
+ "632": 4.9688,
+ "633": 4.93837,
+ "634": 5.25732,
+ "635": 4.9147,
+ "636": 5.25839,
+ "637": 4.92259,
+ "638": 4.91081,
+ "639": 4.92229,
+ "640": 4.92687,
+ "641": 4.91335,
+ "642": 4.91557,
+ "643": 4.91922,
+ "644": 4.91847,
+ "645": 4.92121,
+ "646": 4.92251,
+ "647": 4.91255,
+ "648": 4.91291,
+ "649": 4.91003,
+ "650": 4.90867,
+ "651": 4.91235,
+ "652": 4.90719,
+ "653": 4.90865,
+ "654": 4.90719,
+ "655": 4.91306,
+ "656": 4.90861,
+ "657": 4.90901,
+ "658": 4.91095,
+ "659": 4.90726,
+ "660": 4.90915,
+ "661": 4.91011,
+ "662": 4.90721,
+ "663": 4.90907,
+ "664": 4.91699,
+ "665": 4.91095,
+ "666": 4.90826,
+ "667": 4.90687,
+ "668": 4.90738,
+ "669": 5.25716,
+ "670": 5.25453,
+ "671": 5.28603,
+ "672": 5.25386,
+ "673": 6.29304,
+ "674": 4.91719,
+ "675": 4.9174,
+ "676": 4.92014,
+ "677": 4.92048,
+ "678": 4.90878,
+ "679": 4.90967,
+ "680": 4.90981,
+ "681": 4.91054,
+ "682": 4.90885,
+ "683": 4.90932,
+ "684": 4.915,
+ "685": 4.90701,
+ "686": 4.91124,
+ "687": 4.91733,
+ "688": 4.91577,
+ "689": 4.91189,
+ "690": 4.90854,
+ "691": 4.90631,
+ "692": 4.90689,
+ "693": 4.9142,
+ "694": 4.90933,
+ "695": 4.90064,
+ "696": 4.88962,
+ "697": 4.89317,
+ "698": 4.89665,
+ "699": 4.90473,
+ "700": 4.90675,
+ "701": 4.90072,
+ "702": 4.90347,
+ "703": 4.90535,
+ "704": 4.90243,
+ "705": 4.90653,
+ "706": 4.90494,
+ "707": 4.90715,
+ "708": 4.89971,
+ "709": 5.25068,
+ "710": 5.24447,
+ "711": 4.91173,
+ "712": 4.91607,
+ "713": 5.26011,
+ "714": 4.90966,
+ "715": 4.90512,
+ "716": 5.63181,
+ "717": 5.62011,
+ "718": 5.23301,
+ "719": 4.91317,
+ "720": 4.90779,
+ "721": 4.90675,
+ "722": 4.90612,
+ "723": 4.90554,
+ "724": 4.90952,
+ "725": 4.90669,
+ "726": 4.90589,
+ "727": 4.9062,
+ "728": 4.91028,
+ "729": 4.905,
+ "730": 4.90848,
+ "731": 4.90621,
+ "732": 4.91216,
+ "733": 4.90248,
+ "734": 4.90051,
+ "735": 4.90319,
+ "736": 4.90401,
+ "737": 4.90646,
+ "738": 4.90558,
+ "739": 4.90438,
+ "740": 4.90694,
+ "741": 4.9036,
+ "742": 4.90521,
+ "743": 4.90326,
+ "744": 4.90534,
+ "745": 4.90658,
+ "746": 5.24876,
+ "747": 4.91293,
+ "748": 5.24944,
+ "749": 4.90712,
+ "750": 4.90572,
+ "751": 4.90977,
+ "752": 4.90683,
+ "753": 4.90815,
+ "754": 4.90611,
+ "755": 4.91427,
+ "756": 4.9129,
+ "757": 4.91264,
+ "758": 5.25755,
+ "759": 4.91199,
+ "760": 5.2647,
+ "761": 4.91559,
+ "762": 5.64712,
+ "763": 5.59149,
+ "764": 4.91566,
+ "765": 4.91348,
+ "766": 4.92052,
+ "767": 4.9149,
+ "768": 4.91624,
+ "769": 4.90919,
+ "770": 4.9208,
+ "771": 4.9111,
+ "772": 4.91242,
+ "773": 4.91183,
+ "774": 4.91856,
+ "775": 4.91524,
+ "776": 4.91642,
+ "777": 4.91271,
+ "778": 4.91587,
+ "779": 4.91173,
+ "780": 4.9163,
+ "781": 4.9101,
+ "782": 4.90927,
+ "783": 4.91594,
+ "784": 5.27562,
+ "785": 5.29399,
+ "786": 4.92064,
+ "787": 4.92508,
+ "788": 4.91936,
+ "789": 4.92025,
+ "790": 4.92839,
+ "791": 4.91829,
+ "792": 4.9234,
+ "793": 4.92615,
+ "794": 4.91968,
+ "795": 4.91417,
+ "796": 4.89214,
+ "797": 4.87642,
+ "798": 4.87726,
+ "799": 4.88691,
+ "800": 4.87753,
+ "801": 4.90361,
+ "802": 4.91538,
+ "803": 5.25822,
+ "804": 5.25769,
+ "805": 4.90985,
+ "806": 4.91228,
+ "807": 5.6423,
+ "808": 5.23836,
+ "809": 4.9314,
+ "810": 4.91226,
+ "811": 4.91382,
+ "812": 4.91588,
+ "813": 4.91005,
+ "814": 4.9202,
+ "815": 4.90766,
+ "816": 4.90744,
+ "817": 4.91497,
+ "818": 4.91,
+ "819": 4.90572,
+ "820": 4.91342,
+ "821": 5.26215,
+ "822": 5.25971,
+ "823": 4.92486,
+ "824": 4.92645,
+ "825": 4.91518,
+ "826": 4.91893,
+ "827": 4.90862,
+ "828": 4.9143,
+ "829": 4.91422,
+ "830": 4.91829,
+ "831": 4.90569,
+ "832": 4.91122,
+ "833": 4.90584,
+ "834": 4.90518,
+ "835": 4.90755,
+ "836": 4.90656,
+ "837": 4.90626,
+ "838": 4.90987,
+ "839": 4.91189,
+ "840": 4.90735,
+ "841": 4.90697,
+ "842": 4.91064,
+ "843": 4.90409,
+ "844": 4.90711,
+ "845": 4.90385,
+ "846": 4.90599,
+ "847": 5.24636,
+ "848": 4.89752,
+ "849": 5.24655,
+ "850": 4.90148,
+ "851": 4.89501,
+ "852": 5.98483,
+ "853": 4.89468,
+ "854": 4.89653,
+ "855": 4.8954,
+ "856": 4.89811,
+ "857": 4.90026,
+ "858": 5.24069,
+ "859": 4.91345,
+ "860": 5.2538,
+ "861": 4.91107,
+ "862": 4.90905,
+ "863": 4.90289,
+ "864": 4.90179,
+ "865": 4.90697,
+ "866": 4.89969,
+ "867": 4.89622,
+ "868": 4.89817,
+ "869": 4.89734,
+ "870": 4.89421,
+ "871": 4.902,
+ "872": 4.89737,
+ "873": 4.90082,
+ "874": 4.8986,
+ "875": 4.9034,
+ "876": 4.90213,
+ "877": 4.89969,
+ "878": 4.90652,
+ "879": 4.90216,
+ "880": 4.90541,
+ "881": 4.90491,
+ "882": 4.89798,
+ "883": 4.89325,
+ "884": 4.89662,
+ "885": 4.91,
+ "886": 4.89481,
+ "887": 4.90025,
+ "888": 4.89887,
+ "889": 4.89458,
+ "890": 4.89351,
+ "891": 4.89343,
+ "892": 5.24625,
+ "893": 4.90075,
+ "894": 5.24719,
+ "895": 4.89439,
+ "896": 5.95508,
+ "897": 5.92842,
+ "898": 4.90126,
+ "899": 4.91443,
+ "900": 4.90222,
+ "901": 4.89928,
+ "902": 4.89952,
+ "903": 4.89905,
+ "904": 4.90536,
+ "905": 4.90627,
+ "906": 4.90188,
+ "907": 4.90671,
+ "908": 4.90531,
+ "909": 4.90614,
+ "910": 4.90319,
+ "911": 4.90668,
+ "912": 4.90614,
+ "913": 4.90641,
+ "914": 4.90219,
+ "915": 4.89858,
+ "916": 4.89788,
+ "917": 4.90114,
+ "918": 4.89062,
+ "919": 4.89675,
+ "920": 4.89412,
+ "921": 4.89851,
+ "922": 4.90258,
+ "923": 4.89837,
+ "924": 4.89168,
+ "925": 4.90558,
+ "926": 4.88926,
+ "927": 4.89631,
+ "928": 4.89481,
+ "929": 4.89896,
+ "930": 4.90349,
+ "931": 4.90254,
+ "932": 4.89424,
+ "933": 5.2393,
+ "934": 4.90447,
+ "935": 5.24957,
+ "936": 4.89799,
+ "937": 5.24757,
+ "938": 4.90497,
+ "939": 5.26023,
+ "940": 4.905,
+ "941": 4.90603,
+ "942": 5.89013,
+ "943": 5.2754,
+ "944": 4.89903,
+ "945": 4.90825,
+ "946": 4.90072,
+ "947": 4.91095,
+ "948": 4.89642,
+ "949": 4.90314,
+ "950": 4.9027,
+ "951": 4.90276,
+ "952": 4.90005,
+ "953": 4.90591,
+ "954": 4.89179,
+ "955": 4.89648,
+ "956": 4.89739,
+ "957": 4.90258,
+ "958": 4.90027,
+ "959": 4.90627,
+ "960": 4.89592,
+ "961": 4.89153,
+ "962": 4.89826,
+ "963": 4.89281,
+ "964": 4.88656,
+ "965": 4.9056,
+ "966": 4.88948,
+ "967": 4.89075,
+ "968": 4.89128,
+ "969": 4.88907,
+ "970": 5.23384,
+ "971": 4.91197,
+ "972": 5.24458,
+ "973": 4.90766,
+ "974": 4.90557,
+ "975": 4.9059,
+ "976": 4.90502,
+ "977": 4.90392,
+ "978": 4.90541,
+ "979": 4.89927,
+ "980": 4.9047,
+ "981": 4.90276,
+ "982": 5.2516,
+ "983": 5.25121,
+ "984": 4.90232,
+ "985": 4.90209,
+ "986": 5.26939,
+ "987": 5.52932,
+ "988": 5.28293,
+ "989": 4.91742,
+ "990": 4.90637,
+ "991": 4.90953,
+ "992": 4.90864,
+ "993": 4.9075,
+ "994": 4.90696,
+ "995": 4.90473,
+ "996": 4.90192,
+ "997": 4.90199,
+ "998": 4.89181,
+ "999": 4.89111,
+ "1000": 4.89025,
+ "1001": 4.9168,
+ "1002": 4.90983,
+ "1003": 4.91875,
+ "1004": 4.90892,
+ "1005": 4.92588,
+ "1006": 4.91678,
+ "1007": 5.262,
+ "1008": 4.92447,
+ "1009": 5.26729,
+ "1010": 4.92803,
+ "1011": 4.92461,
+ "1012": 4.92338,
+ "1013": 4.9218,
+ "1014": 4.92051,
+ "1015": 4.92442,
+ "1016": 4.91248,
+ "1017": 4.92113,
+ "1018": 4.92046,
+ "1019": 4.91949,
+ "1020": 4.92623,
+ "1021": 4.92267,
+ "1022": 4.92249,
+ "1023": 4.91899,
+ "1024": 4.92062,
+ "1025": 5.26804,
+ "1026": 4.92131,
+ "1027": 5.26954,
+ "1028": 4.91856,
+ "1029": 4.91681,
+ "1030": 5.90813,
+ "1031": 4.92456,
+ "1032": 4.92325,
+ "1033": 5.3083,
+ "1034": 4.91916,
+ "1035": 4.91422,
+ "1036": 4.91293,
+ "1037": 4.91223,
+ "1038": 4.9211,
+ "1039": 4.92393,
+ "1040": 4.92009,
+ "1041": 4.92106,
+ "1042": 4.9242,
+ "1043": 4.92005,
+ "1044": 5.26878,
+ "1045": 4.92668,
+ "1046": 4.93095,
+ "1047": 5.27312,
+ "1048": 4.92622,
+ "1049": 4.92229,
+ "1050": 4.92078,
+ "1051": 4.9252,
+ "1052": 4.92398,
+ "1053": 4.92467,
+ "1054": 4.92254,
+ "1055": 4.92721,
+ "1056": 4.92594,
+ "1057": 4.93074,
+ "1058": 4.9202,
+ "1059": 4.92339,
+ "1060": 4.92936,
+ "1061": 4.92316,
+ "1062": 4.91832,
+ "1063": 4.9324,
+ "1064": 4.96238,
+ "1065": 4.94321,
+ "1066": 4.96241,
+ "1067": 4.93128,
+ "1068": 4.92665,
+ "1069": 4.93217,
+ "1070": 5.29473,
+ "1071": 5.27044,
+ "1072": 4.91774,
+ "1073": 4.92979,
+ "1074": 5.30092,
+ "1075": 5.57166,
+ "1076": 4.9336,
+ "1077": 4.91975,
+ "1078": 5.29838,
+ "1079": 4.92345,
+ "1080": 4.92265,
+ "1081": 4.93832,
+ "1082": 5.28966,
+ "1083": 4.94183,
+ "1084": 5.28091,
+ "1085": 4.94506,
+ "1086": 4.94668,
+ "1087": 4.94028,
+ "1088": 4.93858,
+ "1089": 4.93937,
+ "1090": 4.9454,
+ "1091": 4.95599,
+ "1092": 4.95023,
+ "1093": 4.94499,
+ "1094": 4.96028,
+ "1095": 4.95213,
+ "1096": 4.96406,
+ "1097": 4.93905,
+ "1098": 4.92198,
+ "1099": 4.93824,
+ "1100": 4.92789,
+ "1101": 4.92981,
+ "1102": 4.93937,
+ "1103": 4.91985,
+ "1104": 4.91889,
+ "1105": 4.93785,
+ "1106": 4.94007,
+ "1107": 4.93618,
+ "1108": 4.94002,
+ "1109": 4.96964,
+ "1110": 4.93965,
+ "1111": 4.89692,
+ "1112": 4.89611,
+ "1113": 4.89245,
+ "1114": 5.24194,
+ "1115": 4.89604,
+ "1116": 5.23738,
+ "1117": 4.89591,
+ "1118": 4.89712,
+ "1119": 6.2207,
+ "1120": 4.89707,
+ "1121": 5.24025,
+ "1122": 4.89987,
+ "1123": 5.27914,
+ "1124": 4.9043,
+ "1125": 4.89477,
+ "1126": 4.89625,
+ "1127": 4.90132,
+ "1128": 4.90216,
+ "1129": 4.90398,
+ "1130": 4.89594,
+ "1131": 4.90153,
+ "1132": 4.89796,
+ "1133": 4.89536,
+ "1134": 4.89807,
+ "1135": 4.89858,
+ "1136": 4.89867,
+ "1137": 4.89681,
+ "1138": 4.92931,
+ "1139": 4.92599,
+ "1140": 4.89538,
+ "1141": 4.89732,
+ "1142": 4.89242,
+ "1143": 4.89262,
+ "1144": 4.89274,
+ "1145": 4.93085,
+ "1146": 4.9294,
+ "1147": 4.92891,
+ "1148": 4.91881,
+ "1149": 4.89129,
+ "1150": 4.89171,
+ "1151": 4.8862,
+ "1152": 4.89315,
+ "1153": 4.89463,
+ "1154": 4.89481,
+ "1155": 4.89194,
+ "1156": 5.23303,
+ "1157": 4.89025,
+ "1158": 4.89312,
+ "1159": 5.24533,
+ "1160": 5.25573,
+ "1161": 5.23949,
+ "1162": 4.8914,
+ "1163": 4.89247,
+ "1164": 4.8896,
+ "1165": 5.88618,
+ "1166": 4.91824,
+ "1167": 4.89232,
+ "1168": 5.27914,
+ "1169": 4.88638,
+ "1170": 4.89624,
+ "1171": 4.90097,
+ "1172": 4.89335,
+ "1173": 4.90022,
+ "1174": 4.88823,
+ "1175": 4.91533,
+ "1176": 4.91702,
+ "1177": 4.91026,
+ "1178": 4.89204,
+ "1179": 4.89341,
+ "1180": 4.88754,
+ "1181": 4.89101,
+ "1182": 4.89528,
+ "1183": 4.89482,
+ "1184": 4.88208,
+ "1185": 4.87829,
+ "1186": 4.88501,
+ "1187": 4.88593,
+ "1188": 4.87526,
+ "1189": 4.88604,
+ "1190": 4.90872,
+ "1191": 4.88218,
+ "1192": 4.8826,
+ "1193": 4.88606,
+ "1194": 5.22378,
+ "1195": 4.88192,
+ "1196": 4.8877,
+ "1197": 5.23842,
+ "1198": 4.89888,
+ "1199": 4.89039,
+ "1200": 4.89543,
+ "1201": 4.8917,
+ "1202": 4.88928,
+ "1203": 4.88428,
+ "1204": 4.91394,
+ "1205": 5.27535,
+ "1206": 5.27273,
+ "1207": 4.92919,
+ "1208": 4.92498,
+ "1209": 5.60645,
+ "1210": 5.23108,
+ "1211": 4.91823,
+ "1212": 4.91107,
+ "1213": 4.90706,
+ "1214": 5.33395,
+ "1215": 4.91341,
+ "1216": 4.92296,
+ "1217": 4.92797,
+ "1218": 4.91436,
+ "1219": 4.93183,
+ "1220": 4.92763,
+ "1221": 4.91189,
+ "1222": 4.91524,
+ "1223": 4.92927,
+ "1224": 4.90762,
+ "1225": 4.91646,
+ "1226": 4.95199,
+ "1227": 4.93657,
+ "1228": 4.91049,
+ "1229": 4.90576,
+ "1230": 4.92418,
+ "1231": 5.24788,
+ "1232": 4.90922,
+ "1233": 4.90828,
+ "1234": 5.28741,
+ "1235": 4.93359,
+ "1236": 4.92651,
+ "1237": 4.92759,
+ "1238": 4.91812,
+ "1239": 4.96161,
+ "1240": 4.92462,
+ "1241": 4.9408,
+ "1242": 4.95151,
+ "1243": 4.92866,
+ "1244": 4.94942,
+ "1245": 4.93202,
+ "1246": 4.93118,
+ "1247": 4.92787,
+ "1248": 4.93195,
+ "1249": 5.31148,
+ "1250": 4.96525,
+ "1251": 5.27677,
+ "1252": 4.95992,
+ "1253": 4.89092,
+ "1254": 5.87598,
+ "1255": 4.89013,
+ "1256": 4.89328,
+ "1257": 4.88679,
+ "1258": 4.89107,
+ "1259": 5.26785,
+ "1260": 4.89071,
+ "1261": 4.89005,
+ "1262": 4.89216,
+ "1263": 4.89212,
+ "1264": 4.88574,
+ "1265": 4.88902,
+ "1266": 4.88642,
+ "1267": 4.89574,
+ "1268": 4.88631,
+ "1269": 5.22724,
+ "1270": 4.88943,
+ "1271": 5.23761,
+ "1272": 4.90353,
+ "1273": 4.89726,
+ "1274": 4.92161,
+ "1275": 4.92347,
+ "1276": 4.91698,
+ "1277": 4.92233,
+ "1278": 4.91979,
+ "1279": 4.9211,
+ "1280": 4.9179,
+ "1281": 4.92209,
+ "1282": 4.94485,
+ "1283": 4.92932,
+ "1284": 4.92976,
+ "1285": 4.91788,
+ "1286": 4.93408,
+ "1287": 4.92359,
+ "1288": 4.92166,
+ "1289": 4.9185,
+ "1290": 4.91424,
+ "1291": 4.91891,
+ "1292": 4.92028,
+ "1293": 4.9117,
+ "1294": 5.27044,
+ "1295": 5.29676,
+ "1296": 4.91703,
+ "1297": 4.92056,
+ "1298": 4.92207,
+ "1299": 5.91394,
+ "1300": 4.9147,
+ "1301": 4.9131,
+ "1302": 4.9176,
+ "1303": 4.93425,
+ "1304": 5.304,
+ "1305": 4.91978,
+ "1306": 5.27498,
+ "1307": 4.92043,
+ "1308": 4.91675,
+ "1309": 5.27831,
+ "1310": 4.93667,
+ "1311": 4.93075,
+ "1312": 4.92766,
+ "1313": 4.92554,
+ "1314": 4.93753,
+ "1315": 4.93323,
+ "1316": 4.92326,
+ "1317": 4.92226,
+ "1318": 4.9254,
+ "1319": 4.91683,
+ "1320": 4.91352,
+ "1321": 4.93361,
+ "1322": 4.9202,
+ "1323": 4.92888,
+ "1324": 4.94749,
+ "1325": 4.92427,
+ "1326": 4.91993,
+ "1327": 4.94147,
+ "1328": 4.91569,
+ "1329": 4.9082,
+ "1330": 4.90808,
+ "1331": 4.92463,
+ "1332": 4.94304,
+ "1333": 4.91833,
+ "1334": 4.91915,
+ "1335": 4.9569,
+ "1336": 4.91253,
+ "1337": 4.91228,
+ "1338": 4.91599,
+ "1339": 5.26886,
+ "1340": 4.94108,
+ "1341": 5.28895,
+ "1342": 4.92166,
+ "1343": 4.93148,
+ "1344": 6.20454,
+ "1345": 4.93732,
+ "1346": 4.94109,
+ "1347": 5.28178,
+ "1348": 4.92597,
+ "1349": 5.31528,
+ "1350": 4.93124,
+ "1351": 4.9199,
+ "1352": 4.92145,
+ "1353": 4.91761,
+ "1354": 4.91599,
+ "1355": 4.91867,
+ "1356": 4.92286,
+ "1357": 4.91965,
+ "1358": 4.92454,
+ "1359": 4.92188,
+ "1360": 4.91921,
+ "1361": 4.92021,
+ "1362": 4.92372,
+ "1363": 4.91207,
+ "1364": 4.96107,
+ "1365": 4.91388,
+ "1366": 4.91683,
+ "1367": 4.91413,
+ "1368": 4.91691,
+ "1369": 4.91871,
+ "1370": 4.92278,
+ "1371": 4.92605,
+ "1372": 4.92653,
+ "1373": 4.9264,
+ "1374": 4.92864,
+ "1375": 4.92839,
+ "1376": 4.93185,
+ "1377": 4.92304,
+ "1378": 4.92916,
+ "1379": 4.92701,
+ "1380": 4.92797,
+ "1381": 5.27325,
+ "1382": 4.89544,
+ "1383": 4.89064,
+ "1384": 5.60494,
+ "1385": 5.00482,
+ "1386": 5.33879,
+ "1387": 4.92912,
+ "1388": 4.92575,
+ "1389": 5.83703,
+ "1390": 4.91691,
+ "1391": 4.91717,
+ "1392": 4.92005,
+ "1393": 4.92211,
+ "1394": 4.91895,
+ "1395": 5.29903,
+ "1396": 4.92143,
+ "1397": 4.91551,
+ "1398": 4.91427,
+ "1399": 4.91348,
+ "1400": 4.92556,
+ "1401": 4.92553,
+ "1402": 4.91884,
+ "1403": 4.91856,
+ "1404": 4.95579,
+ "1405": 4.88917,
+ "1406": 4.88886,
+ "1407": 4.90262,
+ "1408": 4.88379,
+ "1409": 4.88976,
+ "1410": 4.88681,
+ "1411": 4.8751,
+ "1412": 4.89308,
+ "1413": 4.89122,
+ "1414": 4.88458,
+ "1415": 4.89489,
+ "1416": 4.88438,
+ "1417": 4.88183,
+ "1418": 5.229,
+ "1419": 4.96736,
+ "1420": 4.95735,
+ "1421": 5.29839,
+ "1422": 4.92896,
+ "1423": 4.9679,
+ "1424": 4.96109,
+ "1425": 4.96048,
+ "1426": 4.95854,
+ "1427": 4.95558,
+ "1428": 4.90503,
+ "1429": 5.24486,
+ "1430": 5.24901,
+ "1431": 4.8987,
+ "1432": 4.89075,
+ "1433": 5.22736,
+ "1434": 5.47175,
+ "1435": 4.89209,
+ "1436": 4.8986,
+ "1437": 4.8891,
+ "1438": 4.88697,
+ "1439": 4.88974,
+ "1440": 5.27298,
+ "1441": 4.89403,
+ "1442": 4.90495,
+ "1443": 4.89585,
+ "1444": 4.89766,
+ "1445": 4.89344,
+ "1446": 4.89618,
+ "1447": 4.88721,
+ "1448": 4.88735,
+ "1449": 4.89401,
+ "1450": 4.89435,
+ "1451": 4.89143,
+ "1452": 4.88553,
+ "1453": 4.89139,
+ "1454": 4.89347,
+ "1455": 5.23147,
+ "1456": 4.8987,
+ "1457": 4.90447,
+ "1458": 4.89553,
+ "1459": 5.23187,
+ "1460": 4.90546,
+ "1461": 4.89293,
+ "1462": 4.89652,
+ "1463": 4.88806,
+ "1464": 4.94852,
+ "1465": 4.89339,
+ "1466": 4.88888,
+ "1467": 4.89409,
+ "1468": 4.89028,
+ "1469": 4.89198,
+ "1470": 4.89499,
+ "1471": 4.89853,
+ "1472": 4.89989,
+ "1473": 5.245,
+ "1474": 4.89244,
+ "1475": 5.24744,
+ "1476": 4.88786,
+ "1477": 4.88954,
+ "1478": 5.81074,
+ "1479": 4.90603,
+ "1480": 4.8817,
+ "1481": 4.88853,
+ "1482": 4.88913,
+ "1483": 4.88525,
+ "1484": 4.88091,
+ "1485": 5.26103,
+ "1486": 4.88332,
+ "1487": 4.88482,
+ "1488": 4.88349,
+ "1489": 4.93535,
+ "1490": 4.93713,
+ "1491": 4.94008,
+ "1492": 4.93273,
+ "1493": 5.26558,
+ "1494": 4.92625,
+ "1495": 4.93119,
+ "1496": 4.93326,
+ "1497": 5.29661,
+ "1498": 4.94651,
+ "1499": 4.94563,
+ "1500": 4.94732,
+ "1501": 4.94956,
+ "1502": 4.93949,
+ "1503": 4.94314,
+ "1504": 4.949,
+ "1505": 4.93848,
+ "1506": 4.93655,
+ "1507": 4.93352,
+ "1508": 4.93376,
+ "1509": 4.93575,
+ "1510": 4.93237,
+ "1511": 4.93325,
+ "1512": 4.93443,
+ "1513": 4.93608,
+ "1514": 4.92875,
+ "1515": 4.93822,
+ "1516": 4.92271,
+ "1517": 4.93602,
+ "1518": 4.93135,
+ "1519": 5.28269,
+ "1520": 5.28601,
+ "1521": 4.93214,
+ "1522": 4.93238,
+ "1523": 4.9331,
+ "1524": 5.84985,
+ "1525": 4.93183,
+ "1526": 4.9312,
+ "1527": 4.94067,
+ "1528": 4.94179,
+ "1529": 4.93283,
+ "1530": 5.64255,
+ "1531": 4.93012,
+ "1532": 4.93237,
+ "1533": 4.93188,
+ "1534": 5.28642,
+ "1535": 4.93295,
+ "1536": 4.93351,
+ "1537": 4.93687,
+ "1538": 4.93395,
+ "1539": 4.93892,
+ "1540": 4.93329,
+ "1541": 4.93178,
+ "1542": 4.94011,
+ "1543": 4.93223,
+ "1544": 4.9238,
+ "1545": 4.93295,
+ "1546": 4.92789,
+ "1547": 4.92723,
+ "1548": 4.93344,
+ "1549": 4.93081,
+ "1550": 4.93484,
+ "1551": 4.93247,
+ "1552": 4.94286,
+ "1553": 4.93871,
+ "1554": 4.9346,
+ "1555": 4.93508,
+ "1556": 4.93254,
+ "1557": 4.93621,
+ "1558": 4.93402,
+ "1559": 4.92552,
+ "1560": 4.92871,
+ "1561": 4.9342,
+ "1562": 4.93981,
+ "1563": 4.94231,
+ "1564": 5.28559,
+ "1565": 5.2926,
+ "1566": 4.93393,
+ "1567": 5.27554,
+ "1568": 5.55669,
+ "1569": 5.22897,
+ "1570": 4.93426,
+ "1571": 5.28382,
+ "1572": 4.94938,
+ "1573": 4.95055,
+ "1574": 4.94811,
+ "1575": 4.9489,
+ "1576": 5.33208,
+ "1577": 4.94524,
+ "1578": 4.94592,
+ "1579": 4.94832,
+ "1580": 4.94832,
+ "1581": 4.94408,
+ "1582": 4.93963,
+ "1583": 5.06791,
+ "1584": 4.93161,
+ "1585": 4.93335,
+ "1586": 4.93849,
+ "1587": 4.93237,
+ "1588": 4.93556,
+ "1589": 4.93066,
+ "1590": 4.94768,
+ "1591": 4.93099,
+ "1592": 4.93258,
+ "1593": 4.93981,
+ "1594": 4.92949,
+ "1595": 4.93453,
+ "1596": 4.92827,
+ "1597": 4.92584,
+ "1598": 4.93755,
+ "1599": 4.92974,
+ "1600": 4.94804,
+ "1601": 4.93191,
+ "1602": 4.93369,
+ "1603": 4.93286,
+ "1604": 4.93069,
+ "1605": 5.27051,
+ "1606": 4.92329,
+ "1607": 4.92495,
+ "1608": 5.27779,
+ "1609": 5.28346,
+ "1610": 5.29602,
+ "1611": 4.94123,
+ "1612": 4.93638,
+ "1613": 5.856,
+ "1614": 4.94437,
+ "1615": 4.93653,
+ "1616": 4.93875,
+ "1617": 4.93536,
+ "1618": 4.93896,
+ "1619": 4.93356,
+ "1620": 4.93572,
+ "1621": 5.31736,
+ "1622": 4.94531,
+ "1623": 4.94225,
+ "1624": 4.94386,
+ "1625": 4.93406,
+ "1626": 4.93798,
+ "1627": 4.93633,
+ "1628": 4.93917,
+ "1629": 4.93696,
+ "1630": 4.93053,
+ "1631": 4.92648,
+ "1632": 4.92658,
+ "1633": 4.93841,
+ "1634": 4.93342,
+ "1635": 4.9359,
+ "1636": 4.93181,
+ "1637": 4.93503,
+ "1638": 4.93642,
+ "1639": 4.93683,
+ "1640": 4.93436,
+ "1641": 4.9443,
+ "1642": 5.27794,
+ "1643": 4.94268,
+ "1644": 4.91864,
+ "1645": 4.92135,
+ "1646": 5.26653,
+ "1647": 4.93155,
+ "1648": 4.94793,
+ "1649": 4.92681,
+ "1650": 4.92909,
+ "1651": 4.92222,
+ "1652": 4.93308,
+ "1653": 5.27802,
+ "1654": 5.27831,
+ "1655": 4.92527,
+ "1656": 4.92184,
+ "1657": 4.92535,
+ "1658": 5.84478,
+ "1659": 4.93415,
+ "1660": 4.98533,
+ "1661": 4.95752,
+ "1662": 4.94766,
+ "1663": 4.94933,
+ "1664": 4.95355,
+ "1665": 4.94643,
+ "1666": 5.33217,
+ "1667": 4.93611,
+ "1668": 4.93532,
+ "1669": 4.9092,
+ "1670": 4.90894,
+ "1671": 4.9204,
+ "1672": 4.92236,
+ "1673": 4.9082,
+ "1674": 4.91286,
+ "1675": 4.90919,
+ "1676": 4.90864,
+ "1677": 4.91312,
+ "1678": 4.90871,
+ "1679": 4.92308,
+ "1680": 5.26267,
+ "1681": 4.92022,
+ "1682": 4.91096,
+ "1683": 4.91568,
+ "1684": 5.26065,
+ "1685": 4.90909,
+ "1686": 4.90718,
+ "1687": 4.91023,
+ "1688": 4.91504,
+ "1689": 4.9123,
+ "1690": 4.91353,
+ "1691": 4.90838,
+ "1692": 4.90311,
+ "1693": 4.90235,
+ "1694": 4.90376,
+ "1695": 4.90901,
+ "1696": 4.90724,
+ "1697": 4.91094,
+ "1698": 5.25776,
+ "1699": 4.91455,
+ "1700": 5.2613,
+ "1701": 4.90973,
+ "1702": 4.90149,
+ "1703": 5.82797,
+ "1704": 4.9102,
+ "1705": 4.91831,
+ "1706": 4.90187,
+ "1707": 4.89945,
+ "1708": 4.89865,
+ "1709": 4.89632,
+ "1710": 4.90065,
+ "1711": 5.28146,
+ "1712": 4.90271,
+ "1713": 4.90852,
+ "1714": 4.90365,
+ "1715": 4.90463,
+ "1716": 4.91059,
+ "1717": 5.24655,
+ "1718": 4.91868,
+ "1719": 4.90569,
+ "1720": 4.91426,
+ "1721": 4.91116,
+ "1722": 5.25454,
+ "1723": 4.91058,
+ "1724": 4.90906,
+ "1725": 4.92075,
+ "1726": 4.91839,
+ "1727": 4.91564,
+ "1728": 4.91131,
+ "1729": 4.91291,
+ "1730": 4.90884,
+ "1731": 4.91062,
+ "1732": 4.90638,
+ "1733": 4.9061,
+ "1734": 4.90658,
+ "1735": 4.91543,
+ "1736": 4.90614,
+ "1737": 4.91107,
+ "1738": 4.91084,
+ "1739": 4.90842,
+ "1740": 4.91418,
+ "1741": 4.90881,
+ "1742": 4.90792,
+ "1743": 5.26397,
+ "1744": 4.91738,
+ "1745": 5.25587,
+ "1746": 4.90599,
+ "1747": 4.90321,
+ "1748": 5.78796,
+ "1749": 4.90348,
+ "1750": 4.90858,
+ "1751": 4.89993,
+ "1752": 4.90938,
+ "1753": 4.90593,
+ "1754": 5.25406,
+ "1755": 4.9167,
+ "1756": 4.92732,
+ "1757": 5.32154,
+ "1758": 4.93234,
+ "1759": 5.25874,
+ "1760": 4.90683,
+ "1761": 4.90629,
+ "1762": 4.91525,
+ "1763": 4.91544,
+ "1764": 4.91062,
+ "1765": 4.90636,
+ "1766": 4.90873,
+ "1767": 4.91142,
+ "1768": 4.96573,
+ "1769": 4.90448,
+ "1770": 4.8891,
+ "1771": 4.8932,
+ "1772": 4.88066,
+ "1773": 4.87927,
+ "1774": 4.87496,
+ "1775": 4.90017,
+ "1776": 4.88861,
+ "1777": 4.88943,
+ "1778": 4.88632,
+ "1779": 4.89539,
+ "1780": 4.88673,
+ "1781": 4.89482,
+ "1782": 4.89261,
+ "1783": 4.88921,
+ "1784": 4.89935,
+ "1785": 4.88986,
+ "1786": 4.89061,
+ "1787": 4.88853,
+ "1788": 5.24035,
+ "1789": 5.24993,
+ "1790": 4.91207,
+ "1791": 4.91991,
+ "1792": 5.55415,
+ "1793": 5.49039,
+ "1794": 4.899,
+ "1795": 4.88922,
+ "1796": 5.25127,
+ "1797": 4.89889,
+ "1798": 4.90442,
+ "1799": 4.89627,
+ "1800": 4.89346,
+ "1801": 4.89082,
+ "1802": 5.2731,
+ "1803": 4.89886,
+ "1804": 4.87379,
+ "1805": 4.87577,
+ "1806": 4.88484,
+ "1807": 4.87576,
+ "1808": 4.86783,
+ "1809": 4.8917,
+ "1810": 4.87329,
+ "1811": 4.87182,
+ "1812": 4.8594,
+ "1813": 4.86213,
+ "1814": 4.86701,
+ "1815": 4.86025,
+ "1816": 4.86454,
+ "1817": 4.86162,
+ "1818": 4.85688,
+ "1819": 4.85907,
+ "1820": 4.85765,
+ "1821": 4.85878,
+ "1822": 4.86537,
+ "1823": 4.86101,
+ "1824": 4.86218,
+ "1825": 4.86082,
+ "1826": 4.85916,
+ "1827": 4.86304,
+ "1828": 4.86335,
+ "1829": 4.85846,
+ "1830": 5.21054,
+ "1831": 4.87227,
+ "1832": 5.20618,
+ "1833": 4.86815,
+ "1834": 5.55416,
+ "1835": 4.87798,
+ "1836": 4.89752,
+ "1837": 5.79486,
+ "1838": 4.90553,
+ "1839": 4.90533,
+ "1840": 4.89368,
+ "1841": 4.89475,
+ "1842": 4.89469,
+ "1843": 4.88557,
+ "1844": 4.89,
+ "1845": 4.88668,
+ "1846": 4.89537,
+ "1847": 5.26263,
+ "1848": 4.89245,
+ "1849": 4.89348,
+ "1850": 4.88835,
+ "1851": 4.90708,
+ "1852": 4.90228,
+ "1853": 4.86785,
+ "1854": 4.87736,
+ "1855": 4.87369,
+ "1856": 4.87811,
+ "1857": 4.90299,
+ "1858": 4.88442,
+ "1859": 4.87297,
+ "1860": 4.89531,
+ "1861": 4.90241,
+ "1862": 4.89309,
+ "1863": 4.89512,
+ "1864": 4.90549,
+ "1865": 4.90854,
+ "1866": 4.9047,
+ "1867": 5.2401,
+ "1868": 4.89946,
+ "1869": 4.90883,
+ "1870": 4.90522,
+ "1871": 4.93888,
+ "1872": 5.21372,
+ "1873": 4.87709,
+ "1874": 4.86464,
+ "1875": 4.87233,
+ "1876": 4.88054,
+ "1877": 4.84923,
+ "1878": 5.17207,
+ "1879": 5.1976,
+ "1880": 4.8445,
+ "1881": 4.84388,
+ "1882": 4.84797,
+ "1883": 5.73664,
+ "1884": 4.84672,
+ "1885": 4.84557,
+ "1886": 4.85201,
+ "1887": 4.85018,
+ "1888": 4.84932,
+ "1889": 4.85617,
+ "1890": 4.84416,
+ "1891": 4.85089,
+ "1892": 4.84881,
+ "1893": 5.22668,
+ "1894": 4.8491,
+ "1895": 4.84681,
+ "1896": 4.84529,
+ "1897": 4.84998,
+ "1898": 4.8507,
+ "1899": 4.84271,
+ "1900": 4.84844,
+ "1901": 4.84365,
+ "1902": 4.83991,
+ "1903": 4.84228,
+ "1904": 5.17846,
+ "1905": 4.84978,
+ "1906": 4.84285,
+ "1907": 4.85138,
+ "1908": 4.84338,
+ "1909": 5.19721,
+ "1910": 4.85138,
+ "1911": 4.84739,
+ "1912": 4.84478,
+ "1913": 4.85226,
+ "1914": 4.85002,
+ "1915": 4.85039,
+ "1916": 4.85444,
+ "1917": 4.84588,
+ "1918": 4.8495,
+ "1919": 4.85217,
+ "1920": 4.84949,
+ "1921": 4.84631,
+ "1922": 4.84476,
+ "1923": 5.17493,
+ "1924": 5.19107,
+ "1925": 4.85154,
+ "1926": 4.84261,
+ "1927": 5.44494,
+ "1928": 5.14044,
+ "1929": 4.84927,
+ "1930": 4.84493,
+ "1931": 4.84048,
+ "1932": 4.84204,
+ "1933": 4.84664,
+ "1934": 4.84105,
+ "1935": 4.83981,
+ "1936": 4.841,
+ "1937": 4.84038,
+ "1938": 5.22894,
+ "1939": 4.84209,
+ "1940": 4.84356,
+ "1941": 5.20657,
+ "1942": 4.9004,
+ "1943": 4.90813,
+ "1944": 4.90655,
+ "1945": 4.88214,
+ "1946": 5.21239,
+ "1947": 4.86529,
+ "1948": 4.85849,
+ "1949": 4.85084,
+ "1950": 4.86533,
+ "1951": 4.86,
+ "1952": 4.85847,
+ "1953": 4.86113,
+ "1954": 4.85194,
+ "1955": 4.85611,
+ "1956": 4.87124,
+ "1957": 4.8777,
+ "1958": 4.84686,
+ "1959": 4.84732,
+ "1960": 4.86364,
+ "1961": 4.8509,
+ "1962": 4.8663,
+ "1963": 4.87064,
+ "1964": 4.86099,
+ "1965": 4.86103,
+ "1966": 4.84569,
+ "1967": 5.17792,
+ "1968": 4.84796,
+ "1969": 5.20648,
+ "1970": 4.84901,
+ "1971": 4.84838,
+ "1972": 5.74018,
+ "1973": 4.85813,
+ "1974": 4.85367,
+ "1975": 4.86684,
+ "1976": 4.87041,
+ "1977": 4.90603,
+ "1978": 4.90475,
+ "1979": 5.25145,
+ "1980": 4.94444,
+ "1981": 4.92124,
+ "1982": 4.90832,
+ "1983": 4.94722,
+ "1984": 5.67636,
+ "1985": 4.939,
+ "1986": 4.93543,
+ "1987": 4.96136,
+ "1988": 4.92447,
+ "1989": 4.87603,
+ "1990": 4.86128,
+ "1991": 4.86822,
+ "1992": 4.86666,
+ "1993": 4.85995,
+ "1994": 4.86025,
+ "1995": 4.85738,
+ "1996": 4.86953,
+ "1997": 4.86535,
+ "1998": 4.86591,
+ "1999": 4.86231,
+ "2000": 4.86466
+ }
+ }
+}
\ No newline at end of file
diff --git a/tests/functional_tests/test_cases/gpt/gpt_dynamic_inference_tp1_pp1_dp8_583m_throughputtest_zmq/golden_values_dev_dgx_h100.json b/tests/functional_tests/test_cases/gpt/gpt_dynamic_inference_tp1_pp1_dp8_583m_throughputtest_zmq/golden_values_dev_dgx_h100.json
index 9be8a9dc0ca..b31640a2a28 100644
--- a/tests/functional_tests/test_cases/gpt/gpt_dynamic_inference_tp1_pp1_dp8_583m_throughputtest_zmq/golden_values_dev_dgx_h100.json
+++ b/tests/functional_tests/test_cases/gpt/gpt_dynamic_inference_tp1_pp1_dp8_583m_throughputtest_zmq/golden_values_dev_dgx_h100.json
@@ -1,1028 +1,1028 @@
{
"throughput": [
- 94.6087716527102,
- 115.85992244026639,
- 138.9562527069375,
- 133.18726531918395,
- 81.97861561771212,
- 134.30726469422635,
- 86.456140428456,
- 114.99456351298251,
- 147.3101800153954,
- 3.0364623744653003,
- 124.7590786954667,
- 134.2276982994434,
- 3.0580463134110167,
- 117.03969654341354,
- 130.92134521286803,
- 48.493091604204935,
- 1.4498729599486508,
- 128.01470907994928,
- 1.8330770354872434,
- 66.31842482241125,
- 82.24189975425459,
- 1.07058112939944,
- 1.8815468970982412,
- 0.9373246942729808,
- 134.9963160815443,
- 2.285771114682068,
- 43.068220270070434,
- 134.9677086822377,
- 82.44946740133796,
- 47.71839155542011,
- 114.4199568886962,
- 29.67621576315833,
- 144.1589742491705,
- 95.8164720809401,
- 122.80562228460093,
- 39.21436814433054,
- 3.041180292262413,
- 3.2867844729646842,
- 72.43808226229888,
- 0.8371525937296347,
- 1.2212635079980698,
- 145.6869075644325,
- 42.317711349146016,
- 109.1196064871946,
- 73.6281770453198,
- 140.4495689387567,
- 1.219834296561022,
- 138.66856497329005,
- 23.33818821323391,
- 67.82342558671365,
- 130.09683254313987,
- 147.60199288178146,
- 0.9427431720755464,
- 3.2856495013162523,
- 79.12426666101076,
- 86.41557345094756,
- 120.17346279825053,
- 137.16615251640926,
- 108.93291864542198,
- 110.10504114490513,
- 46.19253755421628,
- 0.950218846923012,
- 136.50642826951463,
- 142.73168666846448,
- 1.2206786818073785,
- 1.898581377105612,
- 131.72636154091063,
- 2.2842414327001976,
- 89.76521170090028,
- 114.66053545744656,
- 58.64474290044525,
- 0.8367865961030284,
- 128.01767795820945,
- 60.87292097103301,
- 124.20016865241587,
- 119.59336898055426,
- 0.9425820346281929,
- 93.70053305431952,
- 1.0728113870213674,
- 135.7596767309971,
- 112.89357243644062,
- 89.2743296587299,
- 137.86411291342458,
- 135.6974706051771,
- 102.59633828443238,
- 129.82058179399326,
- 139.57672703148444,
- 140.5642311163746,
- 78.49182953675201,
- 123.40912657074227,
- 82.74099904578694,
- 75.5490641626476,
- 93.38596238341951,
- 141.19058076067225,
- 1.072254167577298,
- 100.8669047802279,
- 132.77382347347034,
- 92.29086179175866,
- 137.20301032384705,
- 89.57723938765776,
- 67.5465256589703,
- 0.9498935124108836,
- 1.0716887464650027,
- 0.8365472180547067,
- 137.902625307774,
- 132.67132600219722,
- 1.45201860416265,
- 1.8366476879619427,
- 88.65095604379363,
- 132.1806036761347,
- 126.0481874394642,
- 127.43750324083169,
- 93.27238135265156,
- 109.83884164204308,
- 102.30516355984702,
- 141.10387096377744,
- 0.9425154448032942,
- 95.04281981148903,
- 103.11525529548061,
- 0.8361762901534399,
- 135.3171561172067,
- 123.30032998064965,
- 118.75691144485415,
- 82.21375599642211,
- 66.37216333263251,
- 120.02349229491865,
- 27.339414655466246,
- 133.1312422227687,
- 123.02377779863252,
- 111.0798894329,
- 58.88405247768833,
- 131.31767475108893,
- 40.19076958615912,
- 123.58362152151858,
- 130.6541142941889,
- 61.39555613504246,
- 43.92154495664044,
- 1.037012527495492,
- 127.16052127606021,
- 137.06554800183082,
- 85.67161160523041,
- 1.0253417447981334,
- 139.20903624514017,
- 140.19068787455728,
- 117.67416498245059,
- 23.410837515725987,
- 130.73052473972666,
- 22.561824695346466,
- 1.028901717647808,
- 119.30712483977753,
- 117.77548263464804,
- 135.2959098119142,
- 142.10193821260228,
- 1.0366044325624144,
- 1.0350271698893887,
- 132.8943567509843,
- 51.50353963446039,
- 113.39559408843714,
- 124.25424103796537,
- 129.60407993083075,
- 136.8566687186031,
- 1.036163010240988,
- 1.0345739017743927,
- 118.72350056844492,
- 32.453707095990595,
- 43.851925176925825,
- 139.39206855448938,
- 141.0979597861742,
- 132.81461728578432,
- 80.95956255477945,
- 133.42483643501154,
- 57.27721135575491,
- 81.47649794801364,
- 79.39765285063396,
- 56.40255861789973,
- 0.8890603607397893,
- 137.59325887086797,
- 118.03982850100024,
- 53.04390121587005,
- 88.31177924841927,
- 1.0287550608831881,
- 54.67393025836421,
- 54.73556135447348,
- 129.6143036059356,
- 123.57095756116274,
- 146.05184555314386,
- 55.506024155977386,
- 84.40666358740559,
- 62.68531518105107,
- 147.42894642823578,
- 1.0274253590993496,
- 145.9063526676371,
- 76.36231256557768,
- 1.035808949157935,
- 136.1858098182613,
- 93.13144140533397,
- 54.57886608953819,
- 1.0251956490815057,
- 1.0270063804838983,
- 67.96952180390161,
- 136.90103479290272,
- 78.62986077133174,
- 129.97235998681177,
- 70.57784076609056,
- 1.028567312218149,
- 69.64434330087829,
- 1.0266016363366386,
- 25.142311727265525,
- 139.54750333578679,
- 118.80547132463877,
- 1.0342055876192149,
- 132.79991800938092,
- 88.25494664060619,
- 132.4600307114398,
- 1.026200775415348,
- 111.33264788932784,
- 1.031301270403004,
- 104.45912302410692,
- 1.0337771723701492,
- 124.53550504281608,
- 1.0283501183885058,
- 126.53361938982871,
- 139.83512785200963,
- 102.28350299734186,
- 122.68389734539087,
- 139.27095111763788,
- 1.0333552237490158,
- 97.04945381465573,
- 60.63422077140298,
- 1.0248694052483192,
- 96.77644543721476,
- 118.38370846079931,
- 1.0309087229819596,
- 136.0487423665781,
- 1.032932214377732,
- 104.96525711514936,
- 50.75370028394122,
- 125.67617176346853,
- 125.47392048276225,
- 101.59371483024698,
- 119.1183231384482,
- 134.24568445137294,
- 1.0323996653747745,
- 119.28563313083153,
- 50.183581144589674,
- 107.50817556608582,
- 127.4693561344537,
- 116.0234844098742,
- 149.0429439759437,
- 127.77855747904051,
- 1.0319900690130652,
- 129.7400124946839,
- 60.27584011696136,
- 1.0245534026749026,
- 113.8687773549026,
- 129.9927880985222,
- 41.55332067297356,
- 12.991853549713621,
- 144.9384518471586,
- 127.77570879015505,
- 79.09214991388126,
- 1.0326234729165304,
- 144.50618896622706,
- 44.461452482592826,
- 145.75357879817352,
- 150.5618330832813,
- 123.17802281879979,
- 147.0133924731902,
- 57.07203337285457,
- 140.17944630269687,
- 44.5066568841284,
- 150.2834791394652,
- 146.37106237628518,
- 135.59553639884948,
- 21.91845075979551,
- 1.0391172002596458,
- 92.42182316100705,
- 14.98578222593142,
- 19.944740287073653,
- 32.75622847272977,
- 58.94666795839769,
- 1.0428676908165904,
- 97.94938911630567,
- 140.5399781540016,
- 36.397689902912774,
- 1.0322919875583962,
- 33.76444948259586,
- 147.54902815924785,
- 51.316830076622495,
- 153.55703202636914,
- 46.423895018386204,
- 140.271682540213,
- 1.0340651759548871,
- 85.22971449383292,
- 141.80480996358014,
- 1.0234621691055457,
- 1.0355322329825165,
- 136.96321865236195,
- 138.2293990177049,
- 136.89440582973347,
- 96.94919171687799,
- 54.992986423891566,
- 142.91167590864902,
- 138.73615931624403,
- 86.32837448704223,
- 1.0424247604140402,
- 127.58052889290863,
- 138.2472241943501,
- 1.0338260095695477,
- 1.0317372756221133,
- 150.59249576769173,
- 1.0229533138894364,
- 149.1711141084735,
- 1.0419379125129562,
- 1.040305113121658,
- 150.13261057757276,
- 62.47975017460808,
- 70.20443057037575,
- 76.88821624674898,
- 1.0225242667788867,
- 136.83301633777177,
- 1.0414381555227956,
- 131.6044067829552,
- 1.038902005769604,
- 1.0335832618537684,
- 83.38230404797935,
- 3.047737981863063,
- 140.9843162162637,
- 1.0352264324041114,
- 1.0409374510445146,
- 103.17228299164871,
- 1.0383219913492376,
- 67.5151836065632,
- 126.94018489907108,
- 95.29974174831813,
- 1.022161551972834,
- 1.0348032799350415,
- 93.24855217625235,
- 140.00831851627856,
- 142.46553219867087,
- 80.52507876480331,
- 149.47939431741142,
- 125.60095189608528,
- 92.57991472689042,
- 153.09192667088175,
- 98.78787611117323,
- 136.9802701171813,
- 1.0378200246498124,
- 79.05370338483348,
- 145.63143231877774,
- 107.86253722014555,
- 113.1390555766259,
- 150.4596904971142,
- 6.010262757833046,
- 138.11675690694213,
- 1.0371929842524894,
- 55.1702723554103,
- 148.4142582794926,
- 108.62464742566522,
- 142.2515578682958,
- 149.5588988951372,
- 1.0310870179234204,
- 32.798276334675066,
- 145.8363475163408,
- 82.52497836005318,
- 144.77105210255448,
- 140.95035733017403,
- 145.4844811663436,
- 145.0646083055648,
- 139.1641494303434,
- 1.0401220454548914,
- 146.10598185112948,
- 1.0335329080843159,
- 1.0316085392161136,
- 133.98012837767038,
- 129.62059667226987,
- 151.2681266565858,
- 1.030719335336581,
- 135.9600336007384,
- 1.0366589924031362,
- 107.70864165999221,
- 118.06361914834272,
- 148.4615541738592,
- 135.1206190516379,
- 1.0788915925864082,
- 1.0662361391973343,
- 1.0784094142292293,
- 145.5492563111853,
- 100.1745158858024,
- 89.97448812790176,
- 140.13008352060388,
- 8.378443606045758,
- 19.841723966559687,
- 31.11972559764219,
- 127.75589035167928,
- 144.649118240912,
- 83.40454687650907,
- 13.609558087727212,
- 144.14916775068022,
- 143.0831699051951,
- 144.53789580070173,
- 129.35689525213576,
- 126.54760361436873,
- 136.72725454688293,
- 83.66753329456253,
- 35.238850690537326,
- 138.73588075606074,
- 148.39285997484404,
- 141.43706957675556,
- 35.20788617289704,
- 140.22918428708584,
- 141.42288954532623,
- 80.8071906111917,
- 53.480908541665116,
- 96.60869116876205,
- 138.83030943256392,
- 146.89537016655746,
- 1.0659353965573166,
- 138.66041009897964,
- 138.0783824554628,
- 54.95061283513892,
- 1.0688789370964418,
- 145.4981195236156,
- 107.91672388693667,
- 147.39387423946786,
- 143.49840246862203,
- 1.0781871694837721,
- 125.37215873599833,
- 46.390553110182545,
- 1.0683430650310588,
- 60.55314896188811,
- 128.32962060837178,
- 142.6648214311374,
- 1.065532502621677,
- 145.06202945295232,
- 149.5985088362253,
- 43.61426254132819,
- 139.2120402464869,
- 138.80120892663803,
- 142.59390751862693,
- 147.27000174003754,
- 139.5980537408405,
- 142.37081759892675,
- 76.47257166426981,
- 0.8663971721944621,
- 1.067847671923619,
- 1.0752972325757186,
- 139.11225337731244,
- 154.1012640338781,
- 91.85315813315137,
- 7.34066705730821,
- 1.0763437477764217,
- 56.03391448680589,
- 1.067309924884827,
- 1.0747789028833068,
- 1.057667310022394,
- 146.4284745539176,
- 142.32867288307636,
- 132.81801172672715,
- 142.5746724111237,
- 43.178263922620026,
- 140.19958418325498,
- 1.0742201855279276,
- 139.95237701874325,
- 124.69044225989671,
- 89.93275546978569,
- 1.0778110524743836,
- 108.03753008375865,
- 0.8649825661375887,
- 101.22782607000799,
- 138.6615942910557,
- 1.0572642952018412,
- 143.509260845593,
- 1.0651693329533294,
- 97.454990956795,
- 1.075960473594851,
- 104.89429761368234,
- 153.46849816095335,
- 143.28204379991922,
- 112.57923589922926,
- 145.35468060283986,
- 119.53338040876814,
- 132.53105489182144,
- 146.60735281445733,
- 0.8648000721123511,
- 132.61504628627392,
- 140.81953388748138,
- 1.05684091289561,
- 147.29646966899597,
- 1.0646855258714663,
- 1.0772400203863821,
- 137.87592499226204,
- 101.79954304062817,
- 134.45893707567646,
- 1.0737967838723397,
- 147.3289039421509,
- 142.95955673278567,
- 123.11846557585149,
- 139.7223884224781,
- 5.274894457437767,
- 0.8646226703470901,
- 135.27010135142623,
- 134.53222451904563,
- 140.4520894166607,
- 148.6784682726068,
- 148.83999547746723,
- 144.76059628877204,
- 146.09818079047014,
- 0.8644123666240657,
- 133.05795012757028,
- 141.21253159110282,
- 147.08086640702987,
- 153.13511211461227,
- 147.72437078211334,
- 53.87242850230838,
- 61.34701685378028,
- 74.50771860339175,
- 16.40780504974564,
- 16.448796993269678,
- 144.08505364828036,
- 143.78069847853888,
- 145.08382905436133,
- 139.4144567792124,
- 1.113422304912727,
- 23.732299099149245,
- 146.716938504402,
- 1.1150428401994323,
- 1.1070863332993708,
- 147.462815334713,
- 15.300506166735937,
- 142.89311901203018,
- 35.881455163220174,
- 0.8959120615185874,
- 134.50389621984408,
- 79.91603718165896,
- 145.31776951960734,
- 153.19384567886857,
- 142.494036234602,
- 130.58249312188119,
- 1.1128817603274543,
- 56.157995916719756,
- 35.81413980204931,
- 116.5213087641768,
- 63.30354399512571,
- 55.0117106848875,
- 47.52954249314361,
- 153.04709230401787,
- 1.112276523473745,
- 80.1523559974256,
- 136.20373724941714,
- 1.114673225365626,
- 1.1067132158651183,
- 149.29883052073288,
- 145.10950784560325,
- 130.53765167080937,
- 1.111788125890117,
- 0.8957719496064405,
- 1.1050775451489783,
- 17.522300994030367,
- 154.45472111064055,
- 152.07616582090188,
- 1.1020107149905272,
- 138.6808068419634,
- 76.87873177159636,
- 51.43702839643221,
- 138.95045176064437,
- 138.64177504011988,
- 140.72197385602811,
- 132.80947742972836,
- 149.78872816785005,
- 139.94034036065392,
- 154.2632802491591,
- 55.57148538150843,
- 1.1044580058296936,
- 147.1712801496827,
- 77.84198065949245,
- 142.38330204183904,
- 151.76812011990265,
- 145.19131540821485,
- 147.26566215388425,
- 87.12413393605841,
- 1.1038403429439656,
- 141.4935550752979,
- 145.7397470598185,
- 3.3080164659931235,
- 123.0327553358976,
- 146.24080278853327,
- 148.10448175245884,
- 29.234562433775857,
- 151.30177873039895,
- 135.4653748135468,
- 144.3293913931314,
- 148.16163203136404,
- 1.1015876034201657,
- 1.1114790318458536,
- 136.68047783885697,
- 77.72584511329579,
- 125.73692105352463,
- 106.98755729483561,
- 96.25926845246491,
- 1.109721323323522,
- 141.71073652156545,
- 130.22006710827588,
- 145.24478945746003,
- 80.67459353439743,
- 1.1033551544760267,
- 150.03177939272493,
- 154.12875534463626,
- 150.04771421074818,
- 1.1010813815407388,
- 1.1110434127990452,
- 145.385699877379,
- 86.86487551811825,
- 130.16687493633253,
- 143.8726181331947,
- 111.91340621077623,
- 146.0394914387852,
- 1.1006353022455784,
- 134.47903589563677,
- 148.6907436994389,
- 102.87151097507036,
- 137.41724911494663,
- 1.1146766644704549,
- 143.85952373403495,
- 146.92280951248307,
- 1.100156488603178,
- 144.04783334738536,
- 148.53630346113712,
- 58.74848466983248,
- 147.0485685726298,
- 141.32891699761203,
- 142.8441702922343,
- 131.04366253726744,
- 128.6305301075303,
- 1.1106412111686195,
- 147.90025888582002,
- 0.8959265584913588,
- 149.5194069726666,
- 137.43649451567626,
- 1.1068068376551545,
- 68.05269425995475,
- 138.94056631255367,
- 138.43818227469507,
- 69.60391199895408,
- 114.83395091462887,
- 151.34107787433956,
- 141.57237630997332,
- 146.07433910500515,
- 9.941778754980154,
- 131.297822968639,
- 10.386636719874664,
- 10.545636067043365,
- 114.58677137445733,
- 75.28902943071078,
- 90.63452059810655,
- 143.58694736923238,
- 9.901118804514459,
- 144.5206530902411,
- 144.78737732574044,
- 79.81136215142409,
- 84.9314508821071,
- 120.18939827456474,
- 10.225253542151219,
- 9.702822548173124,
- 103.1188517219872,
- 138.5008491242522,
- 92.02238700298246,
- 151.99592340131602,
- 9.807595290716304,
- 150.0447954775559,
- 134.2614008494909,
- 149.38544573345007,
- 149.62298116309924,
- 124.32358754465251,
- 132.817456221544,
- 10.50607995390264,
- 9.78317681034783,
- 151.07916494121415,
- 146.93545537009487,
- 118.45851163082196,
- 145.03008316360754,
- 154.4449202186591,
- 146.86002069809945,
- 150.6932855951215,
- 110.74803327496042,
- 127.40788523389726,
- 150.81323854197058,
- 150.0047673310006,
- 149.6063654551971,
- 133.87244996538675,
- 10.329695475492791,
- 9.414695716712222,
- 106.77032789813472,
- 118.34636653947105,
- 123.44441062862572,
- 144.9015592115516,
- 153.74652990582067,
- 10.065713405335144,
- 129.38998560194165,
- 117.69087049838025,
- 99.15650839997046,
- 127.90462338199198,
- 147.3574863739125,
- 9.696544883885949,
- 9.8853852911422,
- 128.35872796896587,
- 145.2939860705264,
- 128.72081963712404,
- 94.09935653689803,
- 142.8780531031409,
- 130.5213122981276,
- 126.89288883528536,
- 153.36107852781166,
- 149.17239657923582,
- 9.177632630803961,
- 9.387171298727486,
- 109.68196882316985,
- 148.55536204011432,
- 152.61730207818772,
- 9.648922236946333,
- 132.805446535875,
- 138.74295200738652,
- 141.66118217831166,
- 124.0399127789103,
- 113.05005278683446,
- 149.71230902297984,
- 25.727698431920004,
- 129.56419655827216,
- 130.40687823665095,
- 128.46470366050013,
- 150.46298369674685,
- 9.22073843893938,
- 110.36443029340542,
- 148.23878821929193,
- 10.219508495480236,
- 9.615051521185155,
- 9.8723813087942,
- 149.91378148843256,
- 9.149056684599877,
- 130.37704092008303,
- 114.86611671621016,
- 134.53633480709703,
- 131.11593468604048,
- 149.74665952988033,
- 136.60701891253495,
- 146.50864617645632,
- 9.094221140419737,
- 149.69902295915708,
- 126.93245475406366,
- 141.2463933703881,
- 10.18172163650932,
- 136.76582155059438,
- 155.5823388453975,
- 144.68082947663285,
- 142.0128061769988,
- 116.20800508912414,
- 101.13756407758095,
- 10.050927550768915,
- 10.14139856150474,
- 9.573219645146107,
- 146.33874064646594,
- 137.22302119976462,
- 132.14965518046,
- 148.08190796641483,
- 117.6843964457568,
- 153.04352772565807,
- 146.79238076404926,
- 9.522740968586977,
- 145.93484469600287,
- 13.925952420322696,
- 12.697420287309185,
- 146.39122941822845,
- 113.94298610788566,
- 13.844109957456581,
- 154.57922917096633,
- 13.525210269101805,
- 103.83976095796662,
- 97.75660804271413,
- 135.83818209343426,
- 158.60060111529293,
- 111.57793188874757,
- 13.768524263105455,
- 154.2203592546867,
- 108.85242762118563,
- 111.15752259030245,
- 149.5942138872604,
- 119.77102605185765,
- 120.68065341205389,
- 105.29698904913548,
- 151.41465167808087,
- 138.90606724001483,
- 13.437371194424983,
- 119.97194649055415,
- 144.6223725248399,
- 146.9934910169238,
- 149.45319992777343,
- 121.48260402443249,
- 13.662736071688842,
- 14.448955892498802,
- 144.5545360346381,
- 154.00382983055897,
- 151.8635735223181,
- 137.2321484611102,
- 119.71487519948164,
- 88.24978714231261,
- 147.74815341218743,
- 142.1113258863455,
- 132.08775922189477,
- 124.63351274554526,
- 145.72256212355262,
- 100.50708502243579,
- 139.16363846809003,
- 114.82662827063822,
- 154.78307253831395,
- 149.22879563842886,
- 152.6744734255461,
- 145.81022434241217,
- 152.68018782123758,
- 116.75549006136289,
- 12.968595875688791,
- 6.824624970615158,
- 125.05116103474757,
- 147.66072487793718,
- 147.5735120742967,
- 139.1302141298083,
- 146.48542990069834,
- 12.674865288395944,
- 147.88858853602966,
- 6.8124480142416175,
- 137.54766974463703,
- 130.89979405333307,
- 13.364169845161861,
- 14.116086127002273,
- 130.3002929300388,
- 116.98398239487472,
- 152.70827610346095,
- 98.51470626500011,
- 135.1252373635164,
- 14.405992358855888,
- 154.13709739001223,
- 146.28661687368685,
- 137.87827066214206,
- 12.621081453489012,
- 154.04574874294514,
- 6.802625211185703,
- 152.18661864386252,
- 149.30257880598677,
- 13.244501725269068,
- 138.34068638798834,
- 150.95140747506372,
- 141.8441899037163,
- 152.99022366652198,
- 103.95004802425926,
- 140.28144756248412,
- 154.51222806007945,
- 85.40777548962518,
- 154.7067128296305,
- 120.47843952303268,
- 12.568053995018431,
- 12.916583075889136,
- 105.92477484543576,
- 137.92878859711615,
- 135.13853669037294,
- 137.88549737290148,
- 157.83019925734393,
- 145.48927689323145,
- 12.509532718065461,
- 150.6233829715981,
- 119.23669844460764,
- 138.49099023171033,
- 154.0870149904812,
- 140.1862744667834,
- 148.860174031694,
- 147.54629689336036,
- 12.448861769003683,
- 152.4711466483636,
- 102.47079224461186,
- 152.40864885890767,
- 156.21773232766026,
- 13.139291580904986,
- 150.30653960489693,
- 145.43571147072188,
- 132.8965387342577,
- 144.85972103961666,
- 125.5438694385711,
- 158.07457773478276,
- 14.359506122440205,
- 137.7658155977229,
- 153.68125116011197,
- 156.57780724945528,
- 12.394708947912125,
- 12.874702780202174,
- 110.61518572692995,
- 149.4338565730422,
- 149.67552030435513,
- 146.20909415912828,
- 9.308833539527914,
- 26.176147260970783,
- 8.701217384742513,
- 66.92241449340185,
- 105.12940849136734,
- 145.25326276553395,
- 139.68219350261262,
- 131.60335890332783,
- 150.53420884400245,
- 17.552483447968918,
- 99.60476667168517,
- 9.003208512207522,
- 8.539560747895454,
- 9.946172723540226,
- 150.55644446784382,
- 9.608936841972842,
- 104.80864366760326,
- 25.95068644438624,
- 99.42592550150236,
- 108.35979254469888,
- 113.9171427720856,
- 9.905905876631499,
- 131.1684982861573,
- 154.7989292174601,
- 151.34753888952145,
- 150.11816141981262,
- 143.00557828542912,
- 126.2310299151925,
- 113.53830001728545,
- 148.13405630794878,
- 150.7564429392251,
- 155.252325076404,
- 18.20048176554747,
- 25.725436761645142,
- 8.678711562613207,
- 143.3683328827327,
- 127.0294451168928,
- 137.50119476282134,
- 10.068367539846923,
- 155.64822784014916,
- 153.2789382926615,
- 25.46950813818654,
- 142.9138107220956,
- 155.10510899417167,
- 107.40557834412083,
- 9.871948602847068,
- 144.4712732194919,
- 140.17802930301565,
- 9.286026243902361,
- 129.1488895575147,
- 124.35586045151207,
- 140.1410811550992,
- 96.63692877337894,
- 153.62093095799207,
- 156.05800033315097,
- 9.587609950939838,
- 140.09721428165886,
- 134.898750425008,
- 8.652809034763463,
- 8.989448046931262,
- 107.64260577858933,
- 9.825071080298192,
- 150.6237132142087,
- 143.76058852986372,
- 154.01627264735168,
- 140.85322298632985,
- 143.63714834446708,
- 149.7259575806535,
- 8.53942846683121,
- 157.02635815805976,
- 150.83913162907433,
- 154.0283691261865,
- 9.246842209481716,
- 154.5851361854829,
- 133.4662155767381,
- 137.55396410787307,
- 105.77910782321499,
- 148.97953057255376,
- 111.3041581371634,
- 9.543858351726714,
- 142.71996301994741,
- 144.2417836324451,
- 148.5293262803374,
- 8.95331376662564,
- 105.2724164655814,
- 149.16646109060707,
- 151.1947852118465,
- 9.503293907683512,
- 133.40055362812345,
- 8.776394391795916,
- 148.3675722527084,
- 154.66946641450528,
- 122.71674068416665,
- 149.62192317697068,
- 153.40159484208397,
- 9.46860898864519,
- 146.10526710538994,
- 143.96020057925128,
- 8.62472208077336,
- 8.906885562515198,
- 105.7754218686014,
- 150.17957794387223,
- 144.0451331512576,
- 149.95461039551162,
- 151.46311089131117,
- 142.22104279807664,
- 147.3679944003333,
- 140.5394711174869,
- 123.62157744638432,
- 152.32796921399395,
- 156.6603241829257,
- 9.43621164630811,
- 158.2241383954169,
- 149.33346139426692,
- 144.12074054746773,
- 143.1977521817863,
- 8.536662624511228,
- 9.785635570067782,
- 147.61880087321424,
- 9.402323265876474,
- 159.1161790596516,
- 146.56796834276156,
- 147.64890403285438,
- 157.70847517328534,
- 114.64282143770687,
- 148.5000942425868,
- 10.052761003641129,
- 147.38801074409378
+ 41.46611265659158,
+ 44.4918071112372,
+ 46.926673665513704,
+ 46.30487800041612,
+ 45.31117511724168,
+ 39.48427257480573,
+ 41.73807567318408,
+ 44.986328772700176,
+ 46.79460518580979,
+ 2.1481645603133406,
+ 45.3304673980315,
+ 46.361305003734564,
+ 1.2216768370041928,
+ 35.39842883637453,
+ 44.9539795483452,
+ 39.212326267312775,
+ 1.0742220506708642,
+ 45.596949876501405,
+ 1.656518545685144,
+ 41.1853065101293,
+ 45.186903991589205,
+ 2.733636984435035,
+ 1.8859234764357438,
+ 4.103119744826081,
+ 45.69245622017379,
+ 1.6582215083936738,
+ 37.954906657600475,
+ 46.5127757873931,
+ 45.29733823530308,
+ 23.1754689963102,
+ 43.44487109471452,
+ 33.311038622351724,
+ 46.400400898475304,
+ 43.13207624251721,
+ 45.26221685255157,
+ 38.89631907864675,
+ 1.0766827581902934,
+ 3.1955625641377354,
+ 41.00672778846412,
+ 1.225434086753332,
+ 0.951420354873873,
+ 47.29759062957134,
+ 37.27931328255301,
+ 44.02626192577354,
+ 44.567351509891715,
+ 41.19817412895097,
+ 1.4117117845102758,
+ 46.974942144500005,
+ 26.16803432928029,
+ 40.79104304470394,
+ 45.98186302516314,
+ 47.4055947551752,
+ 1.076201435026891,
+ 3.1796394093402074,
+ 41.23717257081556,
+ 42.85213590859161,
+ 44.28329201807133,
+ 46.527540336613534,
+ 43.08848614726634,
+ 44.40830753324719,
+ 41.37604170752994,
+ 0.9482378607333808,
+ 45.48122547719385,
+ 47.20316588665498,
+ 0.9510683482370443,
+ 1.9012380421663475,
+ 46.19550253488152,
+ 2.7330118039774067,
+ 45.74495207812405,
+ 34.67238053318697,
+ 38.85119722571936,
+ 1.225081100472964,
+ 45.15238085691014,
+ 40.396011557170766,
+ 45.488921919651816,
+ 45.29351001493665,
+ 1.0758273605231232,
+ 29.808026495079588,
+ 1.2280820949811997,
+ 46.586185131212794,
+ 42.89263913245724,
+ 42.15612175451927,
+ 46.693253798156995,
+ 46.57003199283068,
+ 46.509087816223484,
+ 38.12557546239959,
+ 45.81548305523131,
+ 46.07453120649211,
+ 40.81605463432999,
+ 45.228424339779814,
+ 42.086064813661196,
+ 42.78740035356858,
+ 45.98922633164769,
+ 41.28717865700289,
+ 1.2274351142907918,
+ 43.46971411790415,
+ 45.4498626576556,
+ 42.51719188567606,
+ 46.624215728553786,
+ 43.26045159027894,
+ 43.962414509948275,
+ 0.9481540147597537,
+ 1.2267700611313974,
+ 1.2246727704472544,
+ 45.950324312195605,
+ 46.02559998344755,
+ 1.413545795432525,
+ 2.1538932898075407,
+ 45.57032628071106,
+ 38.877775528665516,
+ 44.5660811280025,
+ 45.98326532911864,
+ 41.78435738761637,
+ 44.118449498817554,
+ 43.11682781122976,
+ 46.80957208928424,
+ 1.0755822711089933,
+ 29.775928132799514,
+ 42.492052303926506,
+ 1.2241095107799485,
+ 45.796086216431775,
+ 45.258843364665246,
+ 44.97308057669771,
+ 42.89527265230854,
+ 43.91533758581356,
+ 35.81442349583988,
+ 30.65358830169187,
+ 46.3182793971083,
+ 44.145493159555286,
+ 44.2651994526335,
+ 40.09824843769361,
+ 45.68707977480025,
+ 39.990813212941646,
+ 35.79658562417175,
+ 44.86013694329229,
+ 41.83115806056866,
+ 37.15064410140025,
+ 0.996787320025337,
+ 45.66808620182929,
+ 46.6130598481811,
+ 45.60972037064592,
+ 0.9940425141246046,
+ 45.591900274871186,
+ 46.96840985185615,
+ 43.393354375970155,
+ 25.5248831966376,
+ 45.77235244972332,
+ 24.590561326831967,
+ 0.9773483444490005,
+ 34.09417278739622,
+ 43.586572958161206,
+ 46.535859932274164,
+ 45.946757322805404,
+ 0.9962165194499956,
+ 0.992874583950711,
+ 46.119932829039165,
+ 42.179658293228435,
+ 32.997191121192365,
+ 44.17582132320044,
+ 46.14366473770965,
+ 45.81106545186327,
+ 0.9957624959115234,
+ 0.9924622264244217,
+ 39.42192933951627,
+ 37.64229442727469,
+ 21.26565173458009,
+ 45.593412953334585,
+ 46.87304671516134,
+ 45.216027572946594,
+ 42.43765019133474,
+ 46.197382024442064,
+ 40.692114254409056,
+ 45.33796853087654,
+ 27.766522112160985,
+ 40.02641706822085,
+ 1.3017150918854614,
+ 45.591631786019235,
+ 44.34279696011747,
+ 39.28257190816356,
+ 43.72958684288255,
+ 0.9771143356157014,
+ 23.874882409185425,
+ 38.84831650281934,
+ 46.04825715862786,
+ 44.318350427904555,
+ 47.26086876225989,
+ 39.433419122254435,
+ 42.94084765393213,
+ 43.44077111651132,
+ 42.4775425505976,
+ 0.9890763303083981,
+ 47.353878858820345,
+ 40.99026973150018,
+ 0.9955331259047124,
+ 46.52810662522569,
+ 43.71121305319187,
+ 43.098140605333754,
+ 0.9941110054345192,
+ 0.9887007080233833,
+ 41.60423122999918,
+ 45.81533148936388,
+ 42.37614297709579,
+ 45.84171517205181,
+ 41.73162426832469,
+ 0.976838541947363,
+ 14.558863836592382,
+ 0.988317986920056,
+ 27.41518624216025,
+ 46.00613760472248,
+ 44.605125117227445,
+ 0.9923556095766691,
+ 46.06453996269855,
+ 45.69598995103852,
+ 38.29204120955434,
+ 0.9879204612413145,
+ 45.051133494631664,
+ 0.974139430894493,
+ 43.52911731376158,
+ 0.9919675926934881,
+ 45.37964604415822,
+ 0.976397605350521,
+ 36.30289308241207,
+ 45.597233615462315,
+ 43.61071649968794,
+ 43.122470348017536,
+ 46.76087701561043,
+ 0.9915593888202096,
+ 43.301652472823534,
+ 43.35874933591963,
+ 0.9940066207204965,
+ 42.186091123827985,
+ 45.37749985977852,
+ 0.9738097357420213,
+ 46.47531110944141,
+ 0.9911618676375942,
+ 43.561154900046205,
+ 42.50481546978642,
+ 36.28178246877416,
+ 44.229193258120816,
+ 43.274122438133034,
+ 43.16603619055846,
+ 46.24123104179791,
+ 0.9907652867200517,
+ 44.808052346983644,
+ 42.157257924432415,
+ 30.810167635761594,
+ 44.5009455404432,
+ 44.803133707609575,
+ 46.717718944658586,
+ 45.328295623099564,
+ 0.9903649151763216,
+ 45.98765051561304,
+ 43.15949033247262,
+ 0.9938810855133485,
+ 42.5272021864534,
+ 46.202556875553654,
+ 37.69680010665373,
+ 13.506488443568907,
+ 47.084518208092895,
+ 45.34409129030842,
+ 45.528670127709155,
+ 1.0839758382565585,
+ 45.77369572816552,
+ 40.36600389536794,
+ 46.346373598961115,
+ 47.59928731210073,
+ 45.213230445194775,
+ 46.97741000418462,
+ 43.73589527028813,
+ 38.21138599701667,
+ 39.80440406603509,
+ 47.546574744238036,
+ 46.363044750837105,
+ 45.73935328577624,
+ 22.79542790283351,
+ 1.0852955230764447,
+ 46.31190530756646,
+ 10.103645571001175,
+ 20.743583307847267,
+ 34.08924086156784,
+ 40.34233471572178,
+ 1.0825832325439408,
+ 42.93380762165118,
+ 46.538540446937695,
+ 40.56431787179345,
+ 1.0837596134259624,
+ 35.02268200701654,
+ 47.136990718638934,
+ 38.591258432063235,
+ 47.93266376947172,
+ 40.53416662878643,
+ 46.663334136659614,
+ 1.0714520955139675,
+ 27.88935756664922,
+ 45.48047962233704,
+ 1.0758750615408978,
+ 1.0683190801502396,
+ 46.009876361978876,
+ 46.59268594380503,
+ 46.02812612004097,
+ 46.372356575684854,
+ 22.894765755636868,
+ 45.64436406976758,
+ 46.20773355624579,
+ 42.364426646383905,
+ 1.0822510357556412,
+ 44.863056156314066,
+ 46.46090797778492,
+ 1.0710544669423023,
+ 1.083596675232654,
+ 46.253226306136575,
+ 1.075461579555405,
+ 46.46757181265049,
+ 1.081777244820761,
+ 1.079157130525964,
+ 47.44728077576711,
+ 44.18890905454099,
+ 25.69445080780143,
+ 41.61341063520841,
+ 1.0749834632245117,
+ 45.18278804232428,
+ 1.0813046939407982,
+ 45.584290798191994,
+ 1.0851558601194167,
+ 1.0706298125469418,
+ 27.277652622917802,
+ 3.13795203228774,
+ 46.596243996630385,
+ 1.0680343711445561,
+ 1.0808489429820316,
+ 44.07771833504717,
+ 1.0782837622370247,
+ 44.620236842054005,
+ 33.66037405692795,
+ 42.88981761147569,
+ 1.0745719383443746,
+ 1.067541523615096,
+ 43.3531928586852,
+ 46.45260807995745,
+ 46.301433990064965,
+ 45.45037480313856,
+ 42.01190688214572,
+ 43.97592120992246,
+ 44.22612202356458,
+ 46.93790632881387,
+ 43.35324044647867,
+ 46.24983553374027,
+ 1.0779013969854039,
+ 45.68642573969881,
+ 40.71576971597602,
+ 43.609256041900395,
+ 44.75345611987869,
+ 46.683440264062696,
+ 6.250364298356673,
+ 46.58797465847453,
+ 1.0773923535890582,
+ 43.82763570204923,
+ 41.62940460437239,
+ 42.91661388574536,
+ 46.901610347450095,
+ 46.61677212391794,
+ 1.080583826854443,
+ 34.07713605907777,
+ 46.92641126499492,
+ 45.79075334582258,
+ 40.14409222341034,
+ 45.361779654878845,
+ 46.88204342817273,
+ 46.35566639777504,
+ 46.36704829301128,
+ 1.079068056447631,
+ 46.774512434519465,
+ 1.0704507990204184,
+ 1.0837001046492374,
+ 44.56501843026455,
+ 45.92497594226974,
+ 46.819599375484145,
+ 1.0801577199815187,
+ 46.01182819769449,
+ 1.0770346495733834,
+ 46.950613182781744,
+ 30.797706097998343,
+ 46.18180484355316,
+ 46.16072338065117,
+ 1.1133090433838153,
+ 1.1264329475750274,
+ 1.1236172122377037,
+ 47.045544454610436,
+ 46.77875324298633,
+ 28.03992244253687,
+ 45.334641615839494,
+ 8.780689100623139,
+ 20.7913981632672,
+ 32.723036948097274,
+ 45.13282209264667,
+ 46.65435200771115,
+ 45.96287965580367,
+ 9.076296968757461,
+ 45.4816339150996,
+ 46.902872519542036,
+ 46.16846796984993,
+ 45.756891597403175,
+ 44.88315382035088,
+ 46.23903054578556,
+ 45.83324366902273,
+ 17.750809391531607,
+ 45.20000225981293,
+ 47.302482301226895,
+ 45.60218665990497,
+ 36.97764728135097,
+ 46.59609042040382,
+ 46.604767462324304,
+ 45.96159537616419,
+ 22.37221435902452,
+ 43.859502782475616,
+ 46.5164446015921,
+ 46.29329085467359,
+ 1.1262112315718147,
+ 46.308551190848824,
+ 46.12319048896243,
+ 43.60305812792925,
+ 0.9422659923955576,
+ 45.850627271010616,
+ 45.017760412103506,
+ 46.45017372234843,
+ 46.681005137311296,
+ 1.1235052275623567,
+ 45.024655731975905,
+ 42.551907139236725,
+ 0.9419457570631012,
+ 41.1118024425248,
+ 45.63421048620437,
+ 46.022116096626675,
+ 1.1258383546403372,
+ 47.1081443735114,
+ 47.030126605956774,
+ 42.86500455064436,
+ 37.358353939700315,
+ 45.34461986882157,
+ 46.86806884248587,
+ 46.417501701989885,
+ 46.351389315230215,
+ 46.78447423742242,
+ 43.74686698408526,
+ 1.116867665232356,
+ 0.9417093885501255,
+ 1.1193255628248941,
+ 46.36628759364972,
+ 47.0182927090698,
+ 44.33757352470002,
+ 7.691634088129115,
+ 1.1283438070497074,
+ 43.879143747221455,
+ 0.9414915905260655,
+ 1.1187592356622462,
+ 1.1221505116978934,
+ 46.07747894106487,
+ 46.579798906537704,
+ 45.766896552621894,
+ 46.65247758283254,
+ 43.302159908237364,
+ 37.720159108605536,
+ 1.1182282725285237,
+ 46.39182837285494,
+ 44.636636353923784,
+ 43.44450203063323,
+ 1.1233649178804157,
+ 45.04855028838785,
+ 1.1165108506849695,
+ 29.25784442036365,
+ 44.92016113045485,
+ 1.1217307674387187,
+ 46.08594914883392,
+ 1.1256588113160433,
+ 44.33658350966423,
+ 1.1279641443945907,
+ 46.995953225218045,
+ 43.09174152350243,
+ 45.522175701238005,
+ 44.54660682798267,
+ 46.26002914896281,
+ 45.121721334753246,
+ 45.99661519970516,
+ 46.999367551883665,
+ 1.1162274151428622,
+ 34.79092708982097,
+ 45.466303894602824,
+ 1.1214388358967042,
+ 46.3611527229414,
+ 1.1253775196067384,
+ 1.1231558495643674,
+ 45.46781022594765,
+ 46.83967784020296,
+ 35.37244717495285,
+ 1.1180685191822184,
+ 47.0281597759591,
+ 45.004932496628875,
+ 44.35708507257986,
+ 46.65855899768837,
+ 5.505111079406215,
+ 1.115802761131929,
+ 35.602590093008914,
+ 44.671751586624886,
+ 46.281278781026465,
+ 46.65874233841448,
+ 47.449917573209895,
+ 47.11754288927177,
+ 46.84313387306054,
+ 1.1152851890752418,
+ 26.693730551391678,
+ 45.574691537692864,
+ 47.110350441661474,
+ 46.950895044828556,
+ 47.10814947984309,
+ 42.35670263948847,
+ 43.399091167413815,
+ 45.65945467138436,
+ 10.323879128717438,
+ 17.406756102821927,
+ 46.70765041608834,
+ 46.265154949804675,
+ 46.966387230240066,
+ 46.58181691440536,
+ 1.1794390054814614,
+ 40.240832270343546,
+ 39.59688963721167,
+ 1.169177901708881,
+ 1.176889456593387,
+ 46.512318262726104,
+ 16.255791986842784,
+ 46.90191826875892,
+ 38.002332039368945,
+ 1.1673839996531623,
+ 32.855434627015846,
+ 43.339268319257165,
+ 46.75273409704357,
+ 46.82224515218503,
+ 46.7787448289983,
+ 46.08633464118119,
+ 1.1789416201176985,
+ 45.01880600815589,
+ 17.692981429746695,
+ 43.82069805510859,
+ 42.693302457425894,
+ 40.895519742462156,
+ 43.141099312595934,
+ 48.08036522096514,
+ 1.178390117026328,
+ 45.95511642215028,
+ 35.29568405980472,
+ 1.1687957641452225,
+ 1.1765143734981645,
+ 46.688387154545254,
+ 47.06125638807941,
+ 45.346066735128574,
+ 1.1777709765320192,
+ 1.166989666506321,
+ 0.9847523589742398,
+ 18.562855771239047,
+ 47.9065264813057,
+ 46.73354514650198,
+ 1.1735046304883543,
+ 46.412712735423334,
+ 45.16100408019957,
+ 43.83022094061403,
+ 35.89794593782671,
+ 44.97192473982221,
+ 46.7633180339843,
+ 44.329869977212624,
+ 47.38342947643397,
+ 46.79402738420473,
+ 47.634269098703626,
+ 44.0213863595159,
+ 0.9845269249937244,
+ 45.78778499348287,
+ 43.90149865817902,
+ 45.65368969409286,
+ 47.746456721033944,
+ 47.21697228426952,
+ 47.01924612843149,
+ 46.3245200194134,
+ 0.9842560530393194,
+ 45.26992712182612,
+ 46.89243421872701,
+ 3.4924828727877877,
+ 45.25207572636316,
+ 47.25700297914972,
+ 46.94730150195301,
+ 39.12367514310055,
+ 42.117856976344655,
+ 44.28179459170351,
+ 46.596840500912684,
+ 45.392754933120926,
+ 1.1731165363524663,
+ 1.1755941425503302,
+ 46.46126582671268,
+ 45.79994582850055,
+ 31.36362072652773,
+ 43.50384100878153,
+ 45.440038476775335,
+ 1.1661505662188223,
+ 46.52744939333318,
+ 45.250414658311975,
+ 46.53386354717518,
+ 45.796239735104564,
+ 0.9841302985201961,
+ 46.27883497779145,
+ 47.83598353847002,
+ 46.607837943658275,
+ 1.1726681962992465,
+ 1.1751504766334446,
+ 46.84845290565303,
+ 46.07497571222637,
+ 33.33732005606778,
+ 45.813985387630716,
+ 45.57964157112892,
+ 46.41818933014048,
+ 1.1721397028860254,
+ 45.89252926130944,
+ 47.09569465450331,
+ 47.250364539349285,
+ 35.22784278442342,
+ 1.1688030911620526,
+ 46.42186257421796,
+ 46.25658899517002,
+ 1.171409947579052,
+ 45.16137403712752,
+ 47.22442045049697,
+ 44.82261712339744,
+ 32.494327996097915,
+ 44.219079390101115,
+ 46.87735465561079,
+ 44.699203955991905,
+ 45.12568915598884,
+ 1.1747532937483116,
+ 47.069832959511444,
+ 1.1670956785442357,
+ 41.217948435045656,
+ 44.93033926516496,
+ 1.1766349885441727,
+ 35.47522021954888,
+ 46.21124702140885,
+ 46.24628779612773,
+ 34.53125955420697,
+ 46.66578037331865,
+ 43.65856477535035,
+ 45.03361057951491,
+ 46.76526122602155,
+ 10.182019712559228,
+ 45.71366318720834,
+ 9.833945628376052,
+ 9.322117004081543,
+ 46.537564499785105,
+ 31.262138808373493,
+ 37.90592059294092,
+ 46.820091937863225,
+ 10.139423148881114,
+ 46.75580347295349,
+ 46.89455728317566,
+ 39.52390472502032,
+ 42.643467900988064,
+ 38.90725083946543,
+ 9.086630150053459,
+ 8.937192123351853,
+ 40.9872575801166,
+ 46.394128489242924,
+ 41.193529101734704,
+ 47.34329154675404,
+ 10.054610354639179,
+ 43.31828144588645,
+ 44.553079069624026,
+ 46.98279134065351,
+ 46.830147489351724,
+ 45.31329233494219,
+ 45.552850223950976,
+ 9.295212965663417,
+ 10.01436272470524,
+ 43.57022598341257,
+ 45.70609566213184,
+ 43.449062338174066,
+ 46.855675373016474,
+ 47.68860594538369,
+ 47.09689498272573,
+ 47.173878516378814,
+ 46.069788054621185,
+ 38.92002107306488,
+ 46.38712908030891,
+ 47.104897416242906,
+ 46.938337511897245,
+ 45.36212980855197,
+ 9.7037632831636,
+ 9.265430506589102,
+ 46.11721659871563,
+ 38.06187391881914,
+ 43.25827348162763,
+ 46.84719251692419,
+ 47.03682707869591,
+ 9.90500846057903,
+ 45.68739012850455,
+ 43.47148156475432,
+ 45.23323967788647,
+ 39.81125388088527,
+ 45.95084232488125,
+ 8.919454342379801,
+ 8.706571515609426,
+ 45.29003523159025,
+ 46.867399234540684,
+ 45.35240769107086,
+ 44.80265358061401,
+ 41.83510960528982,
+ 43.92616077285124,
+ 44.61292075723489,
+ 46.86625528407582,
+ 47.230904823696534,
+ 9.643361950798496,
+ 9.236779459262468,
+ 46.27993094745158,
+ 43.29062809284174,
+ 46.53130368901898,
+ 8.891092687715933,
+ 45.323215643957305,
+ 46.38559644193777,
+ 46.8553797027437,
+ 45.16725651833185,
+ 46.26177304715086,
+ 43.16649621953115,
+ 19.53072875578119,
+ 44.16107832748164,
+ 44.46643011473998,
+ 45.302511702487166,
+ 47.59950805589659,
+ 9.206283803180765,
+ 46.31521045156664,
+ 42.932315734513345,
+ 9.081962094633843,
+ 8.862645496755041,
+ 8.681026899042758,
+ 47.175946890403075,
+ 9.613647025719098,
+ 45.37459772842735,
+ 46.657937572561956,
+ 40.090063197986055,
+ 43.91176191056239,
+ 47.1764939819939,
+ 44.932347492473085,
+ 46.951971869749755,
+ 9.588107858966847,
+ 46.890536209011636,
+ 47.457220061858926,
+ 41.820791051617206,
+ 9.051934235829219,
+ 45.46750284471863,
+ 47.1114848526844,
+ 46.90614671206355,
+ 46.81408948407702,
+ 44.76508972637772,
+ 44.94143445208981,
+ 10.013702243637548,
+ 9.016326405341099,
+ 8.836765675846252,
+ 46.724030690708,
+ 45.670931647965055,
+ 45.52105012345985,
+ 46.760404038674345,
+ 46.879394746618935,
+ 44.17372013338399,
+ 45.75158023561404,
+ 8.805217872024683,
+ 45.797390838433785,
+ 13.147893146580197,
+ 10.47047709122617,
+ 46.61575812332005,
+ 46.51823693220529,
+ 4.823033237525791,
+ 46.77438522864306,
+ 12.978009554740229,
+ 38.60487947846694,
+ 42.776667803234396,
+ 46.400158258735026,
+ 47.945284694706544,
+ 46.56814403610221,
+ 4.817274157491479,
+ 46.62284523101857,
+ 43.12368820615556,
+ 41.32670008561977,
+ 47.18041683967238,
+ 43.946314235571926,
+ 44.21062282398479,
+ 46.19942835901387,
+ 43.058732279332816,
+ 45.38189559700182,
+ 12.884302510247224,
+ 41.31993708388949,
+ 46.47169213829526,
+ 47.19006572402318,
+ 47.14982705362978,
+ 47.06368907184152,
+ 4.812880414029111,
+ 11.16220592067454,
+ 46.574241250493166,
+ 46.97994816848278,
+ 47.45816665639938,
+ 46.13083135931701,
+ 44.32000975084153,
+ 43.41804159092183,
+ 42.66169852490167,
+ 45.48613569289166,
+ 44.33345445574926,
+ 43.452008302705025,
+ 46.81171828117368,
+ 43.10993692872848,
+ 45.994793877105536,
+ 46.800586622051604,
+ 44.27154316655175,
+ 46.105917327794614,
+ 47.46844284412024,
+ 46.26483577817879,
+ 47.53682651754337,
+ 44.570703276937955,
+ 13.903655242145248,
+ 11.480956559418479,
+ 39.336500908555834,
+ 45.90660459732642,
+ 46.77917515765938,
+ 45.088381020490885,
+ 46.506580602768324,
+ 10.416775312398924,
+ 46.58444309156844,
+ 11.387487180031048,
+ 40.66527760299146,
+ 43.83362837067986,
+ 12.535722984692502,
+ 10.862075986088263,
+ 45.57849071079437,
+ 44.54752207894966,
+ 47.368339209936586,
+ 44.99292457355705,
+ 40.53083756344339,
+ 11.0636299214144,
+ 47.688667053142176,
+ 46.49150277169404,
+ 45.74006902822907,
+ 10.33525884882965,
+ 47.48557960393818,
+ 11.308966508889716,
+ 43.29259854243531,
+ 46.1099584752184,
+ 12.17957601526656,
+ 45.17415787692287,
+ 47.42069363597441,
+ 46.61857073840612,
+ 47.2421945434337,
+ 45.43588217737557,
+ 40.87274833234901,
+ 46.70759606653805,
+ 36.65554403597885,
+ 47.00974843039727,
+ 44.27238095134427,
+ 10.215116571612004,
+ 13.7852700376187,
+ 46.056843647274086,
+ 40.6532114020977,
+ 44.73992298080998,
+ 45.68916428641405,
+ 47.31026005200245,
+ 46.82535713731543,
+ 10.130547297609347,
+ 47.03536361799409,
+ 46.991892284267614,
+ 40.158116078863046,
+ 46.709887162762875,
+ 46.67477141304538,
+ 46.52127067854677,
+ 46.8876604645323,
+ 10.042145383707755,
+ 47.028109894652104,
+ 45.7372913308103,
+ 43.35504560755716,
+ 46.94810107337359,
+ 11.8541419498795,
+ 46.48396692070885,
+ 46.650791251635994,
+ 45.251645228092976,
+ 46.90500963017914,
+ 47.44769079351513,
+ 45.17830741847997,
+ 10.999409433497265,
+ 46.47750683850478,
+ 46.775120397902185,
+ 47.814786925390884,
+ 9.948141267257297,
+ 13.587316761063226,
+ 46.55485731583328,
+ 42.77962873201528,
+ 45.79657353014755,
+ 46.78648032853886,
+ 6.092950585496579,
+ 16.427217699690395,
+ 6.041669306781378,
+ 33.44834000640586,
+ 45.71021173581392,
+ 40.44649791159415,
+ 44.41704966518361,
+ 45.16867811008679,
+ 46.553484065254395,
+ 11.951659518508801,
+ 40.964520355583325,
+ 17.222473173678548,
+ 15.810785212495478,
+ 5.896598504159821,
+ 46.15486957962745,
+ 6.267247605496281,
+ 38.65955739206124,
+ 16.334240831872595,
+ 40.92114763036668,
+ 44.25538155878388,
+ 46.79667178943268,
+ 5.886210147826818,
+ 45.086831193223446,
+ 47.3009972481073,
+ 47.07801971653764,
+ 46.80397795995714,
+ 46.806845163101094,
+ 43.42411625011456,
+ 46.37426980773864,
+ 41.17909401763616,
+ 46.16226579941339,
+ 47.44507636385267,
+ 11.930205494257288,
+ 16.233747914032552,
+ 6.031411752952078,
+ 45.92910900092996,
+ 47.47110773753601,
+ 39.494621036199604,
+ 16.734374432604927,
+ 47.37802539239185,
+ 46.74469194379278,
+ 16.087259096423576,
+ 46.92051488410033,
+ 47.34732444333283,
+ 46.40587690730415,
+ 5.872780467931287,
+ 44.55593583365237,
+ 45.7052618242163,
+ 6.085826627872682,
+ 44.846431805065144,
+ 45.41689502907426,
+ 45.289189315257374,
+ 44.95210230627078,
+ 42.99904025714732,
+ 46.839026962763846,
+ 6.250954782033121,
+ 44.8453124032084,
+ 45.278261112862296,
+ 6.020810288080093,
+ 17.182296973833214,
+ 46.63633652424215,
+ 5.866101016705892,
+ 46.160696572751434,
+ 46.32038287353405,
+ 46.89907461120633,
+ 45.95374406526204,
+ 46.925975948392896,
+ 46.42837166656114,
+ 15.78999329881552,
+ 44.465193132950446,
+ 46.21771478110725,
+ 47.314131714710484,
+ 6.0756954521719475,
+ 47.654756058723834,
+ 45.70610138140926,
+ 46.42506531228388,
+ 46.278376731444745,
+ 42.38396099575264,
+ 42.30031354989153,
+ 6.238343970049818,
+ 44.63197875047801,
+ 45.842276161134954,
+ 47.290515920449934,
+ 17.100464476837107,
+ 46.03336595920761,
+ 42.199011552033475,
+ 46.12151306088509,
+ 6.22230433569469,
+ 42.38409981463419,
+ 16.065182030558717,
+ 47.159068653554634,
+ 47.325440650358736,
+ 47.304702743784624,
+ 41.95305830151048,
+ 46.32090634094613,
+ 6.205841232502227,
+ 45.21525043209204,
+ 46.68630635575757,
+ 6.014917714514858,
+ 16.99660741175496,
+ 46.04707312586917,
+ 42.19662106675615,
+ 45.454018018858854,
+ 47.15352407193948,
+ 46.93603762078255,
+ 46.83396897378934,
+ 47.15013333226566,
+ 46.77541231643884,
+ 47.24502443147304,
+ 42.759813321329425,
+ 47.001201569266215,
+ 6.192232905623395,
+ 47.13098385966453,
+ 47.01234120088298,
+ 46.79153288884898,
+ 46.373378014241005,
+ 15.754365078113269,
+ 5.8675558701311985,
+ 45.42074545020536,
+ 6.176488223442546,
+ 47.27337589918247,
+ 46.90578973015155,
+ 47.16448140788897,
+ 47.56000914081759,
+ 46.62586586855627,
+ 41.982557140496446,
+ 16.770559660054925,
+ 47.00638722437522
]
-}
+}
\ No newline at end of file
diff --git a/tests/functional_tests/test_cases/gpt/gpt_dynamic_inference_tp1_pp1_dp8_583m_throughputtest_zmq/model_config.yaml b/tests/functional_tests/test_cases/gpt/gpt_dynamic_inference_tp1_pp1_dp8_583m_throughputtest_zmq/model_config.yaml
index 0ef4f95aaaf..89c64d6844a 100644
--- a/tests/functional_tests/test_cases/gpt/gpt_dynamic_inference_tp1_pp1_dp8_583m_throughputtest_zmq/model_config.yaml
+++ b/tests/functional_tests/test_cases/gpt/gpt_dynamic_inference_tp1_pp1_dp8_583m_throughputtest_zmq/model_config.yaml
@@ -44,7 +44,8 @@ MODEL_ARGS:
--inference-dynamic-batching-buffer-size-gb: 20
--inference-dynamic-batching-cuda-graph-max-tokens: 2048
--cuda-graph-impl: local
- --cuda-graph-scope: full
+ --cuda-graph-scope: full_iteration
+ --no-check-for-nan-in-loss-and-grad: true
--disable-chunked-prefill: true
--dist-ckpt-strictness: log_unexpected
--inference-ckpt-non-strict: true # To handle the extra_state errors
diff --git a/tests/functional_tests/test_cases/mixtral/deepseekv3_proxy_flex_tp1pp4emp16etp1cp1_release/model_config.yaml b/tests/functional_tests/test_cases/mixtral/deepseekv3_proxy_flex_tp1pp4emp16etp1cp1_release/model_config.yaml
index c16fedc7860..080f669e6a4 100644
--- a/tests/functional_tests/test_cases/mixtral/deepseekv3_proxy_flex_tp1pp4emp16etp1cp1_release/model_config.yaml
+++ b/tests/functional_tests/test_cases/mixtral/deepseekv3_proxy_flex_tp1pp4emp16etp1cp1_release/model_config.yaml
@@ -12,11 +12,12 @@ ENV_VARS:
NCCL_DEBUG: VERSION
NON_DETERMINSTIC_RESULTS: 1
NVSHMEM_IB_ENABLE_IBGDA: 0
+ CUDA_DEVICE_MAX_CONNECTIONS: 1
TEST_TYPE: "release"
MODEL_ARGS:
# Distributed args
--distributed-timeout-minutes: 60
- --tensor-model-parallel-size: 1
+ --tensor-model-parallel-size: 2
--pipeline-model-parallel-size: 4
--pipeline-model-parallel-layout: Et*2\\|\\(tt\\|\\)*5t\\|tmL # Et*2|(tt|)*5t|tmL
--expert-model-parallel-size: 16
@@ -47,8 +48,8 @@ MODEL_ARGS:
# Data args
--seq-length: 4096
--data-cache-path: ${DATA_CACHE_PATH}
- --tokenizer-type: HuggingFaceTokenizer
- --tokenizer-model: ${TOKENIZER_PATH}
+ --tokenizer-type: GPTSentencePieceTokenizer
+ --tokenizer-model: ${DATA_PATH}/utils/nemotron_2_256k.model
--data-path: $DATA_BLEND
--split: 99,1,0
--no-mmap-bin-files: true
@@ -81,12 +82,11 @@ MODEL_ARGS:
--qk-layernorm: true
# Add learning rate args
- --lr-decay-samples: 584765624
+ --lr-decay-samples: 24413696
--lr-warmup-samples: 1536000
- # Learning rate scaled down from 7.3e-6 (DeepSeek-V3 technical report, GBS=15360) to 3.9e-6 (GBS=8192)
- --lr-warmup-init: 3.9e-7
- --lr: 3.9e-6
- --min-lr: 3.9e-7
+ --lr-warmup-init: 1e-7
+ --lr: 1e-5
+ --min-lr: 1e-6
--lr-decay-style: cosine
--adam-beta1: 0.9
--adam-beta2: 0.95
@@ -127,8 +127,6 @@ MODEL_ARGS:
--eval-interval: 200
# Add checkpointing args
- --no-load-optim: true
- --no-load-rng: true
--auto-detect-ckpt-format:
true
# Add checkpointing args
diff --git a/tests/functional_tests/test_cases/mixtral/deepseekv3_proxy_flex_tp1pp4emp16etp1cp1_release_sm/golden_values_dev_dgx_h100.json b/tests/functional_tests/test_cases/mixtral/deepseekv3_proxy_flex_tp1pp4emp16etp1cp1_release_sm/golden_values_dev_dgx_h100.json
new file mode 100644
index 00000000000..f486950e5a2
--- /dev/null
+++ b/tests/functional_tests/test_cases/mixtral/deepseekv3_proxy_flex_tp1pp4emp16etp1cp1_release_sm/golden_values_dev_dgx_h100.json
@@ -0,0 +1,11492 @@
+{
+ "lm loss": {
+ "start_step": 1,
+ "end_step": 9535,
+ "step_interval": 5,
+ "values": {
+ "1": 13.89756,
+ "5": 13.89155,
+ "10": 13.85814,
+ "15": 13.84947,
+ "20": 13.74128,
+ "25": 13.71269,
+ "30": 13.39136,
+ "35": 13.32418,
+ "40": 13.23329,
+ "45": 13.12045,
+ "50": 12.53632,
+ "55": 12.35058,
+ "60": 12.17187,
+ "65": 12.01029,
+ "70": 11.83519,
+ "75": 11.55823,
+ "80": 11.30557,
+ "85": 11.11711,
+ "90": 10.96045,
+ "95": 10.79835,
+ "100": 10.58719,
+ "105": 10.45871,
+ "110": 10.23985,
+ "115": 10.03197,
+ "120": 9.88087,
+ "125": 9.74001,
+ "130": 9.64895,
+ "135": 9.58316,
+ "140": 9.34895,
+ "145": 9.3363,
+ "150": 9.17736,
+ "155": 9.11162,
+ "160": 9.02957,
+ "165": 8.91504,
+ "170": 8.86399,
+ "175": 8.82531,
+ "180": 8.68067,
+ "185": 8.72019,
+ "190": 8.59287,
+ "195": 8.59803,
+ "200": 8.48665,
+ "205": 8.39681,
+ "210": 8.35424,
+ "215": 8.40636,
+ "220": 8.27837,
+ "225": 8.29496,
+ "230": 8.27773,
+ "235": 8.20463,
+ "240": 8.15385,
+ "245": 8.1344,
+ "250": 8.06891,
+ "255": 8.08354,
+ "260": 7.97761,
+ "265": 7.96264,
+ "270": 7.91745,
+ "275": 7.9055,
+ "280": 7.89502,
+ "285": 7.91233,
+ "290": 7.858,
+ "295": 7.84326,
+ "300": 7.73922,
+ "305": 7.73479,
+ "310": 7.6998,
+ "315": 7.6959,
+ "320": 7.68835,
+ "325": 7.60857,
+ "330": 7.59888,
+ "335": 7.57833,
+ "340": 7.62257,
+ "345": 7.51187,
+ "350": 7.5063,
+ "355": 7.43406,
+ "360": 7.53414,
+ "365": 7.45759,
+ "370": 7.49186,
+ "375": 7.43607,
+ "380": 7.41292,
+ "385": 7.41117,
+ "390": 7.42986,
+ "395": 7.36781,
+ "400": 7.30747,
+ "405": 7.31834,
+ "410": 7.30943,
+ "415": 7.29421,
+ "420": 7.2965,
+ "425": 7.26158,
+ "430": 7.20979,
+ "435": 7.22197,
+ "440": 7.18512,
+ "445": 7.1687,
+ "450": 7.12181,
+ "455": 7.14062,
+ "460": 7.11041,
+ "465": 7.10497,
+ "470": 7.07645,
+ "475": 7.09742,
+ "480": 6.97587,
+ "485": 7.03312,
+ "490": 6.99478,
+ "495": 6.9692,
+ "500": 6.91435,
+ "505": 6.94713,
+ "510": 6.92309,
+ "515": 6.88853,
+ "520": 6.88024,
+ "525": 6.87529,
+ "530": 6.88311,
+ "535": 6.8642,
+ "540": 6.78769,
+ "545": 6.8252,
+ "550": 6.84568,
+ "555": 6.86869,
+ "560": 6.81372,
+ "565": 6.74969,
+ "570": 6.76579,
+ "575": 6.77872,
+ "580": 6.69766,
+ "585": 6.71359,
+ "590": 6.65449,
+ "595": 6.64792,
+ "600": 6.67016,
+ "605": 6.65924,
+ "610": 6.63641,
+ "615": 6.68438,
+ "620": 6.60355,
+ "625": 6.57203,
+ "630": 6.56964,
+ "635": 6.60732,
+ "640": 6.59738,
+ "645": 6.5815,
+ "650": 6.62582,
+ "655": 6.62475,
+ "660": 6.53171,
+ "665": 6.52224,
+ "670": 6.47146,
+ "675": 6.57058,
+ "680": 6.53989,
+ "685": 6.49695,
+ "690": 6.47037,
+ "695": 6.43685,
+ "700": 6.43121,
+ "705": 6.4313,
+ "710": 6.46058,
+ "715": 6.46842,
+ "720": 6.35254,
+ "725": 6.40344,
+ "730": 6.39123,
+ "735": 6.41174,
+ "740": 6.34886,
+ "745": 6.31567,
+ "750": 6.37227,
+ "755": 6.29068,
+ "760": 6.30783,
+ "765": 6.32016,
+ "770": 6.31539,
+ "775": 6.3051,
+ "780": 6.27484,
+ "785": 6.28635,
+ "790": 6.25066,
+ "795": 6.24498,
+ "800": 6.22595,
+ "805": 6.30241,
+ "810": 6.16125,
+ "815": 6.18921,
+ "820": 6.19984,
+ "825": 6.20878,
+ "830": 6.21184,
+ "835": 6.16547,
+ "840": 6.13918,
+ "845": 6.18907,
+ "850": 6.14544,
+ "855": 6.14245,
+ "860": 6.12573,
+ "865": 6.14471,
+ "870": 6.103,
+ "875": 6.14755,
+ "880": 6.09503,
+ "885": 6.08625,
+ "890": 6.14906,
+ "895": 6.03612,
+ "900": 6.06033,
+ "905": 6.07119,
+ "910": 6.04765,
+ "915": 6.02795,
+ "920": 6.01922,
+ "925": 6.00762,
+ "930": 6.04202,
+ "935": 6.03448,
+ "940": 5.96552,
+ "945": 6.00691,
+ "950": 6.02802,
+ "955": 5.9757,
+ "960": 5.9732,
+ "965": 5.8947,
+ "970": 5.93848,
+ "975": 5.94046,
+ "980": 5.91694,
+ "985": 5.91057,
+ "990": 5.96163,
+ "995": 5.87028,
+ "1000": 5.89819,
+ "1005": 5.85552,
+ "1010": 5.89001,
+ "1015": 5.91011,
+ "1020": 5.82121,
+ "1025": 5.81525,
+ "1030": 5.82852,
+ "1035": 5.91121,
+ "1040": 5.83477,
+ "1045": 5.80641,
+ "1050": 5.84029,
+ "1055": 5.82471,
+ "1060": 5.77657,
+ "1065": 5.75965,
+ "1070": 5.80228,
+ "1075": 5.78852,
+ "1080": 5.77993,
+ "1085": 5.79347,
+ "1090": 5.7642,
+ "1095": 5.77727,
+ "1100": 5.73679,
+ "1105": 5.71252,
+ "1110": 5.76864,
+ "1115": 5.69994,
+ "1120": 5.64073,
+ "1125": 5.65212,
+ "1130": 5.71653,
+ "1135": 5.67194,
+ "1140": 5.66144,
+ "1145": 5.65572,
+ "1150": 5.68319,
+ "1155": 5.64543,
+ "1160": 5.63371,
+ "1165": 5.67226,
+ "1170": 5.65589,
+ "1175": 5.62136,
+ "1180": 5.63006,
+ "1185": 5.6181,
+ "1190": 5.60413,
+ "1195": 5.59825,
+ "1200": 5.54202,
+ "1205": 5.65572,
+ "1210": 5.51312,
+ "1215": 5.55359,
+ "1220": 5.63431,
+ "1225": 5.51403,
+ "1230": 5.56754,
+ "1235": 5.521,
+ "1240": 5.55808,
+ "1245": 5.52886,
+ "1250": 5.51046,
+ "1255": 5.50279,
+ "1260": 5.50208,
+ "1265": 5.47964,
+ "1270": 5.44537,
+ "1275": 5.52448,
+ "1280": 5.45447,
+ "1285": 5.4682,
+ "1290": 5.43648,
+ "1295": 5.46181,
+ "1300": 5.46016,
+ "1305": 5.43278,
+ "1310": 5.38271,
+ "1315": 5.44073,
+ "1320": 5.42393,
+ "1325": 5.3568,
+ "1330": 5.41966,
+ "1335": 5.39498,
+ "1340": 5.44678,
+ "1345": 5.4046,
+ "1350": 5.3745,
+ "1355": 5.36722,
+ "1360": 5.37555,
+ "1365": 5.38819,
+ "1370": 5.31687,
+ "1375": 5.3257,
+ "1380": 5.37435,
+ "1385": 5.33822,
+ "1390": 5.32907,
+ "1395": 5.35996,
+ "1400": 5.34708,
+ "1405": 5.32768,
+ "1410": 5.30321,
+ "1415": 5.26874,
+ "1420": 5.31115,
+ "1425": 5.3045,
+ "1430": 5.33954,
+ "1435": 5.24914,
+ "1440": 5.27894,
+ "1445": 5.31118,
+ "1450": 5.28087,
+ "1455": 5.30455,
+ "1460": 5.26455,
+ "1465": 5.26355,
+ "1470": 5.29615,
+ "1475": 5.27116,
+ "1480": 5.26692,
+ "1485": 5.21939,
+ "1490": 5.21283,
+ "1495": 5.23155,
+ "1500": 5.23275,
+ "1505": 5.20436,
+ "1510": 5.22447,
+ "1515": 5.15502,
+ "1520": 5.1852,
+ "1525": 5.15413,
+ "1530": 5.17452,
+ "1535": 5.16098,
+ "1540": 5.16276,
+ "1545": 5.19593,
+ "1550": 5.1989,
+ "1555": 5.18478,
+ "1560": 5.1253,
+ "1565": 5.15973,
+ "1570": 5.17281,
+ "1575": 5.1468,
+ "1580": 5.16002,
+ "1585": 5.14495,
+ "1590": 5.12815,
+ "1595": 5.09691,
+ "1600": 5.17173,
+ "1605": 5.09626,
+ "1610": 5.10506,
+ "1615": 5.09978,
+ "1620": 5.1145,
+ "1625": 5.10983,
+ "1630": 5.08211,
+ "1635": 5.12902,
+ "1640": 5.09565,
+ "1645": 5.08916,
+ "1650": 5.08067,
+ "1655": 5.06625,
+ "1660": 5.05546,
+ "1665": 5.04609,
+ "1670": 5.06711,
+ "1675": 5.06871,
+ "1680": 5.00775,
+ "1685": 5.01672,
+ "1690": 4.99799,
+ "1695": 5.00065,
+ "1700": 5.03983,
+ "1705": 5.01824,
+ "1710": 5.00629,
+ "1715": 4.97587,
+ "1720": 4.97437,
+ "1725": 4.9984,
+ "1730": 4.95014,
+ "1735": 5.02541,
+ "1740": 4.95266,
+ "1745": 4.97461,
+ "1750": 4.95639,
+ "1755": 4.97133,
+ "1760": 4.98489,
+ "1765": 4.93728,
+ "1770": 4.93343,
+ "1775": 4.9432,
+ "1780": 4.96314,
+ "1785": 4.91574,
+ "1790": 4.93944,
+ "1795": 4.93848,
+ "1800": 4.88725,
+ "1805": 4.87771,
+ "1810": 4.8976,
+ "1815": 4.89801,
+ "1820": 4.8872,
+ "1825": 4.89371,
+ "1830": 4.8786,
+ "1835": 4.87542,
+ "1840": 4.87209,
+ "1845": 4.85811,
+ "1850": 4.83484,
+ "1855": 4.89133,
+ "1860": 4.84322,
+ "1865": 4.85108,
+ "1870": 4.82648,
+ "1875": 4.83877,
+ "1880": 4.89485,
+ "1885": 4.84392,
+ "1890": 4.8281,
+ "1895": 4.77339,
+ "1900": 4.81423,
+ "1905": 4.81232,
+ "1910": 4.82991,
+ "1915": 4.79768,
+ "1920": 4.78308,
+ "1925": 4.79277,
+ "1930": 4.76544,
+ "1935": 4.7941,
+ "1940": 4.75875,
+ "1945": 4.80214,
+ "1950": 4.83843,
+ "1955": 4.77731,
+ "1960": 4.76768,
+ "1965": 4.72596,
+ "1970": 4.73388,
+ "1975": 4.7973,
+ "1980": 4.73036,
+ "1985": 4.74162,
+ "1990": 4.78353,
+ "1995": 4.74959,
+ "2000": 4.76948,
+ "2005": 4.80113,
+ "2010": 4.70951,
+ "2015": 4.69715,
+ "2020": 4.71284,
+ "2025": 4.75821,
+ "2030": 4.68831,
+ "2035": 4.71528,
+ "2040": 4.67772,
+ "2045": 4.76255,
+ "2050": 4.74404,
+ "2055": 4.7077,
+ "2060": 4.70614,
+ "2065": 4.66526,
+ "2070": 4.67653,
+ "2075": 4.69507,
+ "2080": 4.66174,
+ "2085": 4.69911,
+ "2090": 4.61739,
+ "2095": 4.64746,
+ "2100": 4.61666,
+ "2105": 4.64633,
+ "2110": 4.64123,
+ "2115": 4.65336,
+ "2120": 4.64559,
+ "2125": 4.61059,
+ "2130": 4.61466,
+ "2135": 4.62745,
+ "2140": 4.6232,
+ "2145": 4.58124,
+ "2150": 4.60983,
+ "2155": 4.57956,
+ "2160": 4.60382,
+ "2165": 4.58415,
+ "2170": 4.61387,
+ "2175": 4.60275,
+ "2180": 4.59531,
+ "2185": 4.60788,
+ "2190": 4.58246,
+ "2195": 4.55672,
+ "2200": 4.55346,
+ "2205": 4.56383,
+ "2210": 4.6146,
+ "2215": 4.64276,
+ "2220": 4.59912,
+ "2225": 4.57263,
+ "2230": 4.56854,
+ "2235": 4.61797,
+ "2240": 4.51401,
+ "2245": 4.5176,
+ "2250": 4.52905,
+ "2255": 4.54117,
+ "2260": 4.48536,
+ "2265": 4.56489,
+ "2270": 4.49655,
+ "2275": 4.55547,
+ "2280": 4.51075,
+ "2285": 4.53333,
+ "2290": 4.52269,
+ "2295": 4.52707,
+ "2300": 4.53228,
+ "2305": 4.49287,
+ "2310": 4.53148,
+ "2315": 4.46329,
+ "2320": 4.51121,
+ "2325": 4.49336,
+ "2330": 4.49351,
+ "2335": 4.47787,
+ "2340": 4.48626,
+ "2345": 4.52525,
+ "2350": 4.4674,
+ "2355": 4.47173,
+ "2360": 4.44099,
+ "2365": 4.44682,
+ "2370": 4.44716,
+ "2375": 4.44199,
+ "2380": 4.39487,
+ "2385": 4.43475,
+ "2390": 4.43071,
+ "2395": 4.46719,
+ "2400": 4.42074,
+ "2405": 4.40081,
+ "2410": 4.44955,
+ "2415": 4.42055,
+ "2420": 4.4293,
+ "2425": 4.39783,
+ "2430": 4.42084,
+ "2435": 4.40291,
+ "2440": 4.39501,
+ "2445": 4.40808,
+ "2450": 4.38239,
+ "2455": 4.4178,
+ "2460": 4.36606,
+ "2465": 4.41327,
+ "2470": 4.40023,
+ "2475": 4.41776,
+ "2480": 4.34092,
+ "2485": 4.37423,
+ "2490": 4.37838,
+ "2495": 4.35662,
+ "2500": 4.36528,
+ "2505": 4.37219,
+ "2510": 4.41251,
+ "2515": 4.40356,
+ "2520": 4.34516,
+ "2525": 4.36214,
+ "2530": 4.36786,
+ "2535": 4.36686,
+ "2540": 4.36548,
+ "2545": 4.37687,
+ "2550": 4.30337,
+ "2555": 4.37244,
+ "2560": 4.35158,
+ "2565": 4.30393,
+ "2570": 4.33393,
+ "2575": 4.30697,
+ "2580": 4.30582,
+ "2585": 4.29358,
+ "2590": 4.31272,
+ "2595": 4.28154,
+ "2600": 4.29867,
+ "2605": 4.31115,
+ "2610": 4.32106,
+ "2615": 4.27768,
+ "2620": 4.26935,
+ "2625": 4.30437,
+ "2630": 4.22434,
+ "2635": 4.30369,
+ "2640": 4.30012,
+ "2645": 4.2581,
+ "2650": 4.28639,
+ "2655": 4.26647,
+ "2660": 4.21474,
+ "2665": 4.30436,
+ "2670": 4.26382,
+ "2675": 4.2306,
+ "2680": 4.25227,
+ "2685": 4.25736,
+ "2690": 4.22986,
+ "2695": 4.28379,
+ "2700": 4.19098,
+ "2705": 4.23853,
+ "2710": 4.25092,
+ "2715": 4.23481,
+ "2720": 4.24356,
+ "2725": 4.2225,
+ "2730": 4.22941,
+ "2735": 4.22363,
+ "2740": 4.20346,
+ "2745": 4.18765,
+ "2750": 4.21101,
+ "2755": 4.22237,
+ "2760": 4.22902,
+ "2765": 4.18298,
+ "2770": 4.23755,
+ "2775": 4.17706,
+ "2780": 4.21186,
+ "2785": 4.19469,
+ "2790": 4.21736,
+ "2795": 4.18988,
+ "2800": 4.1159,
+ "2805": 4.16613,
+ "2810": 4.17076,
+ "2815": 4.15389,
+ "2820": 4.1969,
+ "2825": 4.19241,
+ "2830": 4.16864,
+ "2835": 4.17046,
+ "2840": 4.16148,
+ "2845": 4.14967,
+ "2850": 4.16619,
+ "2855": 4.11805,
+ "2860": 4.14572,
+ "2865": 4.17023,
+ "2870": 4.14096,
+ "2875": 4.1596,
+ "2880": 4.08582,
+ "2885": 4.14242,
+ "2890": 4.11503,
+ "2895": 4.15452,
+ "2900": 4.09735,
+ "2905": 4.11101,
+ "2910": 4.10798,
+ "2915": 4.14914,
+ "2920": 4.12546,
+ "2925": 4.10099,
+ "2930": 4.08522,
+ "2935": 4.07896,
+ "2940": 4.09225,
+ "2945": 4.06113,
+ "2950": 4.03479,
+ "2955": 4.03763,
+ "2960": 4.04955,
+ "2965": 4.0643,
+ "2970": 4.08593,
+ "2975": 4.0941,
+ "2980": 4.03102,
+ "2985": 4.07394,
+ "2990": 4.08923,
+ "2995": 4.03231,
+ "3000": 4.0436,
+ "3005": 4.02568,
+ "3010": 4.06747,
+ "3015": 4.02305,
+ "3020": 4.03992,
+ "3025": 4.02491,
+ "3030": 4.0567,
+ "3035": 4.04059,
+ "3040": 4.0544,
+ "3045": 4.04677,
+ "3050": 4.017,
+ "3055": 4.00507,
+ "3060": 3.9904,
+ "3065": 4.02281,
+ "3070": 4.03826,
+ "3075": 3.97211,
+ "3080": 4.0011,
+ "3085": 4.00548,
+ "3090": 4.00887,
+ "3095": 4.02745,
+ "3100": 4.01465,
+ "3105": 3.99035,
+ "3110": 3.99124,
+ "3115": 3.92509,
+ "3120": 4.00505,
+ "3125": 3.94183,
+ "3130": 3.96987,
+ "3135": 3.96132,
+ "3140": 3.95209,
+ "3145": 3.93524,
+ "3150": 3.96949,
+ "3155": 3.96213,
+ "3160": 3.96255,
+ "3165": 3.96146,
+ "3170": 3.96456,
+ "3175": 3.93165,
+ "3180": 3.93784,
+ "3185": 3.90234,
+ "3190": 3.92455,
+ "3195": 3.9116,
+ "3200": 3.89013,
+ "3205": 3.92029,
+ "3210": 3.89711,
+ "3215": 3.90569,
+ "3220": 3.89706,
+ "3225": 3.91097,
+ "3230": 3.89895,
+ "3235": 3.91122,
+ "3240": 3.88912,
+ "3245": 3.88902,
+ "3250": 3.84407,
+ "3255": 3.89259,
+ "3260": 3.88283,
+ "3265": 3.92603,
+ "3270": 3.9052,
+ "3275": 3.85915,
+ "3280": 3.88232,
+ "3285": 3.86652,
+ "3290": 3.86681,
+ "3295": 3.83806,
+ "3300": 3.85349,
+ "3305": 3.86048,
+ "3310": 3.85872,
+ "3315": 3.89673,
+ "3320": 3.85179,
+ "3325": 3.84353,
+ "3330": 3.82539,
+ "3335": 3.86213,
+ "3340": 3.81824,
+ "3345": 3.83129,
+ "3350": 3.85901,
+ "3355": 3.8452,
+ "3360": 3.83241,
+ "3365": 3.83682,
+ "3370": 3.82265,
+ "3375": 3.85232,
+ "3380": 3.79563,
+ "3385": 3.81353,
+ "3390": 3.79143,
+ "3395": 3.86888,
+ "3400": 3.83997,
+ "3405": 3.86197,
+ "3410": 3.77529,
+ "3415": 3.72916,
+ "3420": 3.80048,
+ "3425": 3.81237,
+ "3430": 3.84497,
+ "3435": 3.80796,
+ "3440": 3.8267,
+ "3445": 3.7742,
+ "3450": 3.78787,
+ "3455": 3.80217,
+ "3460": 3.78265,
+ "3465": 3.75891,
+ "3470": 3.77341,
+ "3475": 3.77638,
+ "3480": 3.77988,
+ "3485": 3.80588,
+ "3490": 3.76958,
+ "3495": 3.80315,
+ "3500": 3.77047,
+ "3505": 3.77239,
+ "3510": 3.75092,
+ "3515": 3.80896,
+ "3520": 3.79879,
+ "3525": 3.76372,
+ "3530": 3.75322,
+ "3535": 3.76209,
+ "3540": 3.81796,
+ "3545": 3.72915,
+ "3550": 3.79201,
+ "3555": 3.72604,
+ "3560": 3.78622,
+ "3565": 3.7451,
+ "3570": 3.74254,
+ "3575": 3.71868,
+ "3580": 3.77066,
+ "3585": 3.76174,
+ "3590": 3.68853,
+ "3595": 3.76509,
+ "3600": 3.71336,
+ "3605": 3.71948,
+ "3610": 3.70916,
+ "3615": 3.74868,
+ "3620": 3.7837,
+ "3625": 3.71964,
+ "3630": 3.76519,
+ "3635": 3.68617,
+ "3640": 3.7093,
+ "3645": 3.74263,
+ "3650": 3.69638,
+ "3655": 3.72074,
+ "3660": 3.72832,
+ "3665": 3.74694,
+ "3670": 3.71178,
+ "3675": 3.71065,
+ "3680": 3.72416,
+ "3685": 3.67473,
+ "3690": 3.6936,
+ "3695": 3.68528,
+ "3700": 3.70814,
+ "3705": 3.67651,
+ "3710": 3.68493,
+ "3715": 3.6842,
+ "3720": 3.66563,
+ "3725": 3.64716,
+ "3730": 3.64883,
+ "3735": 3.68782,
+ "3740": 3.6732,
+ "3745": 3.66354,
+ "3750": 3.6757,
+ "3755": 3.66351,
+ "3760": 3.67285,
+ "3765": 3.66004,
+ "3770": 3.6516,
+ "3775": 3.63831,
+ "3780": 3.62453,
+ "3785": 3.6765,
+ "3790": 3.60163,
+ "3795": 3.64291,
+ "3800": 3.63275,
+ "3805": 3.62032,
+ "3810": 3.59475,
+ "3815": 3.63585,
+ "3820": 3.64099,
+ "3825": 3.6535,
+ "3830": 3.63864,
+ "3835": 3.59938,
+ "3840": 3.67685,
+ "3845": 3.65895,
+ "3850": 3.60064,
+ "3855": 3.60428,
+ "3860": 3.65711,
+ "3865": 3.60867,
+ "3870": 3.6721,
+ "3875": 3.58596,
+ "3880": 3.58212,
+ "3885": 3.60502,
+ "3890": 3.60969,
+ "3895": 3.5558,
+ "3900": 3.61685,
+ "3905": 3.59135,
+ "3910": 3.5772,
+ "3915": 3.5862,
+ "3920": 3.57131,
+ "3925": 3.56751,
+ "3930": 3.58005,
+ "3935": 3.5821,
+ "3940": 3.57511,
+ "3945": 3.56965,
+ "3950": 3.61887,
+ "3955": 3.57531,
+ "3960": 3.60735,
+ "3965": 3.58853,
+ "3970": 3.56735,
+ "3975": 3.56709,
+ "3980": 3.5304,
+ "3985": 3.60527,
+ "3990": 3.58124,
+ "3995": 3.60753,
+ "4000": 3.55811,
+ "4005": 3.54162,
+ "4010": 3.58376,
+ "4015": 3.58398,
+ "4020": 3.58355,
+ "4025": 3.57409,
+ "4030": 3.62855,
+ "4035": 3.57033,
+ "4040": 3.5882,
+ "4045": 3.60161,
+ "4050": 3.57522,
+ "4055": 3.57403,
+ "4060": 3.5888,
+ "4065": 3.58382,
+ "4070": 3.51488,
+ "4075": 3.55887,
+ "4080": 3.53108,
+ "4085": 3.54596,
+ "4090": 3.54584,
+ "4095": 3.53161,
+ "4100": 3.55106,
+ "4105": 3.53794,
+ "4110": 3.51736,
+ "4115": 3.56348,
+ "4120": 3.49648,
+ "4125": 3.49769,
+ "4130": 3.55149,
+ "4135": 3.54373,
+ "4140": 3.49112,
+ "4145": 3.51351,
+ "4150": 3.55497,
+ "4155": 3.48797,
+ "4160": 3.54539,
+ "4165": 3.56451,
+ "4170": 3.50424,
+ "4175": 3.50239,
+ "4180": 3.4998,
+ "4185": 3.5138,
+ "4190": 3.5011,
+ "4195": 3.50044,
+ "4200": 3.49424,
+ "4205": 3.53032,
+ "4210": 3.51921,
+ "4215": 3.52292,
+ "4220": 3.53088,
+ "4225": 3.50168,
+ "4230": 3.49756,
+ "4235": 3.52008,
+ "4240": 3.49249,
+ "4245": 3.49542,
+ "4250": 3.48848,
+ "4255": 3.50707,
+ "4260": 3.4676,
+ "4265": 3.48819,
+ "4270": 3.50473,
+ "4275": 3.53933,
+ "4280": 3.48997,
+ "4285": 3.50947,
+ "4290": 3.48405,
+ "4295": 3.48692,
+ "4300": 3.52631,
+ "4305": 3.48704,
+ "4310": 3.51358,
+ "4315": 3.50638,
+ "4320": 3.50379,
+ "4325": 3.51699,
+ "4330": 3.45992,
+ "4335": 3.49232,
+ "4340": 3.50354,
+ "4345": 3.43189,
+ "4350": 3.44845,
+ "4355": 3.52327,
+ "4360": 3.48083,
+ "4365": 3.47079,
+ "4370": 3.47624,
+ "4375": 3.44129,
+ "4380": 3.44296,
+ "4385": 3.42527,
+ "4390": 3.49048,
+ "4395": 3.47699,
+ "4400": 3.47442,
+ "4405": 3.41723,
+ "4410": 3.48335,
+ "4415": 3.44899,
+ "4420": 3.44113,
+ "4425": 3.47273,
+ "4430": 3.44742,
+ "4435": 3.49082,
+ "4440": 3.48522,
+ "4445": 3.43744,
+ "4450": 3.3974,
+ "4455": 3.4624,
+ "4460": 3.43415,
+ "4465": 3.45284,
+ "4470": 3.42199,
+ "4475": 3.45352,
+ "4480": 3.44375,
+ "4485": 3.43643,
+ "4490": 3.43453,
+ "4495": 3.38677,
+ "4500": 3.45384,
+ "4505": 3.43515,
+ "4510": 3.44292,
+ "4515": 3.40605,
+ "4520": 3.43888,
+ "4525": 3.40731,
+ "4530": 3.44131,
+ "4535": 3.3963,
+ "4540": 3.42067,
+ "4545": 3.43217,
+ "4550": 3.47418,
+ "4555": 3.39854,
+ "4560": 3.42732,
+ "4565": 3.37837,
+ "4570": 3.41702,
+ "4575": 3.41117,
+ "4580": 3.45362,
+ "4585": 3.42636,
+ "4590": 3.42388,
+ "4595": 3.39853,
+ "4600": 3.39686,
+ "4605": 3.42144,
+ "4610": 3.41286,
+ "4615": 3.45309,
+ "4620": 3.39526,
+ "4625": 3.42534,
+ "4630": 3.4127,
+ "4635": 3.39195,
+ "4640": 3.4264,
+ "4645": 3.41975,
+ "4650": 3.43542,
+ "4655": 3.40687,
+ "4660": 3.39737,
+ "4665": 3.41231,
+ "4670": 3.446,
+ "4675": 3.40423,
+ "4680": 3.42886,
+ "4685": 3.42464,
+ "4690": 3.39897,
+ "4695": 3.38,
+ "4700": 3.3729,
+ "4705": 3.35029,
+ "4710": 3.40571,
+ "4715": 3.39222,
+ "4720": 3.38774,
+ "4725": 3.35968,
+ "4730": 3.39519,
+ "4735": 3.32069,
+ "4740": 3.36458,
+ "4745": 3.40698,
+ "4750": 3.36053,
+ "4755": 3.39053,
+ "4760": 3.41421,
+ "4765": 3.36022,
+ "4770": 3.36502,
+ "4775": 3.36135,
+ "4780": 3.37362,
+ "4785": 3.374,
+ "4790": 3.41163,
+ "4795": 3.39334,
+ "4800": 3.34583,
+ "4805": 3.41139,
+ "4810": 3.35086,
+ "4815": 3.38903,
+ "4820": 3.34814,
+ "4825": 3.40406,
+ "4830": 3.38314,
+ "4835": 3.3693,
+ "4840": 3.38086,
+ "4845": 3.32726,
+ "4850": 3.39372,
+ "4855": 3.39679,
+ "4860": 3.32727,
+ "4865": 3.36392,
+ "4870": 3.34896,
+ "4875": 3.39123,
+ "4880": 3.39974,
+ "4885": 3.35153,
+ "4890": 3.36191,
+ "4895": 3.35318,
+ "4900": 3.32971,
+ "4905": 3.33008,
+ "4910": 3.32861,
+ "4915": 3.37524,
+ "4920": 3.35807,
+ "4925": 3.31242,
+ "4930": 3.34376,
+ "4935": 3.3273,
+ "4940": 3.28784,
+ "4945": 3.36034,
+ "4950": 3.29629,
+ "4955": 3.40365,
+ "4960": 3.3479,
+ "4965": 3.34204,
+ "4970": 3.33369,
+ "4975": 3.34388,
+ "4980": 3.36573,
+ "4985": 3.35352,
+ "4990": 3.33542,
+ "4995": 3.3795,
+ "5000": 3.30893,
+ "5005": 3.35715,
+ "5010": 3.36146,
+ "5015": 3.30923,
+ "5020": 3.28653,
+ "5025": 3.31605,
+ "5030": 3.32648,
+ "5035": 3.32963,
+ "5040": 3.30481,
+ "5045": 3.34994,
+ "5050": 3.30693,
+ "5055": 3.32632,
+ "5060": 3.28843,
+ "5065": 3.33396,
+ "5070": 3.33431,
+ "5075": 3.34337,
+ "5080": 3.31868,
+ "5085": 3.34518,
+ "5090": 3.32323,
+ "5095": 3.29022,
+ "5100": 3.32026,
+ "5105": 3.32744,
+ "5110": 3.3329,
+ "5115": 3.3038,
+ "5120": 3.34196,
+ "5125": 3.3184,
+ "5130": 3.31738,
+ "5135": 3.30105,
+ "5140": 3.3111,
+ "5145": 3.31125,
+ "5150": 3.32063,
+ "5155": 3.31567,
+ "5160": 3.31039,
+ "5165": 3.34534,
+ "5170": 3.23105,
+ "5175": 3.31877,
+ "5180": 3.28445,
+ "5185": 3.30691,
+ "5190": 3.32611,
+ "5195": 3.30561,
+ "5200": 3.31019,
+ "5205": 3.34654,
+ "5210": 3.28506,
+ "5215": 3.2874,
+ "5220": 3.28219,
+ "5225": 3.28677,
+ "5230": 3.32011,
+ "5235": 3.27975,
+ "5240": 3.27349,
+ "5245": 3.29646,
+ "5250": 3.3023,
+ "5255": 3.28615,
+ "5260": 3.31039,
+ "5265": 3.27007,
+ "5270": 3.25412,
+ "5275": 3.25534,
+ "5280": 3.28407,
+ "5285": 3.30874,
+ "5290": 3.2589,
+ "5295": 3.27448,
+ "5300": 3.27858,
+ "5305": 3.26656,
+ "5310": 3.32809,
+ "5315": 3.25873,
+ "5320": 3.30633,
+ "5325": 3.3111,
+ "5330": 3.27899,
+ "5335": 3.28833,
+ "5340": 3.23016,
+ "5345": 3.28336,
+ "5350": 3.28737,
+ "5355": 3.28737,
+ "5360": 3.23407,
+ "5365": 3.25011,
+ "5370": 3.28855,
+ "5375": 3.26985,
+ "5380": 3.24418,
+ "5385": 3.28394,
+ "5390": 3.28221,
+ "5395": 3.20448,
+ "5400": 3.30114,
+ "5405": 3.21525,
+ "5410": 3.29188,
+ "5415": 3.22284,
+ "5420": 3.25707,
+ "5425": 3.23689,
+ "5430": 3.24779,
+ "5435": 3.2811,
+ "5440": 3.21236,
+ "5445": 3.24176,
+ "5450": 3.24576,
+ "5455": 3.22991,
+ "5460": 3.25196,
+ "5465": 3.29692,
+ "5470": 3.27194,
+ "5475": 3.20136,
+ "5480": 3.28214,
+ "5485": 3.24325,
+ "5490": 3.26633,
+ "5495": 3.27183,
+ "5500": 3.22718,
+ "5505": 3.23914,
+ "5510": 3.28342,
+ "5515": 3.27035,
+ "5520": 3.23742,
+ "5525": 3.28473,
+ "5530": 3.22923,
+ "5535": 3.26258,
+ "5540": 3.25366,
+ "5545": 3.26198,
+ "5550": 3.24962,
+ "5555": 3.22875,
+ "5560": 3.22306,
+ "5565": 3.26845,
+ "5570": 3.22989,
+ "5575": 3.26435,
+ "5580": 3.23553,
+ "5585": 3.18594,
+ "5590": 3.24664,
+ "5595": 3.2105,
+ "5600": 3.25488,
+ "5605": 3.17461,
+ "5610": 3.2604,
+ "5615": 3.25606,
+ "5620": 3.2609,
+ "5625": 3.25214,
+ "5630": 3.24091,
+ "5635": 3.21924,
+ "5640": 3.24377,
+ "5645": 3.20743,
+ "5650": 3.2076,
+ "5655": 3.20542,
+ "5660": 3.20971,
+ "5665": 3.21069,
+ "5670": 3.20056,
+ "5675": 3.22863,
+ "5680": 3.19922,
+ "5685": 3.20573,
+ "5690": 3.2077,
+ "5695": 3.24414,
+ "5700": 3.19628,
+ "5705": 3.18515,
+ "5710": 3.17855,
+ "5715": 3.28582,
+ "5720": 3.2496,
+ "5725": 3.2002,
+ "5730": 3.24085,
+ "5735": 3.22905,
+ "5740": 3.22477,
+ "5745": 3.20281,
+ "5750": 3.23329,
+ "5755": 3.23832,
+ "5760": 3.22288,
+ "5765": 3.22651,
+ "5770": 3.25303,
+ "5775": 3.19712,
+ "5780": 3.21565,
+ "5785": 3.21756,
+ "5790": 3.22715,
+ "5795": 3.22463,
+ "5800": 3.16888,
+ "5805": 3.18332,
+ "5810": 3.22432,
+ "5815": 3.20302,
+ "5820": 3.16241,
+ "5825": 3.20754,
+ "5830": 3.1647,
+ "5835": 3.17395,
+ "5840": 3.20628,
+ "5845": 3.217,
+ "5850": 3.21594,
+ "5855": 3.15148,
+ "5860": 3.17119,
+ "5865": 3.20009,
+ "5870": 3.16136,
+ "5875": 3.20014,
+ "5880": 3.19456,
+ "5885": 3.19488,
+ "5890": 3.21776,
+ "5895": 3.23301,
+ "5900": 3.1895,
+ "5905": 3.21986,
+ "5910": 3.20185,
+ "5915": 3.17464,
+ "5920": 3.1915,
+ "5925": 3.15681,
+ "5930": 3.19135,
+ "5935": 3.19128,
+ "5940": 3.2051,
+ "5945": 3.21968,
+ "5950": 3.20213,
+ "5955": 3.16275,
+ "5960": 3.22598,
+ "5965": 3.17666,
+ "5970": 3.21828,
+ "5975": 3.18539,
+ "5980": 3.25556,
+ "5985": 3.14035,
+ "5990": 3.2373,
+ "5995": 3.15341,
+ "6000": 3.17562,
+ "6005": 3.15642,
+ "6010": 3.15958,
+ "6015": 3.16383,
+ "6020": 3.17057,
+ "6025": 3.20846,
+ "6030": 3.14683,
+ "6035": 3.20108,
+ "6040": 3.18034,
+ "6045": 3.19784,
+ "6050": 3.19841,
+ "6055": 3.17123,
+ "6060": 3.18513,
+ "6065": 3.20946,
+ "6070": 3.16514,
+ "6075": 3.13204,
+ "6080": 3.19182,
+ "6085": 3.15022,
+ "6090": 3.18799,
+ "6095": 3.18454,
+ "6100": 3.13968,
+ "6105": 3.18911,
+ "6110": 3.13194,
+ "6115": 3.18032,
+ "6120": 3.17268,
+ "6125": 3.17817,
+ "6130": 3.16826,
+ "6135": 3.16641,
+ "6140": 3.16491,
+ "6145": 3.14203,
+ "6150": 3.17849,
+ "6155": 3.14973,
+ "6160": 3.12836,
+ "6165": 3.15943,
+ "6170": 3.14366,
+ "6175": 3.14619,
+ "6180": 3.14564,
+ "6185": 3.18694,
+ "6190": 3.15491,
+ "6195": 3.12582,
+ "6200": 3.15218,
+ "6205": 3.14598,
+ "6210": 3.10092,
+ "6215": 3.15518,
+ "6220": 3.1544,
+ "6225": 3.17142,
+ "6230": 3.10668,
+ "6235": 3.14063,
+ "6240": 3.08394,
+ "6245": 3.18223,
+ "6250": 3.14309,
+ "6255": 3.15773,
+ "6260": 3.14125,
+ "6265": 3.15597,
+ "6270": 3.10065,
+ "6275": 3.12382,
+ "6280": 3.13503,
+ "6285": 3.11829,
+ "6290": 3.14415,
+ "6295": 3.15298,
+ "6300": 3.15403,
+ "6305": 3.21086,
+ "6310": 3.11266,
+ "6315": 3.10982,
+ "6320": 3.16047,
+ "6325": 3.10246,
+ "6330": 3.16954,
+ "6335": 3.15391,
+ "6340": 3.10904,
+ "6345": 3.16578,
+ "6350": 3.11808,
+ "6355": 3.11742,
+ "6360": 3.1108,
+ "6365": 3.14775,
+ "6370": 3.16278,
+ "6375": 3.1337,
+ "6380": 3.15125,
+ "6385": 3.17081,
+ "6390": 3.12597,
+ "6395": 3.10466,
+ "6400": 3.10591,
+ "6405": 3.18617,
+ "6410": 3.17298,
+ "6415": 3.12537,
+ "6420": 3.17096,
+ "6425": 3.17458,
+ "6430": 3.16659,
+ "6435": 3.12451,
+ "6440": 3.13606,
+ "6445": 3.15196,
+ "6450": 3.09161,
+ "6455": 3.08666,
+ "6460": 3.13082,
+ "6465": 3.16786,
+ "6470": 3.13951,
+ "6475": 3.13285,
+ "6480": 3.15191,
+ "6485": 3.11206,
+ "6490": 3.0797,
+ "6495": 3.16564,
+ "6500": 3.14177,
+ "6505": 3.08566,
+ "6510": 3.14483,
+ "6515": 3.16369,
+ "6520": 3.09044,
+ "6525": 3.14867,
+ "6530": 3.10896,
+ "6535": 3.12403,
+ "6540": 3.18005,
+ "6545": 3.11404,
+ "6550": 3.11103,
+ "6555": 3.10947,
+ "6560": 3.0737,
+ "6565": 3.07934,
+ "6570": 3.10438,
+ "6575": 3.05844,
+ "6580": 3.17411,
+ "6585": 3.10694,
+ "6590": 3.0877,
+ "6595": 3.10332,
+ "6600": 3.1032,
+ "6605": 3.08625,
+ "6610": 3.08405,
+ "6615": 3.1316,
+ "6620": 3.076,
+ "6625": 3.09705,
+ "6630": 3.09309,
+ "6635": 3.12933,
+ "6640": 3.08864,
+ "6645": 3.10948,
+ "6650": 3.1378,
+ "6655": 3.07416,
+ "6660": 3.11313,
+ "6665": 3.12487,
+ "6670": 3.08048,
+ "6675": 3.10457,
+ "6680": 3.10673,
+ "6685": 3.14077,
+ "6690": 3.11651,
+ "6695": 3.12176,
+ "6700": 3.1127,
+ "6705": 3.09107,
+ "6710": 3.10728,
+ "6715": 3.05842,
+ "6720": 3.13504,
+ "6725": 3.12621,
+ "6730": 3.1099,
+ "6735": 3.10898,
+ "6740": 3.11731,
+ "6745": 3.0901,
+ "6750": 3.10983,
+ "6755": 3.06749,
+ "6760": 3.06624,
+ "6765": 3.08509,
+ "6770": 3.07057,
+ "6775": 3.10523,
+ "6780": 3.07455,
+ "6785": 3.07959,
+ "6790": 3.10472,
+ "6795": 3.07166,
+ "6800": 3.09692,
+ "6805": 3.08719,
+ "6810": 3.10858,
+ "6815": 3.04354,
+ "6820": 3.07401,
+ "6825": 3.10257,
+ "6830": 3.08637,
+ "6835": 3.06002,
+ "6840": 3.0654,
+ "6845": 3.11054,
+ "6850": 3.08009,
+ "6855": 3.11065,
+ "6860": 3.06305,
+ "6865": 3.10876,
+ "6870": 3.07538,
+ "6875": 3.07578,
+ "6880": 3.08642,
+ "6885": 3.05135,
+ "6890": 3.0749,
+ "6895": 3.05299,
+ "6900": 3.05973,
+ "6905": 3.07506,
+ "6910": 3.09159,
+ "6915": 3.11333,
+ "6920": 3.06615,
+ "6925": 3.08379,
+ "6930": 3.06742,
+ "6935": 3.02485,
+ "6940": 3.06623,
+ "6945": 3.05639,
+ "6950": 3.07964,
+ "6955": 3.05853,
+ "6960": 3.05554,
+ "6965": 3.09907,
+ "6970": 3.03589,
+ "6975": 3.1075,
+ "6980": 3.06776,
+ "6985": 3.06784,
+ "6990": 3.11146,
+ "6995": 3.09126,
+ "7000": 3.02783,
+ "7005": 3.09757,
+ "7010": 3.0779,
+ "7015": 3.07385,
+ "7020": 3.10018,
+ "7025": 3.08417,
+ "7030": 3.08746,
+ "7035": 3.04096,
+ "7040": 3.01984,
+ "7045": 3.07968,
+ "7050": 3.09817,
+ "7055": 3.03816,
+ "7060": 3.09848,
+ "7065": 3.11109,
+ "7070": 3.05748,
+ "7075": 3.06319,
+ "7080": 3.11208,
+ "7085": 3.03557,
+ "7090": 3.05692,
+ "7095": 3.04652,
+ "7100": 3.07149,
+ "7105": 3.02035,
+ "7110": 3.0623,
+ "7115": 3.03547,
+ "7120": 3.07999,
+ "7125": 3.03377,
+ "7130": 3.04883,
+ "7135": 3.05627,
+ "7140": 3.06014,
+ "7145": 3.0691,
+ "7150": 3.02375,
+ "7155": 3.08612,
+ "7160": 3.0047,
+ "7165": 3.0418,
+ "7170": 3.07701,
+ "7175": 3.03661,
+ "7180": 3.07042,
+ "7185": 3.09125,
+ "7190": 3.05302,
+ "7195": 3.06058,
+ "7200": 3.06039,
+ "7205": 3.04153,
+ "7210": 3.08703,
+ "7215": 3.06723,
+ "7220": 3.08798,
+ "7225": 3.06993,
+ "7230": 3.07403,
+ "7235": 3.05435,
+ "7240": 3.05017,
+ "7245": 3.07131,
+ "7250": 3.01274,
+ "7255": 3.03229,
+ "7260": 3.06928,
+ "7265": 3.00261,
+ "7270": 3.04138,
+ "7275": 3.04223,
+ "7280": 3.04181,
+ "7285": 3.05407,
+ "7290": 3.07344,
+ "7295": 3.06537,
+ "7300": 3.02809,
+ "7305": 3.02877,
+ "7310": 3.04926,
+ "7315": 3.07646,
+ "7320": 3.05669,
+ "7325": 3.06149,
+ "7330": 3.02592,
+ "7335": 3.02733,
+ "7340": 3.06004,
+ "7345": 3.0091,
+ "7350": 3.06031,
+ "7355": 3.04495,
+ "7360": 3.03923,
+ "7365": 3.03845,
+ "7370": 3.03136,
+ "7375": 2.9999,
+ "7380": 3.06202,
+ "7385": 3.07693,
+ "7390": 3.06411,
+ "7395": 3.02221,
+ "7400": 3.07516,
+ "7405": 3.04382,
+ "7410": 3.06023,
+ "7415": 3.05228,
+ "7420": 3.03261,
+ "7425": 3.08586,
+ "7430": 3.0272,
+ "7435": 3.01757,
+ "7440": 3.0377,
+ "7445": 3.01394,
+ "7450": 2.99482,
+ "7455": 3.04735,
+ "7460": 3.04105,
+ "7465": 3.04977,
+ "7470": 3.05673,
+ "7475": 3.06741,
+ "7480": 3.02749,
+ "7485": 2.98653,
+ "7490": 2.98973,
+ "7495": 2.99863,
+ "7500": 3.02945,
+ "7505": 3.0059,
+ "7510": 2.97871,
+ "7515": 3.02404,
+ "7520": 3.01697,
+ "7525": 2.98295,
+ "7530": 3.02636,
+ "7535": 3.04423,
+ "7540": 3.02494,
+ "7545": 3.0588,
+ "7550": 3.06534,
+ "7555": 3.00732,
+ "7560": 3.01283,
+ "7565": 3.00874,
+ "7570": 3.03442,
+ "7575": 2.97962,
+ "7580": 3.03034,
+ "7585": 3.01793,
+ "7590": 3.01504,
+ "7595": 3.07403,
+ "7600": 3.03015,
+ "7605": 3.02144,
+ "7610": 3.00533,
+ "7615": 2.99602,
+ "7620": 2.99265,
+ "7625": 3.03762,
+ "7630": 3.02026,
+ "7635": 3.01854,
+ "7640": 3.01712,
+ "7645": 3.04845,
+ "7650": 3.04439,
+ "7655": 3.08975,
+ "7660": 2.96325,
+ "7665": 3.02969,
+ "7670": 3.01245,
+ "7675": 3.00305,
+ "7680": 2.9998,
+ "7685": 3.07016,
+ "7690": 3.01368,
+ "7695": 2.99671,
+ "7700": 3.05056,
+ "7705": 3.01282,
+ "7710": 3.05828,
+ "7715": 2.99725,
+ "7720": 3.08276,
+ "7725": 2.98411,
+ "7730": 2.99881,
+ "7735": 3.02714,
+ "7740": 3.00979,
+ "7745": 3.00319,
+ "7750": 3.01,
+ "7755": 3.01954,
+ "7760": 2.98571,
+ "7765": 3.00397,
+ "7770": 3.02732,
+ "7775": 2.98978,
+ "7780": 2.97862,
+ "7785": 3.01472,
+ "7790": 2.99842,
+ "7795": 3.02413,
+ "7800": 3.00827,
+ "7805": 3.01176,
+ "7810": 3.03082,
+ "7815": 3.00244,
+ "7820": 3.0019,
+ "7825": 3.03231,
+ "7830": 3.03143,
+ "7835": 2.96605,
+ "7840": 3.04336,
+ "7845": 2.97937,
+ "7850": 2.93977,
+ "7855": 2.98529,
+ "7860": 2.98344,
+ "7865": 3.02956,
+ "7870": 2.9691,
+ "7875": 2.98838,
+ "7880": 3.00349,
+ "7885": 2.9968,
+ "7890": 3.03811,
+ "7895": 3.02857,
+ "7900": 3.03097,
+ "7905": 2.99876,
+ "7910": 3.0088,
+ "7915": 3.02527,
+ "7920": 3.01259,
+ "7925": 2.99646,
+ "7930": 3.02866,
+ "7935": 2.98913,
+ "7940": 3.03573,
+ "7945": 3.0501,
+ "7950": 2.96381,
+ "7955": 2.98711,
+ "7960": 2.96943,
+ "7965": 2.94566,
+ "7970": 2.9655,
+ "7975": 2.99544,
+ "7980": 3.00887,
+ "7985": 2.97698,
+ "7990": 2.97506,
+ "7995": 2.96124,
+ "8000": 3.02098,
+ "8005": 2.9801,
+ "8010": 2.97649,
+ "8015": 2.96466,
+ "8020": 2.97779,
+ "8025": 2.95601,
+ "8030": 2.97562,
+ "8035": 2.97196,
+ "8040": 2.95703,
+ "8045": 3.01604,
+ "8050": 3.01297,
+ "8055": 2.97453,
+ "8060": 3.00494,
+ "8065": 2.98862,
+ "8070": 2.96753,
+ "8075": 2.97734,
+ "8080": 3.01019,
+ "8085": 2.96754,
+ "8090": 2.98003,
+ "8095": 3.00216,
+ "8100": 2.95105,
+ "8105": 2.99247,
+ "8110": 2.98157,
+ "8115": 2.95999,
+ "8120": 2.97249,
+ "8125": 2.99946,
+ "8130": 2.97003,
+ "8135": 2.98766,
+ "8140": 2.96736,
+ "8145": 2.95939,
+ "8150": 2.98009,
+ "8155": 2.95146,
+ "8160": 2.997,
+ "8165": 2.9913,
+ "8170": 2.95554,
+ "8175": 2.95554,
+ "8180": 3.01376,
+ "8185": 2.98624,
+ "8190": 3.02032,
+ "8195": 2.99613,
+ "8200": 2.96412,
+ "8205": 2.97566,
+ "8210": 2.9781,
+ "8215": 2.99017,
+ "8220": 2.971,
+ "8225": 2.96329,
+ "8230": 2.99505,
+ "8235": 3.00306,
+ "8240": 2.97419,
+ "8245": 2.9738,
+ "8250": 3.00958,
+ "8255": 2.96716,
+ "8260": 2.97331,
+ "8265": 2.95555,
+ "8270": 2.97514,
+ "8275": 2.96718,
+ "8280": 2.94092,
+ "8285": 2.97838,
+ "8290": 2.96734,
+ "8295": 2.95246,
+ "8300": 2.96504,
+ "8305": 2.97504,
+ "8310": 2.97996,
+ "8315": 2.95732,
+ "8320": 2.97776,
+ "8325": 2.929,
+ "8330": 2.89908,
+ "8335": 2.96646,
+ "8340": 2.99201,
+ "8345": 2.94463,
+ "8350": 2.95886,
+ "8355": 2.98631,
+ "8360": 2.96643,
+ "8365": 2.98326,
+ "8370": 2.99094,
+ "8375": 2.93854,
+ "8380": 2.94099,
+ "8385": 2.97126,
+ "8390": 2.9453,
+ "8395": 2.97523,
+ "8400": 2.95927,
+ "8405": 2.97418,
+ "8410": 3.03057,
+ "8415": 2.93533,
+ "8420": 2.91801,
+ "8425": 2.97564,
+ "8430": 2.97808,
+ "8435": 2.93124,
+ "8440": 3.01239,
+ "8445": 2.99121,
+ "8450": 2.96616,
+ "8455": 2.97106,
+ "8460": 2.97975,
+ "8465": 2.92562,
+ "8470": 2.94697,
+ "8475": 2.99054,
+ "8480": 2.93097,
+ "8485": 2.93977,
+ "8490": 2.948,
+ "8495": 2.93336,
+ "8500": 2.96904,
+ "8505": 2.92233,
+ "8510": 3.00332,
+ "8515": 2.94052,
+ "8520": 2.95755,
+ "8525": 2.88522,
+ "8530": 2.95834,
+ "8535": 2.97603,
+ "8540": 2.93194,
+ "8545": 2.95741,
+ "8550": 2.92307,
+ "8555": 2.98961,
+ "8560": 2.99424,
+ "8565": 2.9514,
+ "8570": 2.94707,
+ "8575": 2.93509,
+ "8580": 2.9669,
+ "8585": 2.976,
+ "8590": 2.97659,
+ "8595": 2.97731,
+ "8600": 2.94787,
+ "8605": 2.94545,
+ "8610": 2.95479,
+ "8615": 2.96032,
+ "8620": 2.92346,
+ "8625": 2.94581,
+ "8630": 2.95087,
+ "8635": 2.94522,
+ "8640": 2.92578,
+ "8645": 2.98133,
+ "8650": 2.92232,
+ "8655": 2.96592,
+ "8660": 2.97073,
+ "8665": 2.95471,
+ "8670": 2.96657,
+ "8675": 2.93996,
+ "8680": 2.93576,
+ "8685": 2.94815,
+ "8690": 2.96442,
+ "8695": 2.97067,
+ "8700": 2.94799,
+ "8705": 2.91745,
+ "8710": 2.96979,
+ "8715": 2.91522,
+ "8720": 2.97447,
+ "8725": 2.94876,
+ "8730": 2.94256,
+ "8735": 2.97158,
+ "8740": 2.92587,
+ "8745": 2.96492,
+ "8750": 2.96628,
+ "8755": 2.93098,
+ "8760": 2.94924,
+ "8765": 2.91354,
+ "8770": 2.96822,
+ "8775": 2.94219,
+ "8780": 2.92859,
+ "8785": 2.94726,
+ "8790": 2.92803,
+ "8795": 2.96489,
+ "8800": 2.92662,
+ "8805": 2.90115,
+ "8810": 2.93145,
+ "8815": 2.93283,
+ "8820": 2.90387,
+ "8825": 2.92443,
+ "8830": 2.91245,
+ "8835": 2.89847,
+ "8840": 2.91518,
+ "8845": 2.92785,
+ "8850": 2.95695,
+ "8855": 2.92839,
+ "8860": 2.98878,
+ "8865": 2.93356,
+ "8870": 2.90865,
+ "8875": 2.92162,
+ "8880": 2.9295,
+ "8885": 2.9207,
+ "8890": 2.9404,
+ "8895": 2.92179,
+ "8900": 2.94464,
+ "8905": 2.93594,
+ "8910": 2.91993,
+ "8915": 2.90336,
+ "8920": 2.91127,
+ "8925": 2.97428,
+ "8930": 2.96209,
+ "8935": 2.97189,
+ "8940": 2.94882,
+ "8945": 2.94789,
+ "8950": 2.9328,
+ "8955": 2.91679,
+ "8960": 2.89858,
+ "8965": 2.92721,
+ "8970": 2.94082,
+ "8975": 2.90449,
+ "8980": 2.89797,
+ "8985": 2.92102,
+ "8990": 2.9662,
+ "8995": 2.9373,
+ "9000": 2.89467,
+ "9005": 2.9399,
+ "9010": 2.97901,
+ "9015": 2.90311,
+ "9020": 2.90423,
+ "9025": 2.92238,
+ "9030": 2.94518,
+ "9035": 2.85736,
+ "9040": 2.93491,
+ "9045": 2.92378,
+ "9050": 2.96087,
+ "9055": 2.88884,
+ "9060": 2.95609,
+ "9065": 2.98682,
+ "9070": 2.92665,
+ "9075": 2.94254,
+ "9080": 2.93301,
+ "9085": 2.9439,
+ "9090": 2.93648,
+ "9095": 2.89849,
+ "9100": 2.90017,
+ "9105": 2.89,
+ "9110": 2.93211,
+ "9115": 2.93981,
+ "9120": 2.97397,
+ "9125": 2.91648,
+ "9130": 2.92277,
+ "9135": 2.94086,
+ "9140": 2.94695,
+ "9145": 2.89447,
+ "9150": 2.92217,
+ "9155": 2.93169,
+ "9160": 2.93686,
+ "9165": 2.92557,
+ "9170": 2.9498,
+ "9175": 2.88716,
+ "9180": 2.93307,
+ "9185": 2.8947,
+ "9190": 2.94894,
+ "9195": 2.91222,
+ "9200": 2.93251,
+ "9205": 2.88702,
+ "9210": 2.93304,
+ "9215": 2.87965,
+ "9220": 2.90288,
+ "9225": 2.93315,
+ "9230": 2.86569,
+ "9235": 2.87842,
+ "9240": 2.89576,
+ "9245": 2.88279,
+ "9250": 2.88136,
+ "9255": 2.91192,
+ "9260": 2.87817,
+ "9265": 2.92175,
+ "9270": 2.89613,
+ "9275": 2.91313,
+ "9280": 2.91939,
+ "9285": 2.91903,
+ "9290": 2.93047,
+ "9295": 2.92844,
+ "9300": 2.87877,
+ "9305": 2.90909,
+ "9310": 2.89871,
+ "9315": 2.86609,
+ "9320": 2.86065,
+ "9325": 2.90436,
+ "9330": 2.95511,
+ "9335": 2.87572,
+ "9340": 2.93845,
+ "9345": 2.94693,
+ "9350": 2.9134,
+ "9355": 2.87737,
+ "9360": 2.89674,
+ "9365": 2.8823,
+ "9370": 2.93386,
+ "9375": 2.91236,
+ "9380": 2.86428,
+ "9385": 2.91358,
+ "9390": 2.92324,
+ "9395": 2.92024,
+ "9400": 2.89599,
+ "9405": 2.89197,
+ "9410": 2.9185,
+ "9415": 2.91775,
+ "9420": 2.89381,
+ "9425": 2.89983,
+ "9430": 2.87833,
+ "9435": 2.90417,
+ "9440": 2.89629,
+ "9445": 2.88366,
+ "9450": 2.89069,
+ "9455": 2.88969,
+ "9460": 2.94442,
+ "9465": 2.94721,
+ "9470": 2.88553,
+ "9475": 2.94033,
+ "9480": 2.88982,
+ "9485": 2.87815,
+ "9490": 2.89723,
+ "9495": 2.9225,
+ "9500": 2.89514,
+ "9505": 2.86794,
+ "9510": 2.894,
+ "9515": 2.90369,
+ "9520": 2.91102,
+ "9525": 2.89095,
+ "9530": 2.88696,
+ "9535": 2.91216
+ }
+ },
+ "num-zeros": {
+ "start_step": 1,
+ "end_step": 9535,
+ "step_interval": 5,
+ "values": {
+ "1": 1021640256.0,
+ "5": 1024063424.0,
+ "10": 1014250560.0,
+ "15": 1024077504.0,
+ "20": 1022486144.0,
+ "25": 1041373312.0,
+ "30": 1028112896.0,
+ "35": 1035625088.0,
+ "40": 1026328384.0,
+ "45": 1022350080.0,
+ "50": 1030098560.0,
+ "55": 1028966144.0,
+ "60": 1036320640.0,
+ "65": 1034679168.0,
+ "70": 1029374848.0,
+ "75": 1028745088.0,
+ "80": 1047575040.0,
+ "85": 1029448064.0,
+ "90": 1020467392.0,
+ "95": 1028310016.0,
+ "100": 1040961344.0,
+ "105": 1039436544.0,
+ "110": 1026879104.0,
+ "115": 1052312832.0,
+ "120": 1018863104.0,
+ "125": 1045372160.0,
+ "130": 1034330368.0,
+ "135": 1016615680.0,
+ "140": 1038582272.0,
+ "145": 1020688640.0,
+ "150": 1039788096.0,
+ "155": 1032796928.0,
+ "160": 1020952640.0,
+ "165": 1032424512.0,
+ "170": 1017396096.0,
+ "175": 1033427072.0,
+ "180": 1036119424.0,
+ "185": 1030573760.0,
+ "190": 1035673984.0,
+ "195": 1034555520.0,
+ "200": 1040973824.0,
+ "205": 1048500352.0,
+ "210": 1054481024.0,
+ "215": 1025159552.0,
+ "220": 1044962496.0,
+ "225": 1038076416.0,
+ "230": 1026222720.0,
+ "235": 1051134976.0,
+ "240": 1029276416.0,
+ "245": 1031397824.0,
+ "250": 1027879616.0,
+ "255": 1016929792.0,
+ "260": 1045008896.0,
+ "265": 1021330688.0,
+ "270": 1030964864.0,
+ "275": 1036911744.0,
+ "280": 1031743488.0,
+ "285": 1015014016.0,
+ "290": 1018756352.0,
+ "295": 1017237504.0,
+ "300": 1034761152.0,
+ "305": 1032166144.0,
+ "310": 1035583104.0,
+ "315": 1012734272.0,
+ "320": 1008275072.0,
+ "325": 1042741760.0,
+ "330": 1042870656.0,
+ "335": 1033508480.0,
+ "340": 1014464512.0,
+ "345": 1042618880.0,
+ "350": 1031852736.0,
+ "355": 1050844800.0,
+ "360": 1030258432.0,
+ "365": 1034595648.0,
+ "370": 1019436032.0,
+ "375": 1022144832.0,
+ "380": 1021326592.0,
+ "385": 1025589504.0,
+ "390": 1023195072.0,
+ "395": 1019653952.0,
+ "400": 1033520512.0,
+ "405": 1023880192.0,
+ "410": 1017910016.0,
+ "415": 1024288000.0,
+ "420": 1020624256.0,
+ "425": 1025854848.0,
+ "430": 1033854336.0,
+ "435": 1028182400.0,
+ "440": 1022090752.0,
+ "445": 1036768256.0,
+ "450": 1024997376.0,
+ "455": 1013852096.0,
+ "460": 1022093824.0,
+ "465": 1041431552.0,
+ "470": 1029038016.0,
+ "475": 1010065792.0,
+ "480": 1047607616.0,
+ "485": 1029724928.0,
+ "490": 1044668160.0,
+ "495": 1025229952.0,
+ "500": 1037464960.0,
+ "505": 1032181376.0,
+ "510": 1042853056.0,
+ "515": 1026159744.0,
+ "520": 1013409792.0,
+ "525": 1035147520.0,
+ "530": 1016375552.0,
+ "535": 1040113024.0,
+ "540": 1035052352.0,
+ "545": 1032113664.0,
+ "550": 1018673408.0,
+ "555": 1008638656.0,
+ "560": 1011927680.0,
+ "565": 1041824320.0,
+ "570": 1034942208.0,
+ "575": 1010199040.0,
+ "580": 1032210496.0,
+ "585": 1041262144.0,
+ "590": 1038867968.0,
+ "595": 1035743104.0,
+ "600": 1023772736.0,
+ "605": 1032294272.0,
+ "610": 1037748672.0,
+ "615": 1005974784.0,
+ "620": 1040407424.0,
+ "625": 1045209216.0,
+ "630": 1034414464.0,
+ "635": 1028523008.0,
+ "640": 1022644928.0,
+ "645": 1035876032.0,
+ "650": 1009255680.0,
+ "655": 997757696.0,
+ "660": 1029710464.0,
+ "665": 1025532608.0,
+ "670": 1048812288.0,
+ "675": 1025202688.0,
+ "680": 1019340032.0,
+ "685": 1027832512.0,
+ "690": 1029230080.0,
+ "695": 1040024576.0,
+ "700": 1042031680.0,
+ "705": 1034382976.0,
+ "710": 1020441792.0,
+ "715": 1031472128.0,
+ "720": 1040274560.0,
+ "725": 1023279936.0,
+ "730": 1022792704.0,
+ "735": 1025085696.0,
+ "740": 1038382656.0,
+ "745": 1045205504.0,
+ "750": 1013180928.0,
+ "755": 1031644032.0,
+ "760": 1032783552.0,
+ "765": 1027135936.0,
+ "770": 1023967232.0,
+ "775": 1025895168.0,
+ "780": 1038166464.0,
+ "785": 1025486400.0,
+ "790": 1040810624.0,
+ "795": 1032531200.0,
+ "800": 1039592768.0,
+ "805": 1024318016.0,
+ "810": 1034725632.0,
+ "815": 1036000448.0,
+ "820": 1035671552.0,
+ "825": 1051375360.0,
+ "830": 1035406784.0,
+ "835": 1022547776.0,
+ "840": 1036875648.0,
+ "845": 1025700352.0,
+ "850": 1048529920.0,
+ "855": 1014986432.0,
+ "860": 1033098624.0,
+ "865": 1031543040.0,
+ "870": 1040902912.0,
+ "875": 1023938304.0,
+ "880": 1028395904.0,
+ "885": 1054406656.0,
+ "890": 1019537152.0,
+ "895": 1045189824.0,
+ "900": 1031772928.0,
+ "905": 1020970688.0,
+ "910": 1031386112.0,
+ "915": 1032926912.0,
+ "920": 1038459392.0,
+ "925": 1026754560.0,
+ "930": 1025378752.0,
+ "935": 1031126464.0,
+ "940": 1057933568.0,
+ "945": 1029823104.0,
+ "950": 1014412480.0,
+ "955": 1032173696.0,
+ "960": 1026152064.0,
+ "965": 1062678976.0,
+ "970": 1030096128.0,
+ "975": 1036903680.0,
+ "980": 1027049216.0,
+ "985": 1030676736.0,
+ "990": 1020676864.0,
+ "995": 1042301760.0,
+ "1000": 1036831616.0,
+ "1005": 1050206080.0,
+ "1010": 1023801984.0,
+ "1015": 1020539008.0,
+ "1020": 1042587392.0,
+ "1025": 1037943808.0,
+ "1030": 1049210048.0,
+ "1035": 1012483456.0,
+ "1040": 1023092032.0,
+ "1045": 1039520768.0,
+ "1050": 1026825728.0,
+ "1055": 1034861184.0,
+ "1060": 1046128704.0,
+ "1065": 1036804096.0,
+ "1070": 1019994880.0,
+ "1075": 1025341696.0,
+ "1080": 1014979200.0,
+ "1085": 1030007744.0,
+ "1090": 1029062016.0,
+ "1095": 1020309888.0,
+ "1100": 1039835008.0,
+ "1105": 1048600064.0,
+ "1110": 1020704448.0,
+ "1115": 1024782720.0,
+ "1120": 1061896576.0,
+ "1125": 1043311616.0,
+ "1130": 1031219456.0,
+ "1135": 1041360512.0,
+ "1140": 1021486272.0,
+ "1145": 1051696128.0,
+ "1150": 1035590400.0,
+ "1155": 1029590528.0,
+ "1160": 1042564800.0,
+ "1165": 1026810496.0,
+ "1170": 1018001408.0,
+ "1175": 1033684032.0,
+ "1180": 1035633536.0,
+ "1185": 1023928960.0,
+ "1190": 1033160320.0,
+ "1195": 1024228608.0,
+ "1200": 1039116544.0,
+ "1205": 1031740800.0,
+ "1210": 1053250560.0,
+ "1215": 1024617600.0,
+ "1220": 1009041280.0,
+ "1225": 1036679680.0,
+ "1230": 1041257984.0,
+ "1235": 1053974912.0,
+ "1240": 1030356224.0,
+ "1245": 1017684864.0,
+ "1250": 1022772992.0,
+ "1255": 1033439104.0,
+ "1260": 1034284736.0,
+ "1265": 1034003840.0,
+ "1270": 1037323264.0,
+ "1275": 1029345792.0,
+ "1280": 1046489856.0,
+ "1285": 1028285120.0,
+ "1290": 1036578176.0,
+ "1295": 1032421696.0,
+ "1300": 1033065728.0,
+ "1305": 1030027008.0,
+ "1310": 1051262976.0,
+ "1315": 1035373184.0,
+ "1320": 1028263936.0,
+ "1325": 1049972736.0,
+ "1330": 1030133376.0,
+ "1335": 1031164800.0,
+ "1340": 1012758912.0,
+ "1345": 1044639232.0,
+ "1350": 1034957312.0,
+ "1355": 1033623744.0,
+ "1360": 1036683392.0,
+ "1365": 1038588672.0,
+ "1370": 1039851904.0,
+ "1375": 1034117632.0,
+ "1380": 1022886656.0,
+ "1385": 1018084096.0,
+ "1390": 1049054400.0,
+ "1395": 1034868352.0,
+ "1400": 1034998144.0,
+ "1405": 1034131456.0,
+ "1410": 1036368256.0,
+ "1415": 1043577600.0,
+ "1420": 1026111104.0,
+ "1425": 1033320320.0,
+ "1430": 1012808128.0,
+ "1435": 1038394880.0,
+ "1440": 1020971904.0,
+ "1445": 1032459904.0,
+ "1450": 1014039296.0,
+ "1455": 1011673984.0,
+ "1460": 1043275904.0,
+ "1465": 1014361600.0,
+ "1470": 1020655360.0,
+ "1475": 1030231296.0,
+ "1480": 1029370496.0,
+ "1485": 1022997696.0,
+ "1490": 1026783360.0,
+ "1495": 1021815744.0,
+ "1500": 1027177088.0,
+ "1505": 1034882880.0,
+ "1510": 1014397120.0,
+ "1515": 1042136832.0,
+ "1520": 1025792640.0,
+ "1525": 1036335872.0,
+ "1530": 1039948992.0,
+ "1535": 1047640192.0,
+ "1540": 1043539840.0,
+ "1545": 1034043520.0,
+ "1550": 1016108736.0,
+ "1555": 1015573504.0,
+ "1560": 1055021824.0,
+ "1565": 1015593728.0,
+ "1570": 1018243840.0,
+ "1575": 1032515456.0,
+ "1580": 1012984768.0,
+ "1585": 1025327680.0,
+ "1590": 1034127360.0,
+ "1595": 1057393664.0,
+ "1600": 1026867584.0,
+ "1605": 1019994624.0,
+ "1610": 1031268736.0,
+ "1615": 1035274880.0,
+ "1620": 1018016000.0,
+ "1625": 1028272512.0,
+ "1630": 1027205376.0,
+ "1635": 1023799040.0,
+ "1640": 1034120832.0,
+ "1645": 1021814528.0,
+ "1650": 1015262080.0,
+ "1655": 1018280064.0,
+ "1660": 1047982976.0,
+ "1665": 1027060352.0,
+ "1670": 1048219904.0,
+ "1675": 1021102912.0,
+ "1680": 1043288320.0,
+ "1685": 1052719360.0,
+ "1690": 1026724032.0,
+ "1695": 1040385280.0,
+ "1700": 1018036352.0,
+ "1705": 1020480640.0,
+ "1710": 1021024448.0,
+ "1715": 1026932992.0,
+ "1720": 1028350208.0,
+ "1725": 1034363136.0,
+ "1730": 1013692352.0,
+ "1735": 1018429696.0,
+ "1740": 1057257024.0,
+ "1745": 1029261952.0,
+ "1750": 1024357888.0,
+ "1755": 1029970112.0,
+ "1760": 1022192512.0,
+ "1765": 1040477056.0,
+ "1770": 1029669760.0,
+ "1775": 1046196864.0,
+ "1780": 1021955712.0,
+ "1785": 1035109376.0,
+ "1790": 1028263808.0,
+ "1795": 1031023616.0,
+ "1800": 1028300480.0,
+ "1805": 1025669248.0,
+ "1810": 1021556096.0,
+ "1815": 1033440256.0,
+ "1820": 1034885888.0,
+ "1825": 1020208448.0,
+ "1830": 1013885632.0,
+ "1835": 1031382272.0,
+ "1840": 1040391040.0,
+ "1845": 1034828800.0,
+ "1850": 1014480064.0,
+ "1855": 1019418816.0,
+ "1860": 1019569536.0,
+ "1865": 1035942400.0,
+ "1870": 1026242368.0,
+ "1875": 1031525248.0,
+ "1880": 1011590784.0,
+ "1885": 1041065536.0,
+ "1890": 1035000704.0,
+ "1895": 1028959488.0,
+ "1900": 1033997568.0,
+ "1905": 1027123776.0,
+ "1910": 1029217792.0,
+ "1915": 1030492864.0,
+ "1920": 1042920384.0,
+ "1925": 1038419392.0,
+ "1930": 1019304512.0,
+ "1935": 1032535936.0,
+ "1940": 1027806336.0,
+ "1945": 1034205056.0,
+ "1950": 1006036224.0,
+ "1955": 1032577600.0,
+ "1960": 1015720256.0,
+ "1965": 1029088512.0,
+ "1970": 1021554176.0,
+ "1975": 1034048000.0,
+ "1980": 1029366912.0,
+ "1985": 1027784960.0,
+ "1990": 1020947840.0,
+ "1995": 1010422912.0,
+ "2000": 1039617152.0,
+ "2005": 1001486208.0,
+ "2010": 1020422912.0,
+ "2015": 1032034048.0,
+ "2020": 1036298624.0,
+ "2025": 1037172352.0,
+ "2030": 1029770752.0,
+ "2035": 1040333312.0,
+ "2040": 1030112768.0,
+ "2045": 1032700800.0,
+ "2050": 1008016064.0,
+ "2055": 1045723840.0,
+ "2060": 1028142400.0,
+ "2065": 1038799488.0,
+ "2070": 1045645184.0,
+ "2075": 1035237952.0,
+ "2080": 1022882304.0,
+ "2085": 1024815424.0,
+ "2090": 1034363392.0,
+ "2095": 1005220672.0,
+ "2100": 1034644096.0,
+ "2105": 1035581312.0,
+ "2110": 1030685952.0,
+ "2115": 1029798528.0,
+ "2120": 1018846080.0,
+ "2125": 1021863168.0,
+ "2130": 1026638080.0,
+ "2135": 1053279488.0,
+ "2140": 1017060608.0,
+ "2145": 1019635072.0,
+ "2150": 1037130752.0,
+ "2155": 1033302784.0,
+ "2160": 1049035776.0,
+ "2165": 1039682816.0,
+ "2170": 1020308096.0,
+ "2175": 1027338752.0,
+ "2180": 1041703168.0,
+ "2185": 1028895360.0,
+ "2190": 1029309888.0,
+ "2195": 1028944768.0,
+ "2200": 1039639680.0,
+ "2205": 1036972288.0,
+ "2210": 1031740544.0,
+ "2215": 1021404480.0,
+ "2220": 1020910848.0,
+ "2225": 1033403072.0,
+ "2230": 1014201856.0,
+ "2235": 1029395968.0,
+ "2240": 1029885184.0,
+ "2245": 1026005824.0,
+ "2250": 1046268800.0,
+ "2255": 1032951936.0,
+ "2260": 1047494592.0,
+ "2265": 1023721088.0,
+ "2270": 1022566144.0,
+ "2275": 1028537600.0,
+ "2280": 1034973568.0,
+ "2285": 1031819968.0,
+ "2290": 1038650048.0,
+ "2295": 1028816000.0,
+ "2300": 1034450496.0,
+ "2305": 1032314496.0,
+ "2310": 1013586496.0,
+ "2315": 1048182656.0,
+ "2320": 1035210368.0,
+ "2325": 1046966016.0,
+ "2330": 1014696192.0,
+ "2335": 1027382272.0,
+ "2340": 1036736512.0,
+ "2345": 1020186944.0,
+ "2350": 1031017728.0,
+ "2355": 1037474240.0,
+ "2360": 1032608128.0,
+ "2365": 1028041856.0,
+ "2370": 1021004224.0,
+ "2375": 1022912000.0,
+ "2380": 1048556224.0,
+ "2385": 1044140736.0,
+ "2390": 1021986816.0,
+ "2395": 1020595584.0,
+ "2400": 1026930816.0,
+ "2405": 1038387200.0,
+ "2410": 1045395200.0,
+ "2415": 1048454656.0,
+ "2420": 1032227712.0,
+ "2425": 1029562176.0,
+ "2430": 1030386176.0,
+ "2435": 1029217856.0,
+ "2440": 1029168000.0,
+ "2445": 1033132160.0,
+ "2450": 1038557824.0,
+ "2455": 1034721536.0,
+ "2460": 1039984192.0,
+ "2465": 1032500992.0,
+ "2470": 1024143872.0,
+ "2475": 1016539520.0,
+ "2480": 1023613248.0,
+ "2485": 1021030592.0,
+ "2490": 1035920448.0,
+ "2495": 1032967360.0,
+ "2500": 1028107008.0,
+ "2505": 1015385600.0,
+ "2510": 1030967104.0,
+ "2515": 1025700096.0,
+ "2520": 1033326208.0,
+ "2525": 1029692800.0,
+ "2530": 1023986560.0,
+ "2535": 1071069696.0,
+ "2540": 1024537984.0,
+ "2545": 1033798784.0,
+ "2550": 1029448064.0,
+ "2555": 1029183488.0,
+ "2560": 1018115072.0,
+ "2565": 1031598528.0,
+ "2570": 1022847232.0,
+ "2575": 1026503104.0,
+ "2580": 1038622592.0,
+ "2585": 1025899456.0,
+ "2590": 1026100800.0,
+ "2595": 1046623104.0,
+ "2600": 1031103360.0,
+ "2605": 1001910656.0,
+ "2610": 1028423360.0,
+ "2615": 1025564544.0,
+ "2620": 1038651392.0,
+ "2625": 1026996352.0,
+ "2630": 1036831424.0,
+ "2635": 1021198400.0,
+ "2640": 1021865856.0,
+ "2645": 1039153408.0,
+ "2650": 1025943488.0,
+ "2655": 1013255808.0,
+ "2660": 1032645248.0,
+ "2665": 1035218048.0,
+ "2670": 1036437632.0,
+ "2675": 1039296064.0,
+ "2680": 1041661696.0,
+ "2685": 1034565504.0,
+ "2690": 1058871168.0,
+ "2695": 1019879552.0,
+ "2700": 1062626816.0,
+ "2705": 1035376320.0,
+ "2710": 1019542400.0,
+ "2715": 1031885824.0,
+ "2720": 1016403200.0,
+ "2725": 1040594688.0,
+ "2730": 1019586688.0,
+ "2735": 1030889856.0,
+ "2740": 1029290752.0,
+ "2745": 1040687744.0,
+ "2750": 1023880448.0,
+ "2755": 1011865664.0,
+ "2760": 1027684864.0,
+ "2765": 1030882240.0,
+ "2770": 1033119872.0,
+ "2775": 1026332352.0,
+ "2780": 1033684224.0,
+ "2785": 1024589888.0,
+ "2790": 1033734272.0,
+ "2795": 1045949184.0,
+ "2800": 1040286016.0,
+ "2805": 1019944192.0,
+ "2810": 1031449600.0,
+ "2815": 1030932736.0,
+ "2820": 1037855616.0,
+ "2825": 1041684096.0,
+ "2830": 1030459904.0,
+ "2835": 1013508352.0,
+ "2840": 1031449600.0,
+ "2845": 1030129920.0,
+ "2850": 1026617600.0,
+ "2855": 1024705280.0,
+ "2860": 1031700096.0,
+ "2865": 1027428800.0,
+ "2870": 1026690048.0,
+ "2875": 1012777024.0,
+ "2880": 1038301568.0,
+ "2885": 1017901184.0,
+ "2890": 1044200064.0,
+ "2895": 1036459136.0,
+ "2900": 1030652928.0,
+ "2905": 1035957376.0,
+ "2910": 1038718272.0,
+ "2915": 1039385408.0,
+ "2920": 1034781248.0,
+ "2925": 1043267840.0,
+ "2930": 1038229696.0,
+ "2935": 1021222144.0,
+ "2940": 1042307456.0,
+ "2945": 1045232384.0,
+ "2950": 1047525952.0,
+ "2955": 1034172928.0,
+ "2960": 1020891904.0,
+ "2965": 1027307840.0,
+ "2970": 1038796288.0,
+ "2975": 1034007296.0,
+ "2980": 1049590400.0,
+ "2985": 1034846016.0,
+ "2990": 1026008576.0,
+ "2995": 1034919296.0,
+ "3000": 1039017856.0,
+ "3005": 1038158848.0,
+ "3010": 1010907712.0,
+ "3015": 1044976064.0,
+ "3020": 1034050688.0,
+ "3025": 1037763840.0,
+ "3030": 1027722816.0,
+ "3035": 1041821056.0,
+ "3040": 1035311872.0,
+ "3045": 1027255296.0,
+ "3050": 1029708032.0,
+ "3055": 1028029568.0,
+ "3060": 1049976960.0,
+ "3065": 1024067200.0,
+ "3070": 1011545728.0,
+ "3075": 1042846272.0,
+ "3080": 1036094912.0,
+ "3085": 1030387456.0,
+ "3090": 1035262976.0,
+ "3095": 1013803008.0,
+ "3100": 1030144896.0,
+ "3105": 1017609088.0,
+ "3110": 1033370816.0,
+ "3115": 1023737728.0,
+ "3120": 1024877504.0,
+ "3125": 1046537216.0,
+ "3130": 1024676160.0,
+ "3135": 1025722496.0,
+ "3140": 1043778176.0,
+ "3145": 1044372672.0,
+ "3150": 1016483328.0,
+ "3155": 1042487936.0,
+ "3160": 1026834688.0,
+ "3165": 1031199360.0,
+ "3170": 1024332800.0,
+ "3175": 1024368640.0,
+ "3180": 1018204288.0,
+ "3185": 1034352512.0,
+ "3190": 1019221888.0,
+ "3195": 1028425408.0,
+ "3200": 1036080640.0,
+ "3205": 1016076160.0,
+ "3210": 1034109312.0,
+ "3215": 1031349312.0,
+ "3220": 1040833664.0,
+ "3225": 1022835008.0,
+ "3230": 1033255744.0,
+ "3235": 1019975488.0,
+ "3240": 1038131840.0,
+ "3245": 1031643136.0,
+ "3250": 1022390656.0,
+ "3255": 1032876672.0,
+ "3260": 1037751616.0,
+ "3265": 1021622656.0,
+ "3270": 1031242880.0,
+ "3275": 1038461184.0,
+ "3280": 1023236992.0,
+ "3285": 1031615424.0,
+ "3290": 1045247616.0,
+ "3295": 1043177536.0,
+ "3300": 1035084224.0,
+ "3305": 1042662400.0,
+ "3310": 1058092096.0,
+ "3315": 1024282880.0,
+ "3320": 1046015296.0,
+ "3325": 1023179008.0,
+ "3330": 1048037248.0,
+ "3335": 1036690560.0,
+ "3340": 1042123392.0,
+ "3345": 1030897920.0,
+ "3350": 1020621696.0,
+ "3355": 1025960576.0,
+ "3360": 1030305344.0,
+ "3365": 1031171520.0,
+ "3370": 1036454144.0,
+ "3375": 1023472384.0,
+ "3380": 1032383744.0,
+ "3385": 1038081536.0,
+ "3390": 1052811072.0,
+ "3395": 1012090496.0,
+ "3400": 1019209600.0,
+ "3405": 1021780224.0,
+ "3410": 1028433728.0,
+ "3415": 1058222400.0,
+ "3420": 1033492480.0,
+ "3425": 1029580352.0,
+ "3430": 1021150976.0,
+ "3435": 1034991872.0,
+ "3440": 1017961600.0,
+ "3445": 1025537280.0,
+ "3450": 1032254336.0,
+ "3455": 1036261312.0,
+ "3460": 1052071808.0,
+ "3465": 1027114240.0,
+ "3470": 1043729536.0,
+ "3475": 1033265792.0,
+ "3480": 1026619776.0,
+ "3485": 1029215232.0,
+ "3490": 1041041408.0,
+ "3495": 1019252224.0,
+ "3500": 1032059904.0,
+ "3505": 1025753728.0,
+ "3510": 1044367616.0,
+ "3515": 1013817280.0,
+ "3520": 1021846400.0,
+ "3525": 1032175552.0,
+ "3530": 1029789056.0,
+ "3535": 1034568704.0,
+ "3540": 1017731456.0,
+ "3545": 1035658880.0,
+ "3550": 1024535296.0,
+ "3555": 1035866112.0,
+ "3560": 1029737600.0,
+ "3565": 1028900160.0,
+ "3570": 1046029888.0,
+ "3575": 1039186304.0,
+ "3580": 1010838336.0,
+ "3585": 1031737728.0,
+ "3590": 1041450688.0,
+ "3595": 1037636800.0,
+ "3600": 1032763584.0,
+ "3605": 1045822272.0,
+ "3610": 1039235200.0,
+ "3615": 1036870144.0,
+ "3620": 1026929664.0,
+ "3625": 1033931136.0,
+ "3630": 1017582464.0,
+ "3635": 1026629056.0,
+ "3640": 1039529088.0,
+ "3645": 1022655872.0,
+ "3650": 1036842624.0,
+ "3655": 1023990144.0,
+ "3660": 1014987456.0,
+ "3665": 1026118784.0,
+ "3670": 1041672448.0,
+ "3675": 1033250304.0,
+ "3680": 1015353984.0,
+ "3685": 1029122304.0,
+ "3690": 1026204416.0,
+ "3695": 1043800832.0,
+ "3700": 1028613504.0,
+ "3705": 1049485312.0,
+ "3710": 1027180672.0,
+ "3715": 1016134912.0,
+ "3720": 1040818560.0,
+ "3725": 1032763776.0,
+ "3730": 1030920960.0,
+ "3735": 1019008640.0,
+ "3740": 1023825600.0,
+ "3745": 1046289152.0,
+ "3750": 1034462336.0,
+ "3755": 1032090048.0,
+ "3760": 1019366912.0,
+ "3765": 1031916736.0,
+ "3770": 1026677120.0,
+ "3775": 1035708288.0,
+ "3780": 1030671104.0,
+ "3785": 1027208128.0,
+ "3790": 1019584064.0,
+ "3795": 1030306048.0,
+ "3800": 1035614976.0,
+ "3805": 1035423360.0,
+ "3810": 1033294144.0,
+ "3815": 1033988608.0,
+ "3820": 1041105792.0,
+ "3825": 1024534976.0,
+ "3830": 1037630528.0,
+ "3835": 1040347968.0,
+ "3840": 1023445888.0,
+ "3845": 1048466688.0,
+ "3850": 1052489280.0,
+ "3855": 1028907264.0,
+ "3860": 1019532672.0,
+ "3865": 1035487744.0,
+ "3870": 1028491712.0,
+ "3875": 1041164800.0,
+ "3880": 1048854912.0,
+ "3885": 1027725248.0,
+ "3890": 1027487616.0,
+ "3895": 1034190592.0,
+ "3900": 1027645312.0,
+ "3905": 1027976128.0,
+ "3910": 1041572480.0,
+ "3915": 1043995392.0,
+ "3920": 1041063424.0,
+ "3925": 1030836160.0,
+ "3930": 1027072896.0,
+ "3935": 1033782016.0,
+ "3940": 1042275712.0,
+ "3945": 1036248064.0,
+ "3950": 1021430976.0,
+ "3955": 1036304128.0,
+ "3960": 1024184192.0,
+ "3965": 1027065856.0,
+ "3970": 1015984640.0,
+ "3975": 1041421632.0,
+ "3980": 1032455488.0,
+ "3985": 1037680640.0,
+ "3990": 1038684992.0,
+ "3995": 1023654528.0,
+ "4000": 1054410240.0,
+ "4005": 1029983424.0,
+ "4010": 1025138112.0,
+ "4015": 1030978560.0,
+ "4020": 1018472448.0,
+ "4025": 1027124352.0,
+ "4030": 1010306816.0,
+ "4035": 1038641088.0,
+ "4040": 1022256640.0,
+ "4045": 1025038208.0,
+ "4050": 1032348800.0,
+ "4055": 1022420864.0,
+ "4060": 1024520768.0,
+ "4065": 1032871168.0,
+ "4070": 1027791232.0,
+ "4075": 1025596928.0,
+ "4080": 1029366656.0,
+ "4085": 1020823552.0,
+ "4090": 1033322496.0,
+ "4095": 1024142656.0,
+ "4100": 1040948864.0,
+ "4105": 1027266496.0,
+ "4110": 1038791424.0,
+ "4115": 1023497088.0,
+ "4120": 1038943168.0,
+ "4125": 1048274176.0,
+ "4130": 1021490752.0,
+ "4135": 1034570880.0,
+ "4140": 1034613824.0,
+ "4145": 1044447232.0,
+ "4150": 1000353664.0,
+ "4155": 1028363392.0,
+ "4160": 1024242624.0,
+ "4165": 1033688704.0,
+ "4170": 1018888000.0,
+ "4175": 1026492608.0,
+ "4180": 1045409024.0,
+ "4185": 1033631616.0,
+ "4190": 1029574592.0,
+ "4195": 1038777984.0,
+ "4200": 1025102336.0,
+ "4205": 1019074816.0,
+ "4210": 1029560704.0,
+ "4215": 1032269184.0,
+ "4220": 1026242048.0,
+ "4225": 1031925888.0,
+ "4230": 1030269824.0,
+ "4235": 1027603328.0,
+ "4240": 1031480832.0,
+ "4245": 1028765056.0,
+ "4250": 1026987008.0,
+ "4255": 1021240064.0,
+ "4260": 1042082432.0,
+ "4265": 1025411200.0,
+ "4270": 1030169984.0,
+ "4275": 1012472448.0,
+ "4280": 1044505600.0,
+ "4285": 1019898304.0,
+ "4290": 1033058560.0,
+ "4295": 1033596032.0,
+ "4300": 1031638912.0,
+ "4305": 1023847936.0,
+ "4310": 1021568512.0,
+ "4315": 1047221504.0,
+ "4320": 1026520576.0,
+ "4325": 1005865600.0,
+ "4330": 1037666688.0,
+ "4335": 1022006464.0,
+ "4340": 1029009920.0,
+ "4345": 1033474496.0,
+ "4350": 1036886144.0,
+ "4355": 1026808832.0,
+ "4360": 1022938240.0,
+ "4365": 1028779648.0,
+ "4370": 1029624704.0,
+ "4375": 1042196864.0,
+ "4380": 1016100096.0,
+ "4385": 1045551296.0,
+ "4390": 1026270848.0,
+ "4395": 1029796416.0,
+ "4400": 1047365760.0,
+ "4405": 1029297344.0,
+ "4410": 1033424256.0,
+ "4415": 1028298304.0,
+ "4420": 1028148928.0,
+ "4425": 1033575552.0,
+ "4430": 1031374592.0,
+ "4435": 1028571136.0,
+ "4440": 1033123328.0,
+ "4445": 1028293504.0,
+ "4450": 1052210944.0,
+ "4455": 1026286080.0,
+ "4460": 1034885888.0,
+ "4465": 1031725696.0,
+ "4470": 1035446528.0,
+ "4475": 1036971712.0,
+ "4480": 1025117824.0,
+ "4485": 1034104960.0,
+ "4490": 1024630912.0,
+ "4495": 1047974912.0,
+ "4500": 1024707840.0,
+ "4505": 1038850048.0,
+ "4510": 1043723776.0,
+ "4515": 1044276736.0,
+ "4520": 1036872320.0,
+ "4525": 1058073536.0,
+ "4530": 1030973568.0,
+ "4535": 1032592256.0,
+ "4540": 1036428160.0,
+ "4545": 1025726400.0,
+ "4550": 1021749312.0,
+ "4555": 1037546112.0,
+ "4560": 1020099200.0,
+ "4565": 1036055296.0,
+ "4570": 1020501120.0,
+ "4575": 1050412608.0,
+ "4580": 1010437888.0,
+ "4585": 1022960768.0,
+ "4590": 1039710272.0,
+ "4595": 1023274880.0,
+ "4600": 1042477824.0,
+ "4605": 1039746688.0,
+ "4610": 1046104192.0,
+ "4615": 1017999744.0,
+ "4620": 1044734592.0,
+ "4625": 1030479104.0,
+ "4630": 1027260800.0,
+ "4635": 1026995200.0,
+ "4640": 1034901248.0,
+ "4645": 1036420352.0,
+ "4650": 1033711488.0,
+ "4655": 1035461056.0,
+ "4660": 1035324800.0,
+ "4665": 1020265664.0,
+ "4670": 1020057344.0,
+ "4675": 1054848768.0,
+ "4680": 1024895872.0,
+ "4685": 1027820160.0,
+ "4690": 1034449664.0,
+ "4695": 1039151744.0,
+ "4700": 1038865024.0,
+ "4705": 1027655808.0,
+ "4710": 1020522560.0,
+ "4715": 1031825536.0,
+ "4720": 1030300416.0,
+ "4725": 1030298368.0,
+ "4730": 1044096704.0,
+ "4735": 1046133376.0,
+ "4740": 1036178112.0,
+ "4745": 1039043840.0,
+ "4750": 1031790528.0,
+ "4755": 1047723392.0,
+ "4760": 1026178176.0,
+ "4765": 1034695040.0,
+ "4770": 1036521856.0,
+ "4775": 1029375168.0,
+ "4780": 1028543488.0,
+ "4785": 1028414976.0,
+ "4790": 1019620224.0,
+ "4795": 1033060160.0,
+ "4800": 1051866880.0,
+ "4805": 1015414400.0,
+ "4810": 1029454336.0,
+ "4815": 1009572096.0,
+ "4820": 1041051200.0,
+ "4825": 1026708608.0,
+ "4830": 1020450816.0,
+ "4835": 1051307840.0,
+ "4840": 1019456512.0,
+ "4845": 1032315008.0,
+ "4850": 1036794496.0,
+ "4855": 1031052736.0,
+ "4860": 1033131776.0,
+ "4865": 1032064384.0,
+ "4870": 1049832576.0,
+ "4875": 1025110528.0,
+ "4880": 1048476160.0,
+ "4885": 1016853056.0,
+ "4890": 1037317312.0,
+ "4895": 1024323136.0,
+ "4900": 1043374208.0,
+ "4905": 1033397120.0,
+ "4910": 1032830272.0,
+ "4915": 1016889856.0,
+ "4920": 1022294784.0,
+ "4925": 1034965888.0,
+ "4930": 1034630016.0,
+ "4935": 1025885312.0,
+ "4940": 1048398272.0,
+ "4945": 1025248576.0,
+ "4950": 1024208768.0,
+ "4955": 1007485952.0,
+ "4960": 1040213824.0,
+ "4965": 1018775296.0,
+ "4970": 1014274688.0,
+ "4975": 1038025472.0,
+ "4980": 1020917888.0,
+ "4985": 1029045888.0,
+ "4990": 1028394816.0,
+ "4995": 1032020480.0,
+ "5000": 1039791104.0,
+ "5005": 1024351552.0,
+ "5010": 1029147968.0,
+ "5015": 1021807296.0,
+ "5020": 1023506944.0,
+ "5025": 1037603456.0,
+ "5030": 1041947136.0,
+ "5035": 1047130304.0,
+ "5040": 1060956096.0,
+ "5045": 1032108544.0,
+ "5050": 1029534336.0,
+ "5055": 1024552192.0,
+ "5060": 1035282304.0,
+ "5065": 1021205504.0,
+ "5070": 1035756288.0,
+ "5075": 1015771264.0,
+ "5080": 1027040064.0,
+ "5085": 1021792192.0,
+ "5090": 1034973568.0,
+ "5095": 1015499712.0,
+ "5100": 1032257600.0,
+ "5105": 1017981568.0,
+ "5110": 1019586304.0,
+ "5115": 1036063936.0,
+ "5120": 1032695040.0,
+ "5125": 1019076992.0,
+ "5130": 1033404672.0,
+ "5135": 1041203072.0,
+ "5140": 1026258752.0,
+ "5145": 1033705856.0,
+ "5150": 1022043520.0,
+ "5155": 1032265664.0,
+ "5160": 1039625984.0,
+ "5165": 1031576448.0,
+ "5170": 1035555328.0,
+ "5175": 1026116224.0,
+ "5180": 1030316032.0,
+ "5185": 1024495680.0,
+ "5190": 1019492608.0,
+ "5195": 1035626496.0,
+ "5200": 1016905344.0,
+ "5205": 1013435648.0,
+ "5210": 1049395456.0,
+ "5215": 1030833280.0,
+ "5220": 1025276800.0,
+ "5225": 1035239936.0,
+ "5230": 1025930624.0,
+ "5235": 1025120000.0,
+ "5240": 1046308224.0,
+ "5245": 1022740608.0,
+ "5250": 1027062336.0,
+ "5255": 1023887360.0,
+ "5260": 1033821440.0,
+ "5265": 1045733696.0,
+ "5270": 1052500480.0,
+ "5275": 1033018112.0,
+ "5280": 1030073920.0,
+ "5285": 1025212608.0,
+ "5290": 1026575616.0,
+ "5295": 1032653440.0,
+ "5300": 1024367872.0,
+ "5305": 1029634368.0,
+ "5310": 1033197312.0,
+ "5315": 1032988992.0,
+ "5320": 1019521664.0,
+ "5325": 1022718336.0,
+ "5330": 1021335168.0,
+ "5335": 1039275776.0,
+ "5340": 1037219648.0,
+ "5345": 1039188096.0,
+ "5350": 1023701888.0,
+ "5355": 1029935872.0,
+ "5360": 1047046080.0,
+ "5365": 1037426432.0,
+ "5370": 1024381568.0,
+ "5375": 1042070656.0,
+ "5380": 1020368384.0,
+ "5385": 1021765696.0,
+ "5390": 1035133184.0,
+ "5395": 1049653568.0,
+ "5400": 1026015744.0,
+ "5405": 1036453120.0,
+ "5410": 1027635776.0,
+ "5415": 1042285824.0,
+ "5420": 1039941888.0,
+ "5425": 1028381184.0,
+ "5430": 1043799808.0,
+ "5435": 1032653312.0,
+ "5440": 1033384448.0,
+ "5445": 1034144640.0,
+ "5450": 1025299328.0,
+ "5455": 1034079424.0,
+ "5460": 1026812416.0,
+ "5465": 1027399552.0,
+ "5470": 1028969216.0,
+ "5475": 1037233920.0,
+ "5480": 1023830272.0,
+ "5485": 1019186752.0,
+ "5490": 1030891520.0,
+ "5495": 1029399424.0,
+ "5500": 1032681216.0,
+ "5505": 1018275200.0,
+ "5510": 1023987648.0,
+ "5515": 1025156032.0,
+ "5520": 1039527296.0,
+ "5525": 1018024576.0,
+ "5530": 1037663936.0,
+ "5535": 1031599232.0,
+ "5540": 1027564544.0,
+ "5545": 1033212160.0,
+ "5550": 1032115968.0,
+ "5555": 1044802304.0,
+ "5560": 1028511232.0,
+ "5565": 1029686016.0,
+ "5570": 1042027776.0,
+ "5575": 1025379392.0,
+ "5580": 1023716736.0,
+ "5585": 1044093696.0,
+ "5590": 1041319936.0,
+ "5595": 1031549824.0,
+ "5600": 1023400320.0,
+ "5605": 1040115456.0,
+ "5610": 1034087552.0,
+ "5615": 1021042816.0,
+ "5620": 1031004800.0,
+ "5625": 1030188544.0,
+ "5630": 1023502080.0,
+ "5635": 1026684096.0,
+ "5640": 1034589120.0,
+ "5645": 1018655744.0,
+ "5650": 1052378752.0,
+ "5655": 1048933504.0,
+ "5660": 1050077696.0,
+ "5665": 1033958144.0,
+ "5670": 1033750016.0,
+ "5675": 1025392640.0,
+ "5680": 1039378304.0,
+ "5685": 1033056576.0,
+ "5690": 1031464576.0,
+ "5695": 1021946368.0,
+ "5700": 1038065664.0,
+ "5705": 1043684736.0,
+ "5710": 1057231616.0,
+ "5715": 1014462848.0,
+ "5720": 1021258816.0,
+ "5725": 1041822272.0,
+ "5730": 1039454912.0,
+ "5735": 1025128576.0,
+ "5740": 1026045440.0,
+ "5745": 1036990208.0,
+ "5750": 1044552256.0,
+ "5755": 1011860416.0,
+ "5760": 1028389568.0,
+ "5765": 1028245504.0,
+ "5770": 1021530368.0,
+ "5775": 1051210240.0,
+ "5780": 1034984512.0,
+ "5785": 1037513920.0,
+ "5790": 1016957184.0,
+ "5795": 1027873536.0,
+ "5800": 1029780736.0,
+ "5805": 1050694912.0,
+ "5810": 1018478336.0,
+ "5815": 1036123520.0,
+ "5820": 1048408704.0,
+ "5825": 1030977920.0,
+ "5830": 1031572096.0,
+ "5835": 1034045440.0,
+ "5840": 1039843776.0,
+ "5845": 1021746048.0,
+ "5850": 1029807744.0,
+ "5855": 1038789376.0,
+ "5860": 1031436288.0,
+ "5865": 1026397568.0,
+ "5870": 1029861824.0,
+ "5875": 1032841856.0,
+ "5880": 1032675968.0,
+ "5885": 1024576128.0,
+ "5890": 1026798976.0,
+ "5895": 1015796160.0,
+ "5900": 1049707008.0,
+ "5905": 1025653248.0,
+ "5910": 1019150720.0,
+ "5915": 1042739136.0,
+ "5920": 1028047232.0,
+ "5925": 1034016448.0,
+ "5930": 1030963328.0,
+ "5935": 1038102784.0,
+ "5940": 1019172864.0,
+ "5945": 1025130112.0,
+ "5950": 1035530240.0,
+ "5955": 1050437184.0,
+ "5960": 1024548736.0,
+ "5965": 1029923712.0,
+ "5970": 1016427776.0,
+ "5975": 1036682752.0,
+ "5980": 1024118464.0,
+ "5985": 1035386624.0,
+ "5990": 1010550784.0,
+ "5995": 1047019200.0,
+ "6000": 1021245568.0,
+ "6005": 1040460416.0,
+ "6010": 1025358720.0,
+ "6015": 1050179072.0,
+ "6020": 1039514496.0,
+ "6025": 1030254592.0,
+ "6030": 1025931968.0,
+ "6035": 1021745408.0,
+ "6040": 1034117056.0,
+ "6045": 1028282112.0,
+ "6050": 1020112320.0,
+ "6055": 1040397056.0,
+ "6060": 1026347008.0,
+ "6065": 1022198400.0,
+ "6070": 1040668416.0,
+ "6075": 1046037440.0,
+ "6080": 1038583168.0,
+ "6085": 1041485568.0,
+ "6090": 1037205888.0,
+ "6095": 1036282880.0,
+ "6100": 1030454720.0,
+ "6105": 1019216640.0,
+ "6110": 1035357824.0,
+ "6115": 1019452544.0,
+ "6120": 1032188800.0,
+ "6125": 1020922624.0,
+ "6130": 1012013952.0,
+ "6135": 1038733824.0,
+ "6140": 1041736896.0,
+ "6145": 1041917056.0,
+ "6150": 1018958208.0,
+ "6155": 1024649344.0,
+ "6160": 1047972160.0,
+ "6165": 1050408832.0,
+ "6170": 1032505344.0,
+ "6175": 1045793664.0,
+ "6180": 1040067072.0,
+ "6185": 1029710464.0,
+ "6190": 1023293760.0,
+ "6195": 1050897728.0,
+ "6200": 1035035776.0,
+ "6205": 1036275584.0,
+ "6210": 1039772736.0,
+ "6215": 1033200256.0,
+ "6220": 1026162432.0,
+ "6225": 1036741120.0,
+ "6230": 1025144192.0,
+ "6235": 1019352832.0,
+ "6240": 1057104384.0,
+ "6245": 1018413952.0,
+ "6250": 1035337344.0,
+ "6255": 1025380992.0,
+ "6260": 1034863744.0,
+ "6265": 1027703424.0,
+ "6270": 1042116480.0,
+ "6275": 1037659008.0,
+ "6280": 1018270208.0,
+ "6285": 1032642304.0,
+ "6290": 1038598592.0,
+ "6295": 1031803456.0,
+ "6300": 1034635200.0,
+ "6305": 1011066624.0,
+ "6310": 1039458624.0,
+ "6315": 1030054272.0,
+ "6320": 1030534208.0,
+ "6325": 1038642496.0,
+ "6330": 1033908800.0,
+ "6335": 1032297856.0,
+ "6340": 1033544448.0,
+ "6345": 1031036416.0,
+ "6350": 1037451264.0,
+ "6355": 1028075968.0,
+ "6360": 1043313408.0,
+ "6365": 1025223808.0,
+ "6370": 1033939200.0,
+ "6375": 1036038720.0,
+ "6380": 1029108096.0,
+ "6385": 1025395072.0,
+ "6390": 1025517952.0,
+ "6395": 1048611584.0,
+ "6400": 1040734976.0,
+ "6405": 1024247936.0,
+ "6410": 1017489280.0,
+ "6415": 1042827072.0,
+ "6420": 1025202432.0,
+ "6425": 1027164928.0,
+ "6430": 1040568256.0,
+ "6435": 1022908800.0,
+ "6440": 1047994624.0,
+ "6445": 1036089088.0,
+ "6450": 1048532224.0,
+ "6455": 1037272320.0,
+ "6460": 1036750912.0,
+ "6465": 1033652032.0,
+ "6470": 1018135232.0,
+ "6475": 1034691648.0,
+ "6480": 1028994048.0,
+ "6485": 1033258880.0,
+ "6490": 1035638656.0,
+ "6495": 1024470016.0,
+ "6500": 1020572096.0,
+ "6505": 1059327104.0,
+ "6510": 1020472576.0,
+ "6515": 1018688064.0,
+ "6520": 1051470592.0,
+ "6525": 1035544512.0,
+ "6530": 1027897216.0,
+ "6535": 1022722240.0,
+ "6540": 1023273984.0,
+ "6545": 1033173120.0,
+ "6550": 1029488512.0,
+ "6555": 1029575296.0,
+ "6560": 1056438784.0,
+ "6565": 1054295040.0,
+ "6570": 1032319040.0,
+ "6575": 1041208320.0,
+ "6580": 1028134400.0,
+ "6585": 1036504832.0,
+ "6590": 1042456192.0,
+ "6595": 1038568832.0,
+ "6600": 1031388096.0,
+ "6605": 1045715456.0,
+ "6610": 1034713472.0,
+ "6615": 1015576448.0,
+ "6620": 1039115136.0,
+ "6625": 1054654208.0,
+ "6630": 1043092928.0,
+ "6635": 1032226304.0,
+ "6640": 1016738496.0,
+ "6645": 1016178816.0,
+ "6650": 1034692672.0,
+ "6655": 1031753472.0,
+ "6660": 1041401920.0,
+ "6665": 1024657984.0,
+ "6670": 1023820032.0,
+ "6675": 1038306176.0,
+ "6680": 1025624064.0,
+ "6685": 1045394048.0,
+ "6690": 1046390720.0,
+ "6695": 1027754368.0,
+ "6700": 1033473920.0,
+ "6705": 1038857152.0,
+ "6710": 1047485888.0,
+ "6715": 1043229440.0,
+ "6720": 1022995456.0,
+ "6725": 1018910144.0,
+ "6730": 1027525504.0,
+ "6735": 1016937856.0,
+ "6740": 1027238016.0,
+ "6745": 1030263680.0,
+ "6750": 1006373760.0,
+ "6755": 1034765056.0,
+ "6760": 1040735296.0,
+ "6765": 1023827008.0,
+ "6770": 1036441344.0,
+ "6775": 1019627712.0,
+ "6780": 1043723904.0,
+ "6785": 1037409280.0,
+ "6790": 1029403072.0,
+ "6795": 1026349440.0,
+ "6800": 1036628224.0,
+ "6805": 1024579712.0,
+ "6810": 1042340544.0,
+ "6815": 1035274112.0,
+ "6820": 1022594880.0,
+ "6825": 1034793344.0,
+ "6830": 1029862400.0,
+ "6835": 1041609600.0,
+ "6840": 1042283776.0,
+ "6845": 1018954624.0,
+ "6850": 1032171136.0,
+ "6855": 1034434752.0,
+ "6860": 1042054848.0,
+ "6865": 1021813568.0,
+ "6870": 1037015424.0,
+ "6875": 1030379968.0,
+ "6880": 1029360768.0,
+ "6885": 1030435968.0,
+ "6890": 1039890432.0,
+ "6895": 1027267712.0,
+ "6900": 1035174016.0,
+ "6905": 1043975424.0,
+ "6910": 1019763072.0,
+ "6915": 1017476608.0,
+ "6920": 1017184256.0,
+ "6925": 1030650688.0,
+ "6930": 1036672384.0,
+ "6935": 1042835712.0,
+ "6940": 1040313216.0,
+ "6945": 1044196992.0,
+ "6950": 1040513472.0,
+ "6955": 1036112704.0,
+ "6960": 1036436224.0,
+ "6965": 1019161024.0,
+ "6970": 1034729088.0,
+ "6975": 1019134464.0,
+ "6980": 1028436160.0,
+ "6985": 1023240128.0,
+ "6990": 1026994688.0,
+ "6995": 1027547520.0,
+ "7000": 1058819840.0,
+ "7005": 1013737856.0,
+ "7010": 1028959488.0,
+ "7015": 1037288768.0,
+ "7020": 1011880576.0,
+ "7025": 1017313280.0,
+ "7030": 1028301440.0,
+ "7035": 1035955392.0,
+ "7040": 1042966016.0,
+ "7045": 1028185856.0,
+ "7050": 1017979584.0,
+ "7055": 1035088000.0,
+ "7060": 1051802624.0,
+ "7065": 1007664640.0,
+ "7070": 1035819008.0,
+ "7075": 1031039552.0,
+ "7080": 1026143296.0,
+ "7085": 1044906432.0,
+ "7090": 1046261760.0,
+ "7095": 1043760512.0,
+ "7100": 1035089024.0,
+ "7105": 1049143296.0,
+ "7110": 1010962944.0,
+ "7115": 1033869504.0,
+ "7120": 1031267456.0,
+ "7125": 1037496832.0,
+ "7130": 1024881856.0,
+ "7135": 1031991808.0,
+ "7140": 1019090176.0,
+ "7145": 1033081088.0,
+ "7150": 1037554112.0,
+ "7155": 1015729728.0,
+ "7160": 1024724608.0,
+ "7165": 1030895808.0,
+ "7170": 1037367808.0,
+ "7175": 1028816896.0,
+ "7180": 1037633280.0,
+ "7185": 1016174080.0,
+ "7190": 1019808128.0,
+ "7195": 1040915392.0,
+ "7200": 1041375360.0,
+ "7205": 1026538240.0,
+ "7210": 1022638720.0,
+ "7215": 1041890560.0,
+ "7220": 1017742720.0,
+ "7225": 1027296640.0,
+ "7230": 1030200448.0,
+ "7235": 1035726848.0,
+ "7240": 1037854848.0,
+ "7245": 1023971008.0,
+ "7250": 1044708096.0,
+ "7255": 1031900480.0,
+ "7260": 1030128256.0,
+ "7265": 1036887104.0,
+ "7270": 1050097152.0,
+ "7275": 1029225216.0,
+ "7280": 1020231808.0,
+ "7285": 1029842048.0,
+ "7290": 1017219328.0,
+ "7295": 1029139584.0,
+ "7300": 1031533824.0,
+ "7305": 1027298176.0,
+ "7310": 1029089664.0,
+ "7315": 1022782272.0,
+ "7320": 1036458176.0,
+ "7325": 1036851840.0,
+ "7330": 1021706496.0,
+ "7335": 1030715904.0,
+ "7340": 1039382976.0,
+ "7345": 1040177664.0,
+ "7350": 1034973568.0,
+ "7355": 1033656320.0,
+ "7360": 1031254912.0,
+ "7365": 1048742016.0,
+ "7370": 1027298304.0,
+ "7375": 1041854848.0,
+ "7380": 1016725760.0,
+ "7385": 1017578368.0,
+ "7390": 1017234944.0,
+ "7395": 1046793600.0,
+ "7400": 1048441216.0,
+ "7405": 1013394304.0,
+ "7410": 1017386368.0,
+ "7415": 1017815360.0,
+ "7420": 1028043008.0,
+ "7425": 1012840576.0,
+ "7430": 1034042368.0,
+ "7435": 1032530432.0,
+ "7440": 1002692928.0,
+ "7445": 1034451200.0,
+ "7450": 1039304832.0,
+ "7455": 1019027008.0,
+ "7460": 1014740928.0,
+ "7465": 1027204736.0,
+ "7470": 1030422784.0,
+ "7475": 1033792064.0,
+ "7480": 1043317376.0,
+ "7485": 1038215168.0,
+ "7490": 1049000960.0,
+ "7495": 1028982720.0,
+ "7500": 1027426816.0,
+ "7505": 1028695936.0,
+ "7510": 1048886528.0,
+ "7515": 1035648704.0,
+ "7520": 1017198848.0,
+ "7525": 1036572736.0,
+ "7530": 1029261952.0,
+ "7535": 1027190144.0,
+ "7540": 1028338048.0,
+ "7545": 1025986304.0,
+ "7550": 1023025856.0,
+ "7555": 1033025344.0,
+ "7560": 1031404672.0,
+ "7565": 1022710528.0,
+ "7570": 1037591552.0,
+ "7575": 1022603136.0,
+ "7580": 1018123584.0,
+ "7585": 1033054208.0,
+ "7590": 1010993280.0,
+ "7595": 1018260352.0,
+ "7600": 1049904448.0,
+ "7605": 1037361216.0,
+ "7610": 1040415744.0,
+ "7615": 1035247488.0,
+ "7620": 1024230912.0,
+ "7625": 1020317184.0,
+ "7630": 1034939584.0,
+ "7635": 1043224192.0,
+ "7640": 1033491520.0,
+ "7645": 1034444608.0,
+ "7650": 1039804800.0,
+ "7655": 1031240576.0,
+ "7660": 1056628096.0,
+ "7665": 1031076096.0,
+ "7670": 1033685120.0,
+ "7675": 1030681600.0,
+ "7680": 1035398720.0,
+ "7685": 1018661760.0,
+ "7690": 1031921024.0,
+ "7695": 1025858880.0,
+ "7700": 1017715200.0,
+ "7705": 1036531200.0,
+ "7710": 1029893248.0,
+ "7715": 1053230656.0,
+ "7720": 1019514240.0,
+ "7725": 1042193216.0,
+ "7730": 1035620992.0,
+ "7735": 1020726144.0,
+ "7740": 1045576128.0,
+ "7745": 1026932992.0,
+ "7750": 1048550208.0,
+ "7755": 1022539264.0,
+ "7760": 1049532032.0,
+ "7765": 1029370176.0,
+ "7770": 1018375296.0,
+ "7775": 1021364672.0,
+ "7780": 1039770624.0,
+ "7785": 1039914112.0,
+ "7790": 1030516992.0,
+ "7795": 1039353728.0,
+ "7800": 1028187904.0,
+ "7805": 1027635776.0,
+ "7810": 1020970368.0,
+ "7815": 1035878400.0,
+ "7820": 1017666240.0,
+ "7825": 1018067392.0,
+ "7830": 1035104128.0,
+ "7835": 1044507648.0,
+ "7840": 1027836224.0,
+ "7845": 1032101504.0,
+ "7850": 1034609408.0,
+ "7855": 1025464832.0,
+ "7860": 1059051648.0,
+ "7865": 1016626240.0,
+ "7870": 1033729408.0,
+ "7875": 1044185600.0,
+ "7880": 1029084352.0,
+ "7885": 1040308288.0,
+ "7890": 1029556480.0,
+ "7895": 1032947008.0,
+ "7900": 1021409216.0,
+ "7905": 1020955904.0,
+ "7910": 1008993856.0,
+ "7915": 1023120768.0,
+ "7920": 1023070976.0,
+ "7925": 1030094080.0,
+ "7930": 1020712704.0,
+ "7935": 1019443776.0,
+ "7940": 1017809152.0,
+ "7945": 1014447552.0,
+ "7950": 1026303616.0,
+ "7955": 1034518272.0,
+ "7960": 1056026304.0,
+ "7965": 1031047872.0,
+ "7970": 1030417152.0,
+ "7975": 1022189888.0,
+ "7980": 1034474624.0,
+ "7985": 1047305024.0,
+ "7990": 1032066176.0,
+ "7995": 1044264704.0,
+ "8000": 1028876672.0,
+ "8005": 1028045440.0,
+ "8010": 1050665408.0,
+ "8015": 1019758976.0,
+ "8020": 1043297408.0,
+ "8025": 1039018560.0,
+ "8030": 1030868800.0,
+ "8035": 1045304192.0,
+ "8040": 1026310784.0,
+ "8045": 1024970368.0,
+ "8050": 1018405632.0,
+ "8055": 1033736960.0,
+ "8060": 1012986816.0,
+ "8065": 1022016640.0,
+ "8070": 1034776064.0,
+ "8075": 1042759616.0,
+ "8080": 1027758784.0,
+ "8085": 1037205376.0,
+ "8090": 1007008256.0,
+ "8095": 1030374528.0,
+ "8100": 1030726016.0,
+ "8105": 1027794944.0,
+ "8110": 1031557248.0,
+ "8115": 1037685248.0,
+ "8120": 1037692992.0,
+ "8125": 1031097472.0,
+ "8130": 1028627072.0,
+ "8135": 1029680256.0,
+ "8140": 1049904256.0,
+ "8145": 1043463552.0,
+ "8150": 1040087424.0,
+ "8155": 1046780288.0,
+ "8160": 1010199040.0,
+ "8165": 1031657728.0,
+ "8170": 1024483264.0,
+ "8175": 1035019648.0,
+ "8180": 1024460544.0,
+ "8185": 1021960448.0,
+ "8190": 1037125504.0,
+ "8195": 1022368384.0,
+ "8200": 1035635968.0,
+ "8205": 1026482496.0,
+ "8210": 1023888000.0,
+ "8215": 1014276416.0,
+ "8220": 1026756224.0,
+ "8225": 1028540160.0,
+ "8230": 1027163072.0,
+ "8235": 1037914048.0,
+ "8240": 1025909376.0,
+ "8245": 1024676608.0,
+ "8250": 1041635840.0,
+ "8255": 1031908224.0,
+ "8260": 1032424512.0,
+ "8265": 1023164800.0,
+ "8270": 1040172544.0,
+ "8275": 1038050688.0,
+ "8280": 1041849216.0,
+ "8285": 1038804352.0,
+ "8290": 1024074880.0,
+ "8295": 1028403648.0,
+ "8300": 1039341440.0,
+ "8305": 1012104192.0,
+ "8310": 1021882048.0,
+ "8315": 1027307200.0,
+ "8320": 1021636992.0,
+ "8325": 1048572160.0,
+ "8330": 1041039616.0,
+ "8335": 1037964928.0,
+ "8340": 1033019136.0,
+ "8345": 1043864192.0,
+ "8350": 1037713792.0,
+ "8355": 1029686400.0,
+ "8360": 1040667776.0,
+ "8365": 1027450304.0,
+ "8370": 1037742848.0,
+ "8375": 1041986944.0,
+ "8380": 1037628416.0,
+ "8385": 1023436160.0,
+ "8390": 1026068224.0,
+ "8395": 1028913408.0,
+ "8400": 1046530560.0,
+ "8405": 1040179456.0,
+ "8410": 1034252672.0,
+ "8415": 1040258688.0,
+ "8420": 1054730752.0,
+ "8425": 1031514880.0,
+ "8430": 1030295680.0,
+ "8435": 1045707200.0,
+ "8440": 1026310784.0,
+ "8445": 1029027392.0,
+ "8450": 1034201920.0,
+ "8455": 1031794688.0,
+ "8460": 1016828032.0,
+ "8465": 1035163648.0,
+ "8470": 1035185152.0,
+ "8475": 1024712960.0,
+ "8480": 1035901184.0,
+ "8485": 1028948480.0,
+ "8490": 1023079168.0,
+ "8495": 1037393280.0,
+ "8500": 1025960064.0,
+ "8505": 1042724992.0,
+ "8510": 1028167936.0,
+ "8515": 1038101056.0,
+ "8520": 1023107328.0,
+ "8525": 1037987328.0,
+ "8530": 1027572800.0,
+ "8535": 1041656128.0,
+ "8540": 1033880960.0,
+ "8545": 1015116160.0,
+ "8550": 1040188160.0,
+ "8555": 1016340672.0,
+ "8560": 1019330048.0,
+ "8565": 1021410112.0,
+ "8570": 1032032320.0,
+ "8575": 1031880128.0,
+ "8580": 1016011264.0,
+ "8585": 1030017408.0,
+ "8590": 1031637248.0,
+ "8595": 1017776128.0,
+ "8600": 1002393216.0,
+ "8605": 1030238336.0,
+ "8610": 1017532288.0,
+ "8615": 1023989248.0,
+ "8620": 1047205696.0,
+ "8625": 1034231552.0,
+ "8630": 1030921280.0,
+ "8635": 1051992512.0,
+ "8640": 1041134208.0,
+ "8645": 1024870720.0,
+ "8650": 1025595392.0,
+ "8655": 1036904832.0,
+ "8660": 1031171200.0,
+ "8665": 1032904640.0,
+ "8670": 1037400576.0,
+ "8675": 1029157248.0,
+ "8680": 1031264704.0,
+ "8685": 1041197568.0,
+ "8690": 1035035392.0,
+ "8695": 1008508416.0,
+ "8700": 1027459072.0,
+ "8705": 1051504896.0,
+ "8710": 1041678016.0,
+ "8715": 1034152256.0,
+ "8720": 1017596544.0,
+ "8725": 1025187456.0,
+ "8730": 1036610816.0,
+ "8735": 1014829568.0,
+ "8740": 1036081536.0,
+ "8745": 1021252416.0,
+ "8750": 1027866496.0,
+ "8755": 1020742272.0,
+ "8760": 1036899712.0,
+ "8765": 1058672448.0,
+ "8770": 1020462464.0,
+ "8775": 1031773056.0,
+ "8780": 1030892544.0,
+ "8785": 1032117504.0,
+ "8790": 1041034112.0,
+ "8795": 1019523968.0,
+ "8800": 1038245632.0,
+ "8805": 1035106752.0,
+ "8810": 1043257088.0,
+ "8815": 1026490496.0,
+ "8820": 1027666944.0,
+ "8825": 1043464064.0,
+ "8830": 1027480192.0,
+ "8835": 1038812928.0,
+ "8840": 1034490752.0,
+ "8845": 1033909760.0,
+ "8850": 1030491008.0,
+ "8855": 1042524992.0,
+ "8860": 1013002880.0,
+ "8865": 1038368128.0,
+ "8870": 1025187456.0,
+ "8875": 1012981760.0,
+ "8880": 1028376704.0,
+ "8885": 1046461056.0,
+ "8890": 1038603840.0,
+ "8895": 1037909504.0,
+ "8900": 1027294848.0,
+ "8905": 1032792064.0,
+ "8910": 1029795264.0,
+ "8915": 1030003968.0,
+ "8920": 1030339968.0,
+ "8925": 1028569984.0,
+ "8930": 1031637376.0,
+ "8935": 1022951424.0,
+ "8940": 1019847872.0,
+ "8945": 1031909248.0,
+ "8950": 1039951744.0,
+ "8955": 1041902720.0,
+ "8960": 1026878464.0,
+ "8965": 1022083968.0,
+ "8970": 1029559424.0,
+ "8975": 1038934400.0,
+ "8980": 1033860160.0,
+ "8985": 1030649472.0,
+ "8990": 1025014144.0,
+ "8995": 1013963648.0,
+ "9000": 1035286400.0,
+ "9005": 1028649280.0,
+ "9010": 1011913280.0,
+ "9015": 1038912128.0,
+ "9020": 1030153856.0,
+ "9025": 1024685056.0,
+ "9030": 1025861888.0,
+ "9035": 1054309248.0,
+ "9040": 1027293952.0,
+ "9045": 1036583040.0,
+ "9050": 1020929664.0,
+ "9055": 1043212800.0,
+ "9060": 1023159104.0,
+ "9065": 1023387520.0,
+ "9070": 1039364480.0,
+ "9075": 1026728320.0,
+ "9080": 1018873408.0,
+ "9085": 1015439104.0,
+ "9090": 1043764736.0,
+ "9095": 1014020224.0,
+ "9100": 1031975296.0,
+ "9105": 1026514304.0,
+ "9110": 1029229568.0,
+ "9115": 1024866432.0,
+ "9120": 999986240.0,
+ "9125": 1032842752.0,
+ "9130": 1038534336.0,
+ "9135": 1031037696.0,
+ "9140": 1025502208.0,
+ "9145": 1030405248.0,
+ "9150": 1029416576.0,
+ "9155": 1038268928.0,
+ "9160": 1046043904.0,
+ "9165": 1017948992.0,
+ "9170": 1040955520.0,
+ "9175": 1031287552.0,
+ "9180": 1037830656.0,
+ "9185": 1040684416.0,
+ "9190": 1028985728.0,
+ "9195": 1034312320.0,
+ "9200": 1035551872.0,
+ "9205": 1029847040.0,
+ "9210": 1026535872.0,
+ "9215": 1030520448.0,
+ "9220": 1025732224.0,
+ "9225": 1048001408.0,
+ "9230": 1041601792.0,
+ "9235": 1027775104.0,
+ "9240": 1025245760.0,
+ "9245": 1036211584.0,
+ "9250": 1041192384.0,
+ "9255": 1020063872.0,
+ "9260": 1035337984.0,
+ "9265": 1023102208.0,
+ "9270": 1038332928.0,
+ "9275": 1036053568.0,
+ "9280": 1026541504.0,
+ "9285": 1014285184.0,
+ "9290": 1018866304.0,
+ "9295": 1026915264.0,
+ "9300": 1037085888.0,
+ "9305": 1045435392.0,
+ "9310": 1033242944.0,
+ "9315": 1039043840.0,
+ "9320": 1048495488.0,
+ "9325": 1023059840.0,
+ "9330": 1031724672.0,
+ "9335": 1035673472.0,
+ "9340": 1013719296.0,
+ "9345": 1022572032.0,
+ "9350": 1026585600.0,
+ "9355": 1034807104.0,
+ "9360": 1029839552.0,
+ "9365": 1019863296.0,
+ "9370": 1006904320.0,
+ "9375": 1036232960.0,
+ "9380": 1049012736.0,
+ "9385": 1015905344.0,
+ "9390": 1029208704.0,
+ "9395": 1008931968.0,
+ "9400": 1026893568.0,
+ "9405": 1027653312.0,
+ "9410": 1040913280.0,
+ "9415": 1035128576.0,
+ "9420": 1030792640.0,
+ "9425": 1027581056.0,
+ "9430": 1032727360.0,
+ "9435": 1031796288.0,
+ "9440": 1051730048.0,
+ "9445": 1019626752.0,
+ "9450": 1044505152.0,
+ "9455": 1035773696.0,
+ "9460": 1013828224.0,
+ "9465": 1023403904.0,
+ "9470": 1023576832.0,
+ "9475": 1039164416.0,
+ "9480": 1029597056.0,
+ "9485": 1032075200.0,
+ "9490": 1020994560.0,
+ "9495": 1021375616.0,
+ "9500": 1035594304.0,
+ "9505": 1034478464.0,
+ "9510": 1014286592.0,
+ "9515": 1031309312.0,
+ "9520": 1026563904.0,
+ "9525": 1035853184.0,
+ "9530": 1031624448.0,
+ "9535": 1025926720.0
+ }
+ },
+ "mem-allocated-bytes": {
+ "start_step": 1,
+ "end_step": 9535,
+ "step_interval": 5,
+ "values": {
+ "1": 33307314176.0,
+ "5": 33307424768.0,
+ "10": 33307447296.0,
+ "15": 33307439104.0,
+ "20": 33307533312.0,
+ "25": 33307473920.0,
+ "30": 33307504640.0,
+ "35": 33307639808.0,
+ "40": 33307637760.0,
+ "45": 33307568128.0,
+ "50": 33307418624.0,
+ "55": 33307326464.0,
+ "60": 33307346944.0,
+ "65": 33307490304.0,
+ "70": 33307312128.0,
+ "75": 33307308032.0,
+ "80": 33307404288.0,
+ "85": 33307314176.0,
+ "90": 33307285504.0,
+ "95": 33307392000.0,
+ "100": 33307260928.0,
+ "105": 33307129856.0,
+ "110": 33307037696.0,
+ "115": 33306703872.0,
+ "120": 33307355136.0,
+ "125": 33306873856.0,
+ "130": 33307017216.0,
+ "135": 33307305984.0,
+ "140": 33307004928.0,
+ "145": 33307121664.0,
+ "150": 33307312128.0,
+ "155": 33307176960.0,
+ "160": 33307103232.0,
+ "165": 33307174912.0,
+ "170": 33307832320.0,
+ "175": 33307199488.0,
+ "180": 33307355136.0,
+ "185": 33307355136.0,
+ "190": 33307131904.0,
+ "195": 33307256832.0,
+ "200": 33307326464.0,
+ "205": 33307492352.0,
+ "210": 33307500544.0,
+ "215": 33307086848.0,
+ "220": 33306857472.0,
+ "225": 33306933248.0,
+ "230": 33307092992.0,
+ "235": 33307183104.0,
+ "240": 33307303936.0,
+ "245": 33307426816.0,
+ "250": 33307308032.0,
+ "255": 33307295744.0,
+ "260": 33306767360.0,
+ "265": 33307461632.0,
+ "270": 33307467776.0,
+ "275": 33307469824.0,
+ "280": 33307254784.0,
+ "285": 33307947008.0,
+ "290": 33307191296.0,
+ "295": 33308014592.0,
+ "300": 33307856896.0,
+ "305": 33308340224.0,
+ "310": 33307815936.0,
+ "315": 33307181056.0,
+ "320": 33307512832.0,
+ "325": 33307488256.0,
+ "330": 33307977728.0,
+ "335": 33307947008.0,
+ "340": 33308606464.0,
+ "345": 33308037120.0,
+ "350": 33307693056.0,
+ "355": 33308000256.0,
+ "360": 33307348992.0,
+ "365": 33307451392.0,
+ "370": 33308000256.0,
+ "375": 33307283456.0,
+ "380": 33307570176.0,
+ "385": 33307860992.0,
+ "390": 33307416576.0,
+ "395": 33307031552.0,
+ "400": 33307246592.0,
+ "405": 33307676672.0,
+ "410": 33306935296.0,
+ "415": 33307752448.0,
+ "420": 33307529216.0,
+ "425": 33307314176.0,
+ "430": 33306988544.0,
+ "435": 33307455488.0,
+ "440": 33307369472.0,
+ "445": 33307709440.0,
+ "450": 33307588608.0,
+ "455": 33306963968.0,
+ "460": 33307193344.0,
+ "465": 33306845184.0,
+ "470": 33307766784.0,
+ "475": 33306464256.0,
+ "480": 33307566080.0,
+ "485": 33307682816.0,
+ "490": 33307389952.0,
+ "495": 33307179008.0,
+ "500": 33307969536.0,
+ "505": 33307629568.0,
+ "510": 33308192768.0,
+ "515": 33307279360.0,
+ "520": 33306544128.0,
+ "525": 33307265024.0,
+ "530": 33307025408.0,
+ "535": 33307648000.0,
+ "540": 33307582464.0,
+ "545": 33307297792.0,
+ "550": 33307396096.0,
+ "555": 33307301888.0,
+ "560": 33307899904.0,
+ "565": 33307379712.0,
+ "570": 33307553792.0,
+ "575": 33307136000.0,
+ "580": 33305892864.0,
+ "585": 33306945536.0,
+ "590": 33307629568.0,
+ "595": 33307860992.0,
+ "600": 33306873856.0,
+ "605": 33307357184.0,
+ "610": 33306556416.0,
+ "615": 33306349568.0,
+ "620": 33307791360.0,
+ "625": 33306378240.0,
+ "630": 33307168768.0,
+ "635": 33306767360.0,
+ "640": 33306116096.0,
+ "645": 33308092416.0,
+ "650": 33307277312.0,
+ "655": 33307131904.0,
+ "660": 33308485632.0,
+ "665": 33307334656.0,
+ "670": 33307959296.0,
+ "675": 33307701248.0,
+ "680": 33306863616.0,
+ "685": 33306697728.0,
+ "690": 33307863040.0,
+ "695": 33307293696.0,
+ "700": 33306263552.0,
+ "705": 33306955776.0,
+ "710": 33308225536.0,
+ "715": 33307174912.0,
+ "720": 33307107328.0,
+ "725": 33307324416.0,
+ "730": 33308231680.0,
+ "735": 33307224064.0,
+ "740": 33307815936.0,
+ "745": 33307938816.0,
+ "750": 33307779072.0,
+ "755": 33308463104.0,
+ "760": 33306349568.0,
+ "765": 33308266496.0,
+ "770": 33306603520.0,
+ "775": 33307424768.0,
+ "780": 33308608512.0,
+ "785": 33307969536.0,
+ "790": 33308188672.0,
+ "795": 33307656192.0,
+ "800": 33307547648.0,
+ "805": 33307619328.0,
+ "810": 33307910144.0,
+ "815": 33307170816.0,
+ "820": 33307029504.0,
+ "825": 33307443200.0,
+ "830": 33307422720.0,
+ "835": 33307262976.0,
+ "840": 33307613184.0,
+ "845": 33307928576.0,
+ "850": 33306238976.0,
+ "855": 33307396096.0,
+ "860": 33307938816.0,
+ "865": 33307701248.0,
+ "870": 33307940864.0,
+ "875": 33307545600.0,
+ "880": 33307527168.0,
+ "885": 33307336704.0,
+ "890": 33308262400.0,
+ "895": 33307717632.0,
+ "900": 33306474496.0,
+ "905": 33307480064.0,
+ "910": 33307725824.0,
+ "915": 33308303360.0,
+ "920": 33307770880.0,
+ "925": 33307566080.0,
+ "930": 33307451392.0,
+ "935": 33307975680.0,
+ "940": 33306320896.0,
+ "945": 33306429440.0,
+ "950": 33307136000.0,
+ "955": 33307846656.0,
+ "960": 33307611136.0,
+ "965": 33307465728.0,
+ "970": 33308293120.0,
+ "975": 33307078656.0,
+ "980": 33307568128.0,
+ "985": 33307080704.0,
+ "990": 33307367424.0,
+ "995": 33306861568.0,
+ "1000": 33307889664.0,
+ "1005": 33305956352.0,
+ "1010": 33307508736.0,
+ "1015": 33306671104.0,
+ "1020": 33306669056.0,
+ "1025": 33306509312.0,
+ "1030": 33307117568.0,
+ "1035": 33308332032.0,
+ "1040": 33307353088.0,
+ "1045": 33308368896.0,
+ "1050": 33306615808.0,
+ "1055": 33306802176.0,
+ "1060": 33307103232.0,
+ "1065": 33307404288.0,
+ "1070": 33307070464.0,
+ "1075": 33308188672.0,
+ "1080": 33307011072.0,
+ "1085": 33307027456.0,
+ "1090": 33308086272.0,
+ "1095": 33307086848.0,
+ "1100": 33307287552.0,
+ "1105": 33308497920.0,
+ "1110": 33307461632.0,
+ "1115": 33307533312.0,
+ "1120": 33307777024.0,
+ "1125": 33307809792.0,
+ "1130": 33307484160.0,
+ "1135": 33308082176.0,
+ "1140": 33307029504.0,
+ "1145": 33307432960.0,
+ "1150": 33307574272.0,
+ "1155": 33307551744.0,
+ "1160": 33307561984.0,
+ "1165": 33307086848.0,
+ "1170": 33307856896.0,
+ "1175": 33306976256.0,
+ "1180": 33308237824.0,
+ "1185": 33307875328.0,
+ "1190": 33307369472.0,
+ "1195": 33308231680.0,
+ "1200": 33307197440.0,
+ "1205": 33307480064.0,
+ "1210": 33305866240.0,
+ "1215": 33308297216.0,
+ "1220": 33307451392.0,
+ "1225": 33307518976.0,
+ "1230": 33307688960.0,
+ "1235": 33307901952.0,
+ "1240": 33307394048.0,
+ "1245": 33307842560.0,
+ "1250": 33307281408.0,
+ "1255": 33306906624.0,
+ "1260": 33307301888.0,
+ "1265": 33307674624.0,
+ "1270": 33307150336.0,
+ "1275": 33307686912.0,
+ "1280": 33307430912.0,
+ "1285": 33306974208.0,
+ "1290": 33307529216.0,
+ "1295": 33307901952.0,
+ "1300": 33307002880.0,
+ "1305": 33308059648.0,
+ "1310": 33306939392.0,
+ "1315": 33307336704.0,
+ "1320": 33307262976.0,
+ "1325": 33307011072.0,
+ "1330": 33306550272.0,
+ "1335": 33307181056.0,
+ "1340": 33307406336.0,
+ "1345": 33307463680.0,
+ "1350": 33308135424.0,
+ "1355": 33307480064.0,
+ "1360": 33307533312.0,
+ "1365": 33307066368.0,
+ "1370": 33306595328.0,
+ "1375": 33307891712.0,
+ "1380": 33307830272.0,
+ "1385": 33308487680.0,
+ "1390": 33306521600.0,
+ "1395": 33307338752.0,
+ "1400": 33308430336.0,
+ "1405": 33307768832.0,
+ "1410": 33308041216.0,
+ "1415": 33307797504.0,
+ "1420": 33306605568.0,
+ "1425": 33307240448.0,
+ "1430": 33307322368.0,
+ "1435": 33307559936.0,
+ "1440": 33306662912.0,
+ "1445": 33307058176.0,
+ "1450": 33307705344.0,
+ "1455": 33307291648.0,
+ "1460": 33306861568.0,
+ "1465": 33306312704.0,
+ "1470": 33307394048.0,
+ "1475": 33307211776.0,
+ "1480": 33306527744.0,
+ "1485": 33307361280.0,
+ "1490": 33307693056.0,
+ "1495": 33307271168.0,
+ "1500": 33306820608.0,
+ "1505": 33307092992.0,
+ "1510": 33306624000.0,
+ "1515": 33307097088.0,
+ "1520": 33306931200.0,
+ "1525": 33307635712.0,
+ "1530": 33307353088.0,
+ "1535": 33306468352.0,
+ "1540": 33307172864.0,
+ "1545": 33307693056.0,
+ "1550": 33307938816.0,
+ "1555": 33307832320.0,
+ "1560": 33308182528.0,
+ "1565": 33307099136.0,
+ "1570": 33306798080.0,
+ "1575": 33307492352.0,
+ "1580": 33307688960.0,
+ "1585": 33307326464.0,
+ "1590": 33306988544.0,
+ "1595": 33306818560.0,
+ "1600": 33307836416.0,
+ "1605": 33307590656.0,
+ "1610": 33307168768.0,
+ "1615": 33306931200.0,
+ "1620": 33306732544.0,
+ "1625": 33308260352.0,
+ "1630": 33308227584.0,
+ "1635": 33306957824.0,
+ "1640": 33306759168.0,
+ "1645": 33306021888.0,
+ "1650": 33306689536.0,
+ "1655": 33307332608.0,
+ "1660": 33307170816.0,
+ "1665": 33306583040.0,
+ "1670": 33307535360.0,
+ "1675": 33306912768.0,
+ "1680": 33306675200.0,
+ "1685": 33307774976.0,
+ "1690": 33307783168.0,
+ "1695": 33307971584.0,
+ "1700": 33307623424.0,
+ "1705": 33307652096.0,
+ "1710": 33307731968.0,
+ "1715": 33308090368.0,
+ "1720": 33307172864.0,
+ "1725": 33307672576.0,
+ "1730": 33306355712.0,
+ "1735": 33308229632.0,
+ "1740": 33307142144.0,
+ "1745": 33308151808.0,
+ "1750": 33306898432.0,
+ "1755": 33307105280.0,
+ "1760": 33308000256.0,
+ "1765": 33307750400.0,
+ "1770": 33308450816.0,
+ "1775": 33308184576.0,
+ "1780": 33308129280.0,
+ "1785": 33307936768.0,
+ "1790": 33307238400.0,
+ "1795": 33307922432.0,
+ "1800": 33306900480.0,
+ "1805": 33307203584.0,
+ "1810": 33306923008.0,
+ "1815": 33307617280.0,
+ "1820": 33307664384.0,
+ "1825": 33308440576.0,
+ "1830": 33306843136.0,
+ "1835": 33307979776.0,
+ "1840": 33307588608.0,
+ "1845": 33307602944.0,
+ "1850": 33307774976.0,
+ "1855": 33307529216.0,
+ "1860": 33307054080.0,
+ "1865": 33307097088.0,
+ "1870": 33307373568.0,
+ "1875": 33306265600.0,
+ "1880": 33307275264.0,
+ "1885": 33307224064.0,
+ "1890": 33307324416.0,
+ "1895": 33307283456.0,
+ "1900": 33306810368.0,
+ "1905": 33307191296.0,
+ "1910": 33306884096.0,
+ "1915": 33308162048.0,
+ "1920": 33307664384.0,
+ "1925": 33305972736.0,
+ "1930": 33308504064.0,
+ "1935": 33307377664.0,
+ "1940": 33307119616.0,
+ "1945": 33307416576.0,
+ "1950": 33307746304.0,
+ "1955": 33307420672.0,
+ "1960": 33308073984.0,
+ "1965": 33307148288.0,
+ "1970": 33306775552.0,
+ "1975": 33308207104.0,
+ "1980": 33307473920.0,
+ "1985": 33307095040.0,
+ "1990": 33307527168.0,
+ "1995": 33307037696.0,
+ "2000": 33308801024.0,
+ "2005": 33307985920.0,
+ "2010": 33307516928.0,
+ "2015": 33307604992.0,
+ "2020": 33307406336.0,
+ "2025": 33307719680.0,
+ "2030": 33308381184.0,
+ "2035": 33307914240.0,
+ "2040": 33307324416.0,
+ "2045": 33306476544.0,
+ "2050": 33308246016.0,
+ "2055": 33307430912.0,
+ "2060": 33307912192.0,
+ "2065": 33307543552.0,
+ "2070": 33307670528.0,
+ "2075": 33307482112.0,
+ "2080": 33307871232.0,
+ "2085": 33306722304.0,
+ "2090": 33307549696.0,
+ "2095": 33307260928.0,
+ "2100": 33306765312.0,
+ "2105": 33306847232.0,
+ "2110": 33307332608.0,
+ "2115": 33306480640.0,
+ "2120": 33307168768.0,
+ "2125": 33307277312.0,
+ "2130": 33307314176.0,
+ "2135": 33307752448.0,
+ "2140": 33306710016.0,
+ "2145": 33307478016.0,
+ "2150": 33307729920.0,
+ "2155": 33306943488.0,
+ "2160": 33307508736.0,
+ "2165": 33307049984.0,
+ "2170": 33307158528.0,
+ "2175": 33306599424.0,
+ "2180": 33307054080.0,
+ "2185": 33307017216.0,
+ "2190": 33307119616.0,
+ "2195": 33307289600.0,
+ "2200": 33306726400.0,
+ "2205": 33306636288.0,
+ "2210": 33307639808.0,
+ "2215": 33308215296.0,
+ "2220": 33307314176.0,
+ "2225": 33307437056.0,
+ "2230": 33306318848.0,
+ "2235": 33306941440.0,
+ "2240": 33308131328.0,
+ "2245": 33307707392.0,
+ "2250": 33307256832.0,
+ "2255": 33306845184.0,
+ "2260": 33307736064.0,
+ "2265": 33308620800.0,
+ "2270": 33307357184.0,
+ "2275": 33308151808.0,
+ "2280": 33307981824.0,
+ "2285": 33307922432.0,
+ "2290": 33306767360.0,
+ "2295": 33307670528.0,
+ "2300": 33307179008.0,
+ "2305": 33307545600.0,
+ "2310": 33307924480.0,
+ "2315": 33307396096.0,
+ "2320": 33307725824.0,
+ "2325": 33308024832.0,
+ "2330": 33307793408.0,
+ "2335": 33307019264.0,
+ "2340": 33307162624.0,
+ "2345": 33307934720.0,
+ "2350": 33306232832.0,
+ "2355": 33307719680.0,
+ "2360": 33307375616.0,
+ "2365": 33306537984.0,
+ "2370": 33307279360.0,
+ "2375": 33308131328.0,
+ "2380": 33307136000.0,
+ "2385": 33307490304.0,
+ "2390": 33307316224.0,
+ "2395": 33306587136.0,
+ "2400": 33307594752.0,
+ "2405": 33308393472.0,
+ "2410": 33306726400.0,
+ "2415": 33307506688.0,
+ "2420": 33308407808.0,
+ "2425": 33307942912.0,
+ "2430": 33308116992.0,
+ "2435": 33307308032.0,
+ "2440": 33308362752.0,
+ "2445": 33308071936.0,
+ "2450": 33307740160.0,
+ "2455": 33307959296.0,
+ "2460": 33308258304.0,
+ "2465": 33307299840.0,
+ "2470": 33307056128.0,
+ "2475": 33307224064.0,
+ "2480": 33307713536.0,
+ "2485": 33306550272.0,
+ "2490": 33306992640.0,
+ "2495": 33307232256.0,
+ "2500": 33307095040.0,
+ "2505": 33307107328.0,
+ "2510": 33307488256.0,
+ "2515": 33308360704.0,
+ "2520": 33307369472.0,
+ "2525": 33306959872.0,
+ "2530": 33307258880.0,
+ "2535": 33307082752.0,
+ "2540": 33308633088.0,
+ "2545": 33308542976.0,
+ "2550": 33308002304.0,
+ "2555": 33307961344.0,
+ "2560": 33307328512.0,
+ "2565": 33308299264.0,
+ "2570": 33307770880.0,
+ "2575": 33307877376.0,
+ "2580": 33307990016.0,
+ "2585": 33308016640.0,
+ "2590": 33308135424.0,
+ "2595": 33307617280.0,
+ "2600": 33306667008.0,
+ "2605": 33307422720.0,
+ "2610": 33306683392.0,
+ "2615": 33308669952.0,
+ "2620": 33308616704.0,
+ "2625": 33308366848.0,
+ "2630": 33307574272.0,
+ "2635": 33308166144.0,
+ "2640": 33307983872.0,
+ "2645": 33307609088.0,
+ "2650": 33307807744.0,
+ "2655": 33306955776.0,
+ "2660": 33307273216.0,
+ "2665": 33307709440.0,
+ "2670": 33307693056.0,
+ "2675": 33307731968.0,
+ "2680": 33308227584.0,
+ "2685": 33307742208.0,
+ "2690": 33307734016.0,
+ "2695": 33307424768.0,
+ "2700": 33306644480.0,
+ "2705": 33306300416.0,
+ "2710": 33307881472.0,
+ "2715": 33307488256.0,
+ "2720": 33307318272.0,
+ "2725": 33307604992.0,
+ "2730": 33306710016.0,
+ "2735": 33308049408.0,
+ "2740": 33307437056.0,
+ "2745": 33307572224.0,
+ "2750": 33307136000.0,
+ "2755": 33307584512.0,
+ "2760": 33307355136.0,
+ "2765": 33307713536.0,
+ "2770": 33308000256.0,
+ "2775": 33306460160.0,
+ "2780": 33306923008.0,
+ "2785": 33307017216.0,
+ "2790": 33306720256.0,
+ "2795": 33307785216.0,
+ "2800": 33307234304.0,
+ "2805": 33306685440.0,
+ "2810": 33307469824.0,
+ "2815": 33308069888.0,
+ "2820": 33306460160.0,
+ "2825": 33307467776.0,
+ "2830": 33307666432.0,
+ "2835": 33307371520.0,
+ "2840": 33306904576.0,
+ "2845": 33308061696.0,
+ "2850": 33308520448.0,
+ "2855": 33307695104.0,
+ "2860": 33308487680.0,
+ "2865": 33307058176.0,
+ "2870": 33307303936.0,
+ "2875": 33307324416.0,
+ "2880": 33306968064.0,
+ "2885": 33307641856.0,
+ "2890": 33307785216.0,
+ "2895": 33308221440.0,
+ "2900": 33307596800.0,
+ "2905": 33307533312.0,
+ "2910": 33307459584.0,
+ "2915": 33307799552.0,
+ "2920": 33308461056.0,
+ "2925": 33307938816.0,
+ "2930": 33308268544.0,
+ "2935": 33308594176.0,
+ "2940": 33308170240.0,
+ "2945": 33307578368.0,
+ "2950": 33307590656.0,
+ "2955": 33308131328.0,
+ "2960": 33306839040.0,
+ "2965": 33307111424.0,
+ "2970": 33307570176.0,
+ "2975": 33307766784.0,
+ "2980": 33307600896.0,
+ "2985": 33307123712.0,
+ "2990": 33307641856.0,
+ "2995": 33307527168.0,
+ "3000": 33307863040.0,
+ "3005": 33306927104.0,
+ "3010": 33307738112.0,
+ "3015": 33308217344.0,
+ "3020": 33306697728.0,
+ "3025": 33306970112.0,
+ "3030": 33308127232.0,
+ "3035": 33308213248.0,
+ "3040": 33307578368.0,
+ "3045": 33308327936.0,
+ "3050": 33306910720.0,
+ "3055": 33307004928.0,
+ "3060": 33307602944.0,
+ "3065": 33306970112.0,
+ "3070": 33307985920.0,
+ "3075": 33306945536.0,
+ "3080": 33307312128.0,
+ "3085": 33306533888.0,
+ "3090": 33306933248.0,
+ "3095": 33307906048.0,
+ "3100": 33306793984.0,
+ "3105": 33307127808.0,
+ "3110": 33308295168.0,
+ "3115": 33307295744.0,
+ "3120": 33307897856.0,
+ "3125": 33307066368.0,
+ "3130": 33307781120.0,
+ "3135": 33307762688.0,
+ "3140": 33308196864.0,
+ "3145": 33306904576.0,
+ "3150": 33307140096.0,
+ "3155": 33306660864.0,
+ "3160": 33307514880.0,
+ "3165": 33307246592.0,
+ "3170": 33307613184.0,
+ "3175": 33307375616.0,
+ "3180": 33307551744.0,
+ "3185": 33307842560.0,
+ "3190": 33308342272.0,
+ "3195": 33308350464.0,
+ "3200": 33307799552.0,
+ "3205": 33307099136.0,
+ "3210": 33306869760.0,
+ "3215": 33307678720.0,
+ "3220": 33307111424.0,
+ "3225": 33307146240.0,
+ "3230": 33306972160.0,
+ "3235": 33307387904.0,
+ "3240": 33307521024.0,
+ "3245": 33307287552.0,
+ "3250": 33307523072.0,
+ "3255": 33307639808.0,
+ "3260": 33307092992.0,
+ "3265": 33308338176.0,
+ "3270": 33307273216.0,
+ "3275": 33307713536.0,
+ "3280": 33307719680.0,
+ "3285": 33308049408.0,
+ "3290": 33307484160.0,
+ "3295": 33307594752.0,
+ "3300": 33307228160.0,
+ "3305": 33306580992.0,
+ "3310": 33307541504.0,
+ "3315": 33307211776.0,
+ "3320": 33307324416.0,
+ "3325": 33306615808.0,
+ "3330": 33307777024.0,
+ "3335": 33308135424.0,
+ "3340": 33307351040.0,
+ "3345": 33307131904.0,
+ "3350": 33307031552.0,
+ "3355": 33307791360.0,
+ "3360": 33307410432.0,
+ "3365": 33307090944.0,
+ "3370": 33306187776.0,
+ "3375": 33307113472.0,
+ "3380": 33308071936.0,
+ "3385": 33307717632.0,
+ "3390": 33306648576.0,
+ "3395": 33306781696.0,
+ "3400": 33307734016.0,
+ "3405": 33307570176.0,
+ "3410": 33307750400.0,
+ "3415": 33307920384.0,
+ "3420": 33308157952.0,
+ "3425": 33307500544.0,
+ "3430": 33307168768.0,
+ "3435": 33307645952.0,
+ "3440": 33307185152.0,
+ "3445": 33307459584.0,
+ "3450": 33306804224.0,
+ "3455": 33307662336.0,
+ "3460": 33306748928.0,
+ "3465": 33306497024.0,
+ "3470": 33306796032.0,
+ "3475": 33307947008.0,
+ "3480": 33308039168.0,
+ "3485": 33307676672.0,
+ "3490": 33306728448.0,
+ "3495": 33307115520.0,
+ "3500": 33306628096.0,
+ "3505": 33307537408.0,
+ "3510": 33306945536.0,
+ "3515": 33306902528.0,
+ "3520": 33307553792.0,
+ "3525": 33307590656.0,
+ "3530": 33307852800.0,
+ "3535": 33306773504.0,
+ "3540": 33307953152.0,
+ "3545": 33307463680.0,
+ "3550": 33307123712.0,
+ "3555": 33307738112.0,
+ "3560": 33307766784.0,
+ "3565": 33307088896.0,
+ "3570": 33306882048.0,
+ "3575": 33307443200.0,
+ "3580": 33306951680.0,
+ "3585": 33306841088.0,
+ "3590": 33308293120.0,
+ "3595": 33307723776.0,
+ "3600": 33307756544.0,
+ "3605": 33307930624.0,
+ "3610": 33307985920.0,
+ "3615": 33307222016.0,
+ "3620": 33307430912.0,
+ "3625": 33307148288.0,
+ "3630": 33306388480.0,
+ "3635": 33307035648.0,
+ "3640": 33307455488.0,
+ "3645": 33306906624.0,
+ "3650": 33307545600.0,
+ "3655": 33307336704.0,
+ "3660": 33306910720.0,
+ "3665": 33307623424.0,
+ "3670": 33306824704.0,
+ "3675": 33307590656.0,
+ "3680": 33307373568.0,
+ "3685": 33306505216.0,
+ "3690": 33307817984.0,
+ "3695": 33306890240.0,
+ "3700": 33306802176.0,
+ "3705": 33306945536.0,
+ "3710": 33306904576.0,
+ "3715": 33307754496.0,
+ "3720": 33308395520.0,
+ "3725": 33308112896.0,
+ "3730": 33307652096.0,
+ "3735": 33307867136.0,
+ "3740": 33307805696.0,
+ "3745": 33308069888.0,
+ "3750": 33307826176.0,
+ "3755": 33306439680.0,
+ "3760": 33306849280.0,
+ "3765": 33307471872.0,
+ "3770": 33307095040.0,
+ "3775": 33307492352.0,
+ "3780": 33308141568.0,
+ "3785": 33307910144.0,
+ "3790": 33307656192.0,
+ "3795": 33307727872.0,
+ "3800": 33307246592.0,
+ "3805": 33307848704.0,
+ "3810": 33307490304.0,
+ "3815": 33307357184.0,
+ "3820": 33307346944.0,
+ "3825": 33307619328.0,
+ "3830": 33308102656.0,
+ "3835": 33306849280.0,
+ "3840": 33307678720.0,
+ "3845": 33307258880.0,
+ "3850": 33307686912.0,
+ "3855": 33307467776.0,
+ "3860": 33307471872.0,
+ "3865": 33307439104.0,
+ "3870": 33307676672.0,
+ "3875": 33306865664.0,
+ "3880": 33307232256.0,
+ "3885": 33307099136.0,
+ "3890": 33307854848.0,
+ "3895": 33306370048.0,
+ "3900": 33306900480.0,
+ "3905": 33306824704.0,
+ "3910": 33307361280.0,
+ "3915": 33306591232.0,
+ "3920": 33307213824.0,
+ "3925": 33306980352.0,
+ "3930": 33308110848.0,
+ "3935": 33307179008.0,
+ "3940": 33307379712.0,
+ "3945": 33307813888.0,
+ "3950": 33307277312.0,
+ "3955": 33307203584.0,
+ "3960": 33307234304.0,
+ "3965": 33307121664.0,
+ "3970": 33307303936.0,
+ "3975": 33307144192.0,
+ "3980": 33307869184.0,
+ "3985": 33307660288.0,
+ "3990": 33307779072.0,
+ "3995": 33307795456.0,
+ "4000": 33307131904.0,
+ "4005": 33307238400.0,
+ "4010": 33307875328.0,
+ "4015": 33306726400.0,
+ "4020": 33308227584.0,
+ "4025": 33307799552.0,
+ "4030": 33307318272.0,
+ "4035": 33308190720.0,
+ "4040": 33307932672.0,
+ "4045": 33307291648.0,
+ "4050": 33307959296.0,
+ "4055": 33307447296.0,
+ "4060": 33307486208.0,
+ "4065": 33308088320.0,
+ "4070": 33307183104.0,
+ "4075": 33307201536.0,
+ "4080": 33308184576.0,
+ "4085": 33306406912.0,
+ "4090": 33307891712.0,
+ "4095": 33307031552.0,
+ "4100": 33308100608.0,
+ "4105": 33307258880.0,
+ "4110": 33307492352.0,
+ "4115": 33308344320.0,
+ "4120": 33306552320.0,
+ "4125": 33307611136.0,
+ "4130": 33306083328.0,
+ "4135": 33308463104.0,
+ "4140": 33307611136.0,
+ "4145": 33307455488.0,
+ "4150": 33307658240.0,
+ "4155": 33307133952.0,
+ "4160": 33308233728.0,
+ "4165": 33307408384.0,
+ "4170": 33306888192.0,
+ "4175": 33307852800.0,
+ "4180": 33307150336.0,
+ "4185": 33307127808.0,
+ "4190": 33307582464.0,
+ "4195": 33308610560.0,
+ "4200": 33308231680.0,
+ "4205": 33307906048.0,
+ "4210": 33308307456.0,
+ "4215": 33306363904.0,
+ "4220": 33306980352.0,
+ "4225": 33306318848.0,
+ "4230": 33307731968.0,
+ "4235": 33307142144.0,
+ "4240": 33307432960.0,
+ "4245": 33307097088.0,
+ "4250": 33307783168.0,
+ "4255": 33307365376.0,
+ "4260": 33306947584.0,
+ "4265": 33306611712.0,
+ "4270": 33306347520.0,
+ "4275": 33306624000.0,
+ "4280": 33307185152.0,
+ "4285": 33307922432.0,
+ "4290": 33307508736.0,
+ "4295": 33307658240.0,
+ "4300": 33308405760.0,
+ "4305": 33306474496.0,
+ "4310": 33307557888.0,
+ "4315": 33308307456.0,
+ "4320": 33307719680.0,
+ "4325": 33306824704.0,
+ "4330": 33307594752.0,
+ "4335": 33306144768.0,
+ "4340": 33307852800.0,
+ "4345": 33307342848.0,
+ "4350": 33308139520.0,
+ "4355": 33307713536.0,
+ "4360": 33307373568.0,
+ "4365": 33308065792.0,
+ "4370": 33306681344.0,
+ "4375": 33307770880.0,
+ "4380": 33307361280.0,
+ "4385": 33307086848.0,
+ "4390": 33307019264.0,
+ "4395": 33306986496.0,
+ "4400": 33307103232.0,
+ "4405": 33307664384.0,
+ "4410": 33307996160.0,
+ "4415": 33306990592.0,
+ "4420": 33306546176.0,
+ "4425": 33306904576.0,
+ "4430": 33307303936.0,
+ "4435": 33306763264.0,
+ "4440": 33308063744.0,
+ "4445": 33307242496.0,
+ "4450": 33307283456.0,
+ "4455": 33306654720.0,
+ "4460": 33307205632.0,
+ "4465": 33306867712.0,
+ "4470": 33307916288.0,
+ "4475": 33307791360.0,
+ "4480": 33308450816.0,
+ "4485": 33307547648.0,
+ "4490": 33307090944.0,
+ "4495": 33307000832.0,
+ "4500": 33306935296.0,
+ "4505": 33307099136.0,
+ "4510": 33307525120.0,
+ "4515": 33307367424.0,
+ "4520": 33307813888.0,
+ "4525": 33307715584.0,
+ "4530": 33307901952.0,
+ "4535": 33307174912.0,
+ "4540": 33306880000.0,
+ "4545": 33307138048.0,
+ "4550": 33306873856.0,
+ "4555": 33306316800.0,
+ "4560": 33305849856.0,
+ "4565": 33307187200.0,
+ "4570": 33307260928.0,
+ "4575": 33307410432.0,
+ "4580": 33307201536.0,
+ "4585": 33306920960.0,
+ "4590": 33307355136.0,
+ "4595": 33307346944.0,
+ "4600": 33307856896.0,
+ "4605": 33307752448.0,
+ "4610": 33307095040.0,
+ "4615": 33306286080.0,
+ "4620": 33306699776.0,
+ "4625": 33308069888.0,
+ "4630": 33307439104.0,
+ "4635": 33306900480.0,
+ "4640": 33307076608.0,
+ "4645": 33308160000.0,
+ "4650": 33307758592.0,
+ "4655": 33307865088.0,
+ "4660": 33306255360.0,
+ "4665": 33307641856.0,
+ "4670": 33307912192.0,
+ "4675": 33306603520.0,
+ "4680": 33307799552.0,
+ "4685": 33307488256.0,
+ "4690": 33307394048.0,
+ "4695": 33306763264.0,
+ "4700": 33307873280.0,
+ "4705": 33308106752.0,
+ "4710": 33307617280.0,
+ "4715": 33307047936.0,
+ "4720": 33307901952.0,
+ "4725": 33307793408.0,
+ "4730": 33308123136.0,
+ "4735": 33307451392.0,
+ "4740": 33307623424.0,
+ "4745": 33306857472.0,
+ "4750": 33308436480.0,
+ "4755": 33307260928.0,
+ "4760": 33307975680.0,
+ "4765": 33307965440.0,
+ "4770": 33306859520.0,
+ "4775": 33307922432.0,
+ "4780": 33306978304.0,
+ "4785": 33306869760.0,
+ "4790": 33307084800.0,
+ "4795": 33307226112.0,
+ "4800": 33307961344.0,
+ "4805": 33308334080.0,
+ "4810": 33305587712.0,
+ "4815": 33307928576.0,
+ "4820": 33307875328.0,
+ "4825": 33306957824.0,
+ "4830": 33307797504.0,
+ "4835": 33306116096.0,
+ "4840": 33307654144.0,
+ "4845": 33307131904.0,
+ "4850": 33308055552.0,
+ "4855": 33305792512.0,
+ "4860": 33307402240.0,
+ "4865": 33307086848.0,
+ "4870": 33307637760.0,
+ "4875": 33307789312.0,
+ "4880": 33307701248.0,
+ "4885": 33308010496.0,
+ "4890": 33307039744.0,
+ "4895": 33307369472.0,
+ "4900": 33307127808.0,
+ "4905": 33306988544.0,
+ "4910": 33308276736.0,
+ "4915": 33307090944.0,
+ "4920": 33307015168.0,
+ "4925": 33308043264.0,
+ "4930": 33307607040.0,
+ "4935": 33308209152.0,
+ "4940": 33307725824.0,
+ "4945": 33307985920.0,
+ "4950": 33307582464.0,
+ "4955": 33307297792.0,
+ "4960": 33307639808.0,
+ "4965": 33307445248.0,
+ "4970": 33306869760.0,
+ "4975": 33306787840.0,
+ "4980": 33307099136.0,
+ "4985": 33307635712.0,
+ "4990": 33307406336.0,
+ "4995": 33307471872.0,
+ "5000": 33307375616.0,
+ "5005": 33307672576.0,
+ "5010": 33306970112.0,
+ "5015": 33307244544.0,
+ "5020": 33306966016.0,
+ "5025": 33307705344.0,
+ "5030": 33307463680.0,
+ "5035": 33306818560.0,
+ "5040": 33306972160.0,
+ "5045": 33308157952.0,
+ "5050": 33306376192.0,
+ "5055": 33307594752.0,
+ "5060": 33308471296.0,
+ "5065": 33307455488.0,
+ "5070": 33307301888.0,
+ "5075": 33307488256.0,
+ "5080": 33307910144.0,
+ "5085": 33307635712.0,
+ "5090": 33307406336.0,
+ "5095": 33307254784.0,
+ "5100": 33306828800.0,
+ "5105": 33307852800.0,
+ "5110": 33308258304.0,
+ "5115": 33307228160.0,
+ "5120": 33307955200.0,
+ "5125": 33305640960.0,
+ "5130": 33306683392.0,
+ "5135": 33307336704.0,
+ "5140": 33307834368.0,
+ "5145": 33307060224.0,
+ "5150": 33307023360.0,
+ "5155": 33307308032.0,
+ "5160": 33306664960.0,
+ "5165": 33307123712.0,
+ "5170": 33306935296.0,
+ "5175": 33308094464.0,
+ "5180": 33306566656.0,
+ "5185": 33306796032.0,
+ "5190": 33307545600.0,
+ "5195": 33308067840.0,
+ "5200": 33307754496.0,
+ "5205": 33307445248.0,
+ "5210": 33306785792.0,
+ "5215": 33307551744.0,
+ "5220": 33308188672.0,
+ "5225": 33307338752.0,
+ "5230": 33307283456.0,
+ "5235": 33306976256.0,
+ "5240": 33308041216.0,
+ "5245": 33308340224.0,
+ "5250": 33308153856.0,
+ "5255": 33307590656.0,
+ "5260": 33306896384.0,
+ "5265": 33308303360.0,
+ "5270": 33308796928.0,
+ "5275": 33307949056.0,
+ "5280": 33306157056.0,
+ "5285": 33307904000.0,
+ "5290": 33308143616.0,
+ "5295": 33306533888.0,
+ "5300": 33307912192.0,
+ "5305": 33308338176.0,
+ "5310": 33308688384.0,
+ "5315": 33308045312.0,
+ "5320": 33306206208.0,
+ "5325": 33308219392.0,
+ "5330": 33308012544.0,
+ "5335": 33307602944.0,
+ "5340": 33306685440.0,
+ "5345": 33308209152.0,
+ "5350": 33307150336.0,
+ "5355": 33308176384.0,
+ "5360": 33307273216.0,
+ "5365": 33307850752.0,
+ "5370": 33307222016.0,
+ "5375": 33307803648.0,
+ "5380": 33307617280.0,
+ "5385": 33307179008.0,
+ "5390": 33307389952.0,
+ "5395": 33306927104.0,
+ "5400": 33307518976.0,
+ "5405": 33307400192.0,
+ "5410": 33307598848.0,
+ "5415": 33307846656.0,
+ "5420": 33307490304.0,
+ "5425": 33307459584.0,
+ "5430": 33307283456.0,
+ "5435": 33307453440.0,
+ "5440": 33307383808.0,
+ "5445": 33307117568.0,
+ "5450": 33307832320.0,
+ "5455": 33307582464.0,
+ "5460": 33306963968.0,
+ "5465": 33306947584.0,
+ "5470": 33307355136.0,
+ "5475": 33306748928.0,
+ "5480": 33306435584.0,
+ "5485": 33307590656.0,
+ "5490": 33307787264.0,
+ "5495": 33307568128.0,
+ "5500": 33307351040.0,
+ "5505": 33307568128.0,
+ "5510": 33307426816.0,
+ "5515": 33307451392.0,
+ "5520": 33307549696.0,
+ "5525": 33307000832.0,
+ "5530": 33307566080.0,
+ "5535": 33307664384.0,
+ "5540": 33306966016.0,
+ "5545": 33307781120.0,
+ "5550": 33307275264.0,
+ "5555": 33307269120.0,
+ "5560": 33307576320.0,
+ "5565": 33307377664.0,
+ "5570": 33307052032.0,
+ "5575": 33306978304.0,
+ "5580": 33307965440.0,
+ "5585": 33307494400.0,
+ "5590": 33308055552.0,
+ "5595": 33306943488.0,
+ "5600": 33306542080.0,
+ "5605": 33307680768.0,
+ "5610": 33308542976.0,
+ "5615": 33307826176.0,
+ "5620": 33308108800.0,
+ "5625": 33308225536.0,
+ "5630": 33308069888.0,
+ "5635": 33307760640.0,
+ "5640": 33307500544.0,
+ "5645": 33307930624.0,
+ "5650": 33306755072.0,
+ "5655": 33308192768.0,
+ "5660": 33308631040.0,
+ "5665": 33307418624.0,
+ "5670": 33307504640.0,
+ "5675": 33307715584.0,
+ "5680": 33307910144.0,
+ "5685": 33307996160.0,
+ "5690": 33307478016.0,
+ "5695": 33308164096.0,
+ "5700": 33307906048.0,
+ "5705": 33307750400.0,
+ "5710": 33306779648.0,
+ "5715": 33307219968.0,
+ "5720": 33307750400.0,
+ "5725": 33307537408.0,
+ "5730": 33307262976.0,
+ "5735": 33306767360.0,
+ "5740": 33307508736.0,
+ "5745": 33306753024.0,
+ "5750": 33306636288.0,
+ "5755": 33306943488.0,
+ "5760": 33307553792.0,
+ "5765": 33307842560.0,
+ "5770": 33307047936.0,
+ "5775": 33307348992.0,
+ "5780": 33306361856.0,
+ "5785": 33307709440.0,
+ "5790": 33307832320.0,
+ "5795": 33307406336.0,
+ "5800": 33307056128.0,
+ "5805": 33307631616.0,
+ "5810": 33307766784.0,
+ "5815": 33307971584.0,
+ "5820": 33307447296.0,
+ "5825": 33307084800.0,
+ "5830": 33307324416.0,
+ "5835": 33307127808.0,
+ "5840": 33307729920.0,
+ "5845": 33307088896.0,
+ "5850": 33307635712.0,
+ "5855": 33307119616.0,
+ "5860": 33306703872.0,
+ "5865": 33307291648.0,
+ "5870": 33307613184.0,
+ "5875": 33307893760.0,
+ "5880": 33307893760.0,
+ "5885": 33307301888.0,
+ "5890": 33307830272.0,
+ "5895": 33306671104.0,
+ "5900": 33306488832.0,
+ "5905": 33308141568.0,
+ "5910": 33307373568.0,
+ "5915": 33307330560.0,
+ "5920": 33307656192.0,
+ "5925": 33307533312.0,
+ "5930": 33307848704.0,
+ "5935": 33307586560.0,
+ "5940": 33307602944.0,
+ "5945": 33307631616.0,
+ "5950": 33306615808.0,
+ "5955": 33307719680.0,
+ "5960": 33308553216.0,
+ "5965": 33308676096.0,
+ "5970": 33308313600.0,
+ "5975": 33306810368.0,
+ "5980": 33307222016.0,
+ "5985": 33307367424.0,
+ "5990": 33307119616.0,
+ "5995": 33307166720.0,
+ "6000": 33307822080.0,
+ "6005": 33307553792.0,
+ "6010": 33307756544.0,
+ "6015": 33306392576.0,
+ "6020": 33308116992.0,
+ "6025": 33307738112.0,
+ "6030": 33307459584.0,
+ "6035": 33306920960.0,
+ "6040": 33307701248.0,
+ "6045": 33307932672.0,
+ "6050": 33307496448.0,
+ "6055": 33307133952.0,
+ "6060": 33306370048.0,
+ "6065": 33307521024.0,
+ "6070": 33307244544.0,
+ "6075": 33306447872.0,
+ "6080": 33306963968.0,
+ "6085": 33307932672.0,
+ "6090": 33307293696.0,
+ "6095": 33307058176.0,
+ "6100": 33307449344.0,
+ "6105": 33307613184.0,
+ "6110": 33307779072.0,
+ "6115": 33306832896.0,
+ "6120": 33306732544.0,
+ "6125": 33306488832.0,
+ "6130": 33308866560.0,
+ "6135": 33308000256.0,
+ "6140": 33307906048.0,
+ "6145": 33308504064.0,
+ "6150": 33307826176.0,
+ "6155": 33306906624.0,
+ "6160": 33307533312.0,
+ "6165": 33307578368.0,
+ "6170": 33307891712.0,
+ "6175": 33307537408.0,
+ "6180": 33307803648.0,
+ "6185": 33308125184.0,
+ "6190": 33307342848.0,
+ "6195": 33308135424.0,
+ "6200": 33306468352.0,
+ "6205": 33308026880.0,
+ "6210": 33308028928.0,
+ "6215": 33308157952.0,
+ "6220": 33307662336.0,
+ "6225": 33307344896.0,
+ "6230": 33308231680.0,
+ "6235": 33307148288.0,
+ "6240": 33308809216.0,
+ "6245": 33307017216.0,
+ "6250": 33307234304.0,
+ "6255": 33308430336.0,
+ "6260": 33307246592.0,
+ "6265": 33307418624.0,
+ "6270": 33308319744.0,
+ "6275": 33307090944.0,
+ "6280": 33307404288.0,
+ "6285": 33308227584.0,
+ "6290": 33307656192.0,
+ "6295": 33306865664.0,
+ "6300": 33307596800.0,
+ "6305": 33308192768.0,
+ "6310": 33307695104.0,
+ "6315": 33307361280.0,
+ "6320": 33306775552.0,
+ "6325": 33307557888.0,
+ "6330": 33307639808.0,
+ "6335": 33307820032.0,
+ "6340": 33307410432.0,
+ "6345": 33307410432.0,
+ "6350": 33308256256.0,
+ "6355": 33307082752.0,
+ "6360": 33306855424.0,
+ "6365": 33307418624.0,
+ "6370": 33307066368.0,
+ "6375": 33307891712.0,
+ "6380": 33307779072.0,
+ "6385": 33306128384.0,
+ "6390": 33306884096.0,
+ "6395": 33307060224.0,
+ "6400": 33307250688.0,
+ "6405": 33308135424.0,
+ "6410": 33308155904.0,
+ "6415": 33307101184.0,
+ "6420": 33306318848.0,
+ "6425": 33308065792.0,
+ "6430": 33307813888.0,
+ "6435": 33307842560.0,
+ "6440": 33308571648.0,
+ "6445": 33306138624.0,
+ "6450": 33307762688.0,
+ "6455": 33308119040.0,
+ "6460": 33308037120.0,
+ "6465": 33308467200.0,
+ "6470": 33307181056.0,
+ "6475": 33307246592.0,
+ "6480": 33306855424.0,
+ "6485": 33308440576.0,
+ "6490": 33307863040.0,
+ "6495": 33306857472.0,
+ "6500": 33306529792.0,
+ "6505": 33307097088.0,
+ "6510": 33307842560.0,
+ "6515": 33307095040.0,
+ "6520": 33307848704.0,
+ "6525": 33307596800.0,
+ "6530": 33307117568.0,
+ "6535": 33307811840.0,
+ "6540": 33307645952.0,
+ "6545": 33307211776.0,
+ "6550": 33308196864.0,
+ "6555": 33307213824.0,
+ "6560": 33307326464.0,
+ "6565": 33306490880.0,
+ "6570": 33306877952.0,
+ "6575": 33307199488.0,
+ "6580": 33308370944.0,
+ "6585": 33307828224.0,
+ "6590": 33307871232.0,
+ "6595": 33307590656.0,
+ "6600": 33306578944.0,
+ "6605": 33307496448.0,
+ "6610": 33307912192.0,
+ "6615": 33307521024.0,
+ "6620": 33307189248.0,
+ "6625": 33306961920.0,
+ "6630": 33306800128.0,
+ "6635": 33306957824.0,
+ "6640": 33307762688.0,
+ "6645": 33306427392.0,
+ "6650": 33307672576.0,
+ "6655": 33305133056.0,
+ "6660": 33307598848.0,
+ "6665": 33306884096.0,
+ "6670": 33307500544.0,
+ "6675": 33307592704.0,
+ "6680": 33306923008.0,
+ "6685": 33307084800.0,
+ "6690": 33307402240.0,
+ "6695": 33307963392.0,
+ "6700": 33307336704.0,
+ "6705": 33306845184.0,
+ "6710": 33307230208.0,
+ "6715": 33306310656.0,
+ "6720": 33307834368.0,
+ "6725": 33308094464.0,
+ "6730": 33308327936.0,
+ "6735": 33308092416.0,
+ "6740": 33306873856.0,
+ "6745": 33308082176.0,
+ "6750": 33306112000.0,
+ "6755": 33306810368.0,
+ "6760": 33307394048.0,
+ "6765": 33307414528.0,
+ "6770": 33308286976.0,
+ "6775": 33308618752.0,
+ "6780": 33306904576.0,
+ "6785": 33308182528.0,
+ "6790": 33308057600.0,
+ "6795": 33307049984.0,
+ "6800": 33306744832.0,
+ "6805": 33307242496.0,
+ "6810": 33307176960.0,
+ "6815": 33307779072.0,
+ "6820": 33306849280.0,
+ "6825": 33307623424.0,
+ "6830": 33307887616.0,
+ "6835": 33307670528.0,
+ "6840": 33308348416.0,
+ "6845": 33308184576.0,
+ "6850": 33307727872.0,
+ "6855": 33307252736.0,
+ "6860": 33307680768.0,
+ "6865": 33306963968.0,
+ "6870": 33307099136.0,
+ "6875": 33307037696.0,
+ "6880": 33307635712.0,
+ "6885": 33307615232.0,
+ "6890": 33307652096.0,
+ "6895": 33307369472.0,
+ "6900": 33307947008.0,
+ "6905": 33307334656.0,
+ "6910": 33306824704.0,
+ "6915": 33307537408.0,
+ "6920": 33306619904.0,
+ "6925": 33306408960.0,
+ "6930": 33306765312.0,
+ "6935": 33306609664.0,
+ "6940": 33307623424.0,
+ "6945": 33307160576.0,
+ "6950": 33307463680.0,
+ "6955": 33306507264.0,
+ "6960": 33307185152.0,
+ "6965": 33307019264.0,
+ "6970": 33307598848.0,
+ "6975": 33307435008.0,
+ "6980": 33307238400.0,
+ "6985": 33306222592.0,
+ "6990": 33308581888.0,
+ "6995": 33307254784.0,
+ "7000": 33308035072.0,
+ "7005": 33308233728.0,
+ "7010": 33307092992.0,
+ "7015": 33307193344.0,
+ "7020": 33307643904.0,
+ "7025": 33308274688.0,
+ "7030": 33307019264.0,
+ "7035": 33308454912.0,
+ "7040": 33308086272.0,
+ "7045": 33307277312.0,
+ "7050": 33307172864.0,
+ "7055": 33306599424.0,
+ "7060": 33307613184.0,
+ "7065": 33307031552.0,
+ "7070": 33306243072.0,
+ "7075": 33308037120.0,
+ "7080": 33306759168.0,
+ "7085": 33308033024.0,
+ "7090": 33307971584.0,
+ "7095": 33306873856.0,
+ "7100": 33308522496.0,
+ "7105": 33307363328.0,
+ "7110": 33308063744.0,
+ "7115": 33307770880.0,
+ "7120": 33307906048.0,
+ "7125": 33307443200.0,
+ "7130": 33307574272.0,
+ "7135": 33307541504.0,
+ "7140": 33306765312.0,
+ "7145": 33307854848.0,
+ "7150": 33306853376.0,
+ "7155": 33307856896.0,
+ "7160": 33307906048.0,
+ "7165": 33308184576.0,
+ "7170": 33308272640.0,
+ "7175": 33306417152.0,
+ "7180": 33307107328.0,
+ "7185": 33307860992.0,
+ "7190": 33307078656.0,
+ "7195": 33307494400.0,
+ "7200": 33307613184.0,
+ "7205": 33307680768.0,
+ "7210": 33307990016.0,
+ "7215": 33306822656.0,
+ "7220": 33306730496.0,
+ "7225": 33307539456.0,
+ "7230": 33307744256.0,
+ "7235": 33306136576.0,
+ "7240": 33307189248.0,
+ "7245": 33307236352.0,
+ "7250": 33306980352.0,
+ "7255": 33307832320.0,
+ "7260": 33307426816.0,
+ "7265": 33307340800.0,
+ "7270": 33307844608.0,
+ "7275": 33308094464.0,
+ "7280": 33308602368.0,
+ "7285": 33307498496.0,
+ "7290": 33307920384.0,
+ "7295": 33307426816.0,
+ "7300": 33306392576.0,
+ "7305": 33306718208.0,
+ "7310": 33307260928.0,
+ "7315": 33307527168.0,
+ "7320": 33306963968.0,
+ "7325": 33308188672.0,
+ "7330": 33307799552.0,
+ "7335": 33307717632.0,
+ "7340": 33307238400.0,
+ "7345": 33307365376.0,
+ "7350": 33307314176.0,
+ "7355": 33307940864.0,
+ "7360": 33306284032.0,
+ "7365": 33307893760.0,
+ "7370": 33306275840.0,
+ "7375": 33307873280.0,
+ "7380": 33309245440.0,
+ "7385": 33306730496.0,
+ "7390": 33307758592.0,
+ "7395": 33306609664.0,
+ "7400": 33307652096.0,
+ "7405": 33306427392.0,
+ "7410": 33308524544.0,
+ "7415": 33307961344.0,
+ "7420": 33307242496.0,
+ "7425": 33307811840.0,
+ "7430": 33307119616.0,
+ "7435": 33307428864.0,
+ "7440": 33307709440.0,
+ "7445": 33308342272.0,
+ "7450": 33306980352.0,
+ "7455": 33307351040.0,
+ "7460": 33306730496.0,
+ "7465": 33306537984.0,
+ "7470": 33307664384.0,
+ "7475": 33308037120.0,
+ "7480": 33307179008.0,
+ "7485": 33308467200.0,
+ "7490": 33307822080.0,
+ "7495": 33306638336.0,
+ "7500": 33306689536.0,
+ "7505": 33307717632.0,
+ "7510": 33306789888.0,
+ "7515": 33307518976.0,
+ "7520": 33307260928.0,
+ "7525": 33307676672.0,
+ "7530": 33306916864.0,
+ "7535": 33306996736.0,
+ "7540": 33306566656.0,
+ "7545": 33306720256.0,
+ "7550": 33307584512.0,
+ "7555": 33307471872.0,
+ "7560": 33306736640.0,
+ "7565": 33306292224.0,
+ "7570": 33307066368.0,
+ "7575": 33306871808.0,
+ "7580": 33307324416.0,
+ "7585": 33307115520.0,
+ "7590": 33306341376.0,
+ "7595": 33307744256.0,
+ "7600": 33307482112.0,
+ "7605": 33308149760.0,
+ "7610": 33307525120.0,
+ "7615": 33307656192.0,
+ "7620": 33307224064.0,
+ "7625": 33307158528.0,
+ "7630": 33307742208.0,
+ "7635": 33308012544.0,
+ "7640": 33307049984.0,
+ "7645": 33308631040.0,
+ "7650": 33307865088.0,
+ "7655": 33308229632.0,
+ "7660": 33307043840.0,
+ "7665": 33307037696.0,
+ "7670": 33306791936.0,
+ "7675": 33307320320.0,
+ "7680": 33307293696.0,
+ "7685": 33307432960.0,
+ "7690": 33307103232.0,
+ "7695": 33307568128.0,
+ "7700": 33306312704.0,
+ "7705": 33307795456.0,
+ "7710": 33307996160.0,
+ "7715": 33307133952.0,
+ "7720": 33308164096.0,
+ "7725": 33307254784.0,
+ "7730": 33307830272.0,
+ "7735": 33307721728.0,
+ "7740": 33307492352.0,
+ "7745": 33307783168.0,
+ "7750": 33306728448.0,
+ "7755": 33307734016.0,
+ "7760": 33308614656.0,
+ "7765": 33306791936.0,
+ "7770": 33308278784.0,
+ "7775": 33307873280.0,
+ "7780": 33307078656.0,
+ "7785": 33306990592.0,
+ "7790": 33307062272.0,
+ "7795": 33307680768.0,
+ "7800": 33306982400.0,
+ "7805": 33308090368.0,
+ "7810": 33307308032.0,
+ "7815": 33307078656.0,
+ "7820": 33307951104.0,
+ "7825": 33306480640.0,
+ "7830": 33307258880.0,
+ "7835": 33307891712.0,
+ "7840": 33307432960.0,
+ "7845": 33307066368.0,
+ "7850": 33306910720.0,
+ "7855": 33307938816.0,
+ "7860": 33307308032.0,
+ "7865": 33308264448.0,
+ "7870": 33307729920.0,
+ "7875": 33308129280.0,
+ "7880": 33308352512.0,
+ "7885": 33307398144.0,
+ "7890": 33306920960.0,
+ "7895": 33307156480.0,
+ "7900": 33308221440.0,
+ "7905": 33308047360.0,
+ "7910": 33306146816.0,
+ "7915": 33306910720.0,
+ "7920": 33307090944.0,
+ "7925": 33308264448.0,
+ "7930": 33307908096.0,
+ "7935": 33307465728.0,
+ "7940": 33307375616.0,
+ "7945": 33307848704.0,
+ "7950": 33308090368.0,
+ "7955": 33307043840.0,
+ "7960": 33307168768.0,
+ "7965": 33307846656.0,
+ "7970": 33306454016.0,
+ "7975": 33307635712.0,
+ "7980": 33307555840.0,
+ "7985": 33307131904.0,
+ "7990": 33306732544.0,
+ "7995": 33307430912.0,
+ "8000": 33307674624.0,
+ "8005": 33307746304.0,
+ "8010": 33308002304.0,
+ "8015": 33306906624.0,
+ "8020": 33307895808.0,
+ "8025": 33308231680.0,
+ "8030": 33307664384.0,
+ "8035": 33306888192.0,
+ "8040": 33308024832.0,
+ "8045": 33307693056.0,
+ "8050": 33306583040.0,
+ "8055": 33307201536.0,
+ "8060": 33307594752.0,
+ "8065": 33308260352.0,
+ "8070": 33307426816.0,
+ "8075": 33308108800.0,
+ "8080": 33308178432.0,
+ "8085": 33307308032.0,
+ "8090": 33306513408.0,
+ "8095": 33306968064.0,
+ "8100": 33308413952.0,
+ "8105": 33308241920.0,
+ "8110": 33307471872.0,
+ "8115": 33307832320.0,
+ "8120": 33307193344.0,
+ "8125": 33307295744.0,
+ "8130": 33306775552.0,
+ "8135": 33307097088.0,
+ "8140": 33307865088.0,
+ "8145": 33306746880.0,
+ "8150": 33307023360.0,
+ "8155": 33306806272.0,
+ "8160": 33307373568.0,
+ "8165": 33307631616.0,
+ "8170": 33306769408.0,
+ "8175": 33308239872.0,
+ "8180": 33307240448.0,
+ "8185": 33307471872.0,
+ "8190": 33308184576.0,
+ "8195": 33307754496.0,
+ "8200": 33307459584.0,
+ "8205": 33307850752.0,
+ "8210": 33306810368.0,
+ "8215": 33306222592.0,
+ "8220": 33307795456.0,
+ "8225": 33308078080.0,
+ "8230": 33306132480.0,
+ "8235": 33308764160.0,
+ "8240": 33307432960.0,
+ "8245": 33307867136.0,
+ "8250": 33308260352.0,
+ "8255": 33308334080.0,
+ "8260": 33308233728.0,
+ "8265": 33308528640.0,
+ "8270": 33307699200.0,
+ "8275": 33306748928.0,
+ "8280": 33307635712.0,
+ "8285": 33308008448.0,
+ "8290": 33307590656.0,
+ "8295": 33308041216.0,
+ "8300": 33307516928.0,
+ "8305": 33307879424.0,
+ "8310": 33307576320.0,
+ "8315": 33308366848.0,
+ "8320": 33307496448.0,
+ "8325": 33307256832.0,
+ "8330": 33307680768.0,
+ "8335": 33306669056.0,
+ "8340": 33306990592.0,
+ "8345": 33307936768.0,
+ "8350": 33307955200.0,
+ "8355": 33307791360.0,
+ "8360": 33306640384.0,
+ "8365": 33307586560.0,
+ "8370": 33307648000.0,
+ "8375": 33306890240.0,
+ "8380": 33307764736.0,
+ "8385": 33307871232.0,
+ "8390": 33307023360.0,
+ "8395": 33307664384.0,
+ "8400": 33307510784.0,
+ "8405": 33307338752.0,
+ "8410": 33307316224.0,
+ "8415": 33307566080.0,
+ "8420": 33307891712.0,
+ "8425": 33307676672.0,
+ "8430": 33307693056.0,
+ "8435": 33306812416.0,
+ "8440": 33307762688.0,
+ "8445": 33307447296.0,
+ "8450": 33307426816.0,
+ "8455": 33306660864.0,
+ "8460": 33307385856.0,
+ "8465": 33308121088.0,
+ "8470": 33307664384.0,
+ "8475": 33307023360.0,
+ "8480": 33308082176.0,
+ "8485": 33307346944.0,
+ "8490": 33307471872.0,
+ "8495": 33307889664.0,
+ "8500": 33307492352.0,
+ "8505": 33307502592.0,
+ "8510": 33307815936.0,
+ "8515": 33307983872.0,
+ "8520": 33306431488.0,
+ "8525": 33306537984.0,
+ "8530": 33307199488.0,
+ "8535": 33307848704.0,
+ "8540": 33307459584.0,
+ "8545": 33307432960.0,
+ "8550": 33307600896.0,
+ "8555": 33308553216.0,
+ "8560": 33307701248.0,
+ "8565": 33307799552.0,
+ "8570": 33307934720.0,
+ "8575": 33306324992.0,
+ "8580": 33307648000.0,
+ "8585": 33307951104.0,
+ "8590": 33308108800.0,
+ "8595": 33308037120.0,
+ "8600": 33308182528.0,
+ "8605": 33307410432.0,
+ "8610": 33308102656.0,
+ "8615": 33307342848.0,
+ "8620": 33306077184.0,
+ "8625": 33308153856.0,
+ "8630": 33307807744.0,
+ "8635": 33306734592.0,
+ "8640": 33307867136.0,
+ "8645": 33307129856.0,
+ "8650": 33307430912.0,
+ "8655": 33307545600.0,
+ "8660": 33307975680.0,
+ "8665": 33307822080.0,
+ "8670": 33307156480.0,
+ "8675": 33307758592.0,
+ "8680": 33308340224.0,
+ "8685": 33307357184.0,
+ "8690": 33308479488.0,
+ "8695": 33306523648.0,
+ "8700": 33307404288.0,
+ "8705": 33307791360.0,
+ "8710": 33308004352.0,
+ "8715": 33308108800.0,
+ "8720": 33307424768.0,
+ "8725": 33307564032.0,
+ "8730": 33306877952.0,
+ "8735": 33307199488.0,
+ "8740": 33307734016.0,
+ "8745": 33307248640.0,
+ "8750": 33307912192.0,
+ "8755": 33307215872.0,
+ "8760": 33308012544.0,
+ "8765": 33306640384.0,
+ "8770": 33307977728.0,
+ "8775": 33306624000.0,
+ "8780": 33307357184.0,
+ "8785": 33306353664.0,
+ "8790": 33307518976.0,
+ "8795": 33308178432.0,
+ "8800": 33307113472.0,
+ "8805": 33307045888.0,
+ "8810": 33307252736.0,
+ "8815": 33307430912.0,
+ "8820": 33307568128.0,
+ "8825": 33306791936.0,
+ "8830": 33307529216.0,
+ "8835": 33306691584.0,
+ "8840": 33306529792.0,
+ "8845": 33307303936.0,
+ "8850": 33307901952.0,
+ "8855": 33308196864.0,
+ "8860": 33307965440.0,
+ "8865": 33307971584.0,
+ "8870": 33306595328.0,
+ "8875": 33306419200.0,
+ "8880": 33307508736.0,
+ "8885": 33306345472.0,
+ "8890": 33307373568.0,
+ "8895": 33307631616.0,
+ "8900": 33307330560.0,
+ "8905": 33308209152.0,
+ "8910": 33308155904.0,
+ "8915": 33306943488.0,
+ "8920": 33307381760.0,
+ "8925": 33307437056.0,
+ "8930": 33308041216.0,
+ "8935": 33307142144.0,
+ "8940": 33307768832.0,
+ "8945": 33308551168.0,
+ "8950": 33307682816.0,
+ "8955": 33307656192.0,
+ "8960": 33307787264.0,
+ "8965": 33306220544.0,
+ "8970": 33307693056.0,
+ "8975": 33307529216.0,
+ "8980": 33307027456.0,
+ "8985": 33308442624.0,
+ "8990": 33307588608.0,
+ "8995": 33308315648.0,
+ "9000": 33307787264.0,
+ "9005": 33307951104.0,
+ "9010": 33305649152.0,
+ "9015": 33307592704.0,
+ "9020": 33307033600.0,
+ "9025": 33307232256.0,
+ "9030": 33307793408.0,
+ "9035": 33307385856.0,
+ "9040": 33308012544.0,
+ "9045": 33307287552.0,
+ "9050": 33307701248.0,
+ "9055": 33306814464.0,
+ "9060": 33307975680.0,
+ "9065": 33307693056.0,
+ "9070": 33306888192.0,
+ "9075": 33307168768.0,
+ "9080": 33306818560.0,
+ "9085": 33307557888.0,
+ "9090": 33308200960.0,
+ "9095": 33306867712.0,
+ "9100": 33308563456.0,
+ "9105": 33306994688.0,
+ "9110": 33307004928.0,
+ "9115": 33307439104.0,
+ "9120": 33307340800.0,
+ "9125": 33307295744.0,
+ "9130": 33306771456.0,
+ "9135": 33307031552.0,
+ "9140": 33306497024.0,
+ "9145": 33307629568.0,
+ "9150": 33308002304.0,
+ "9155": 33307484160.0,
+ "9160": 33308100608.0,
+ "9165": 33307611136.0,
+ "9170": 33307897856.0,
+ "9175": 33307473920.0,
+ "9180": 33307977728.0,
+ "9185": 33307203584.0,
+ "9190": 33306693632.0,
+ "9195": 33306931200.0,
+ "9200": 33307779072.0,
+ "9205": 33307205632.0,
+ "9210": 33307637760.0,
+ "9215": 33307090944.0,
+ "9220": 33308454912.0,
+ "9225": 33307471872.0,
+ "9230": 33307322368.0,
+ "9235": 33307422720.0,
+ "9240": 33307242496.0,
+ "9245": 33308026880.0,
+ "9250": 33308203008.0,
+ "9255": 33307389952.0,
+ "9260": 33308825600.0,
+ "9265": 33306505216.0,
+ "9270": 33307426816.0,
+ "9275": 33307865088.0,
+ "9280": 33307435008.0,
+ "9285": 33307258880.0,
+ "9290": 33308000256.0,
+ "9295": 33307498496.0,
+ "9300": 33307301888.0,
+ "9305": 33307674624.0,
+ "9310": 33307031552.0,
+ "9315": 33306327040.0,
+ "9320": 33306834944.0,
+ "9325": 33307971584.0,
+ "9330": 33307910144.0,
+ "9335": 33307213824.0,
+ "9340": 33307385856.0,
+ "9345": 33307385856.0,
+ "9350": 33308127232.0,
+ "9355": 33306615808.0,
+ "9360": 33306697728.0,
+ "9365": 33307463680.0,
+ "9370": 33306355712.0,
+ "9375": 33307219968.0,
+ "9380": 33307224064.0,
+ "9385": 33308024832.0,
+ "9390": 33307830272.0,
+ "9395": 33307535360.0,
+ "9400": 33307031552.0,
+ "9405": 33307418624.0,
+ "9410": 33306822656.0,
+ "9415": 33307267072.0,
+ "9420": 33306994688.0,
+ "9425": 33306892288.0,
+ "9430": 33307199488.0,
+ "9435": 33306980352.0,
+ "9440": 33306451968.0,
+ "9445": 33308420096.0,
+ "9450": 33306755072.0,
+ "9455": 33306341376.0,
+ "9460": 33308131328.0,
+ "9465": 33307023360.0,
+ "9470": 33308307456.0,
+ "9475": 33308221440.0,
+ "9480": 33308037120.0,
+ "9485": 33308055552.0,
+ "9490": 33307908096.0,
+ "9495": 33306486784.0,
+ "9500": 33306490880.0,
+ "9505": 33307967488.0,
+ "9510": 33307125760.0,
+ "9515": 33307242496.0,
+ "9520": 33307670528.0,
+ "9525": 33307496448.0,
+ "9530": 33307731968.0,
+ "9535": 33307435008.0
+ }
+ },
+ "mem-max-allocated-bytes": {
+ "start_step": 1,
+ "end_step": 9535,
+ "step_interval": 5,
+ "values": {
+ "1": 36905754624.0,
+ "5": 45014786048.0,
+ "10": 45173362688.0,
+ "15": 45173362688.0,
+ "20": 45251878912.0,
+ "25": 45286207488.0,
+ "30": 45286207488.0,
+ "35": 45288939520.0,
+ "40": 45288939520.0,
+ "45": 45288939520.0,
+ "50": 45288939520.0,
+ "55": 45288939520.0,
+ "60": 45288939520.0,
+ "65": 45288939520.0,
+ "70": 45288939520.0,
+ "75": 45288939520.0,
+ "80": 45288939520.0,
+ "85": 45288939520.0,
+ "90": 45288939520.0,
+ "95": 45288939520.0,
+ "100": 45288939520.0,
+ "105": 45288939520.0,
+ "110": 45299392512.0,
+ "115": 45314936832.0,
+ "120": 45378736128.0,
+ "125": 45428596736.0,
+ "130": 45428596736.0,
+ "135": 45445640192.0,
+ "140": 45445640192.0,
+ "145": 45445640192.0,
+ "150": 45445640192.0,
+ "155": 45445640192.0,
+ "160": 45445640192.0,
+ "165": 45445640192.0,
+ "170": 45445640192.0,
+ "175": 45445640192.0,
+ "180": 45445640192.0,
+ "185": 45445640192.0,
+ "190": 45445640192.0,
+ "195": 45445640192.0,
+ "200": 45536641024.0,
+ "205": 45638885376.0,
+ "210": 45638885376.0,
+ "215": 45638885376.0,
+ "220": 45638885376.0,
+ "225": 45638885376.0,
+ "230": 45638885376.0,
+ "235": 45713887232.0,
+ "240": 45932376064.0,
+ "245": 45982269440.0,
+ "250": 45982269440.0,
+ "255": 45982269440.0,
+ "260": 46039670784.0,
+ "265": 46039670784.0,
+ "270": 46039670784.0,
+ "275": 46039670784.0,
+ "280": 46293884928.0,
+ "285": 46293884928.0,
+ "290": 46293884928.0,
+ "295": 46293884928.0,
+ "300": 46293884928.0,
+ "305": 46319267840.0,
+ "310": 46319267840.0,
+ "315": 46319267840.0,
+ "320": 46319267840.0,
+ "325": 46319267840.0,
+ "330": 46319267840.0,
+ "335": 46319267840.0,
+ "340": 46319267840.0,
+ "345": 46451261440.0,
+ "350": 46451261440.0,
+ "355": 46451261440.0,
+ "360": 46451261440.0,
+ "365": 46451261440.0,
+ "370": 46451261440.0,
+ "375": 46451261440.0,
+ "380": 46451261440.0,
+ "385": 46451261440.0,
+ "390": 46451261440.0,
+ "395": 46451261440.0,
+ "400": 46451261440.0,
+ "405": 46451261440.0,
+ "410": 46451261440.0,
+ "415": 46451261440.0,
+ "420": 46451261440.0,
+ "425": 46451261440.0,
+ "430": 46451261440.0,
+ "435": 46451261440.0,
+ "440": 46451261440.0,
+ "445": 46451261440.0,
+ "450": 46451261440.0,
+ "455": 46451261440.0,
+ "460": 46451261440.0,
+ "465": 46451261440.0,
+ "470": 46451261440.0,
+ "475": 46451261440.0,
+ "480": 46451261440.0,
+ "485": 46451261440.0,
+ "490": 46451261440.0,
+ "495": 46451261440.0,
+ "500": 46451261440.0,
+ "505": 46451261440.0,
+ "510": 46451261440.0,
+ "515": 46451261440.0,
+ "520": 46451261440.0,
+ "525": 46451261440.0,
+ "530": 46451261440.0,
+ "535": 46451261440.0,
+ "540": 46451261440.0,
+ "545": 46451261440.0,
+ "550": 46451261440.0,
+ "555": 46451261440.0,
+ "560": 46451261440.0,
+ "565": 46451261440.0,
+ "570": 46451261440.0,
+ "575": 46451261440.0,
+ "580": 46451261440.0,
+ "585": 46451261440.0,
+ "590": 46451261440.0,
+ "595": 46451261440.0,
+ "600": 46451261440.0,
+ "605": 46451261440.0,
+ "610": 46451261440.0,
+ "615": 46451261440.0,
+ "620": 46451261440.0,
+ "625": 46451261440.0,
+ "630": 46451261440.0,
+ "635": 46451261440.0,
+ "640": 46451261440.0,
+ "645": 46451261440.0,
+ "650": 46451261440.0,
+ "655": 46451261440.0,
+ "660": 46451261440.0,
+ "665": 46451261440.0,
+ "670": 46451261440.0,
+ "675": 46451261440.0,
+ "680": 46451261440.0,
+ "685": 46451261440.0,
+ "690": 46451261440.0,
+ "695": 46451261440.0,
+ "700": 46451261440.0,
+ "705": 46451261440.0,
+ "710": 46451261440.0,
+ "715": 46451261440.0,
+ "720": 46451261440.0,
+ "725": 46451261440.0,
+ "730": 46451261440.0,
+ "735": 46451261440.0,
+ "740": 46451261440.0,
+ "745": 46451261440.0,
+ "750": 46451261440.0,
+ "755": 46451261440.0,
+ "760": 46451261440.0,
+ "765": 46451261440.0,
+ "770": 46451261440.0,
+ "775": 46451261440.0,
+ "780": 46451261440.0,
+ "785": 46451261440.0,
+ "790": 46451261440.0,
+ "795": 46451261440.0,
+ "800": 46451261440.0,
+ "805": 46451261440.0,
+ "810": 46451261440.0,
+ "815": 46451261440.0,
+ "820": 46451261440.0,
+ "825": 46451261440.0,
+ "830": 46451261440.0,
+ "835": 46451261440.0,
+ "840": 46451261440.0,
+ "845": 46451261440.0,
+ "850": 46451261440.0,
+ "855": 46451261440.0,
+ "860": 46451261440.0,
+ "865": 46451261440.0,
+ "870": 46451261440.0,
+ "875": 46451261440.0,
+ "880": 46451261440.0,
+ "885": 46451261440.0,
+ "890": 46451261440.0,
+ "895": 46451261440.0,
+ "900": 46451261440.0,
+ "905": 46451261440.0,
+ "910": 46451261440.0,
+ "915": 46451261440.0,
+ "920": 46451261440.0,
+ "925": 46451261440.0,
+ "930": 46451261440.0,
+ "935": 46451261440.0,
+ "940": 46451261440.0,
+ "945": 46451261440.0,
+ "950": 46451261440.0,
+ "955": 46451261440.0,
+ "960": 45564735488.0,
+ "965": 45952081920.0,
+ "970": 45952081920.0,
+ "975": 46005657600.0,
+ "980": 46005657600.0,
+ "985": 46005657600.0,
+ "990": 46005657600.0,
+ "995": 46169923584.0,
+ "1000": 46169923584.0,
+ "1005": 46169923584.0,
+ "1010": 46169923584.0,
+ "1015": 46169923584.0,
+ "1020": 46169923584.0,
+ "1025": 46169923584.0,
+ "1030": 46169923584.0,
+ "1035": 46169923584.0,
+ "1040": 46169923584.0,
+ "1045": 46169923584.0,
+ "1050": 46169923584.0,
+ "1055": 46169923584.0,
+ "1060": 46169923584.0,
+ "1065": 46169923584.0,
+ "1070": 46169923584.0,
+ "1075": 46169923584.0,
+ "1080": 46169923584.0,
+ "1085": 46169923584.0,
+ "1090": 46169923584.0,
+ "1095": 46169923584.0,
+ "1100": 46169923584.0,
+ "1105": 46169923584.0,
+ "1110": 46169923584.0,
+ "1115": 46169923584.0,
+ "1120": 46169923584.0,
+ "1125": 46169923584.0,
+ "1130": 46169923584.0,
+ "1135": 46169923584.0,
+ "1140": 46169923584.0,
+ "1145": 46169923584.0,
+ "1150": 46169923584.0,
+ "1155": 46169923584.0,
+ "1160": 46169923584.0,
+ "1165": 46169923584.0,
+ "1170": 46169923584.0,
+ "1175": 46169923584.0,
+ "1180": 46192005120.0,
+ "1185": 46192005120.0,
+ "1190": 46192005120.0,
+ "1195": 46192005120.0,
+ "1200": 46192005120.0,
+ "1205": 46192005120.0,
+ "1210": 46192005120.0,
+ "1215": 46192005120.0,
+ "1220": 46192005120.0,
+ "1225": 46192005120.0,
+ "1230": 46192005120.0,
+ "1235": 46192005120.0,
+ "1240": 46192005120.0,
+ "1245": 46192005120.0,
+ "1250": 46192005120.0,
+ "1255": 46192005120.0,
+ "1260": 46192005120.0,
+ "1265": 46192005120.0,
+ "1270": 46192005120.0,
+ "1275": 46192005120.0,
+ "1280": 46192005120.0,
+ "1285": 46192005120.0,
+ "1290": 46192005120.0,
+ "1295": 46192005120.0,
+ "1300": 46192005120.0,
+ "1305": 46192005120.0,
+ "1310": 46192005120.0,
+ "1315": 46192005120.0,
+ "1320": 46192005120.0,
+ "1325": 46192005120.0,
+ "1330": 46192005120.0,
+ "1335": 46192005120.0,
+ "1340": 46192005120.0,
+ "1345": 46192005120.0,
+ "1350": 46192005120.0,
+ "1355": 46192005120.0,
+ "1360": 46192005120.0,
+ "1365": 46192005120.0,
+ "1370": 46192005120.0,
+ "1375": 46192005120.0,
+ "1380": 46192005120.0,
+ "1385": 46192005120.0,
+ "1390": 46192005120.0,
+ "1395": 46192005120.0,
+ "1400": 46192005120.0,
+ "1405": 46192005120.0,
+ "1410": 46192005120.0,
+ "1415": 46192005120.0,
+ "1420": 46192005120.0,
+ "1425": 46192005120.0,
+ "1430": 46192005120.0,
+ "1435": 46192005120.0,
+ "1440": 46192005120.0,
+ "1445": 46192005120.0,
+ "1450": 46192005120.0,
+ "1455": 46192005120.0,
+ "1460": 46192005120.0,
+ "1465": 46192005120.0,
+ "1470": 46192005120.0,
+ "1475": 46192005120.0,
+ "1480": 46192005120.0,
+ "1485": 46192005120.0,
+ "1490": 46192005120.0,
+ "1495": 46192005120.0,
+ "1500": 46192005120.0,
+ "1505": 46192005120.0,
+ "1510": 46192005120.0,
+ "1515": 46192005120.0,
+ "1520": 46192005120.0,
+ "1525": 46192005120.0,
+ "1530": 46192005120.0,
+ "1535": 46192005120.0,
+ "1540": 46192005120.0,
+ "1545": 46192005120.0,
+ "1550": 46260322304.0,
+ "1555": 46260322304.0,
+ "1560": 46260322304.0,
+ "1565": 46260322304.0,
+ "1570": 46260322304.0,
+ "1575": 46260322304.0,
+ "1580": 46260322304.0,
+ "1585": 46260322304.0,
+ "1590": 46260322304.0,
+ "1595": 46260322304.0,
+ "1600": 46260322304.0,
+ "1605": 46260322304.0,
+ "1610": 46260322304.0,
+ "1615": 46260322304.0,
+ "1620": 46260322304.0,
+ "1625": 46260322304.0,
+ "1630": 46260322304.0,
+ "1635": 46260322304.0,
+ "1640": 46260322304.0,
+ "1645": 46260322304.0,
+ "1650": 46260322304.0,
+ "1655": 46260322304.0,
+ "1660": 46260322304.0,
+ "1665": 46260322304.0,
+ "1670": 46260322304.0,
+ "1675": 46260322304.0,
+ "1680": 46260322304.0,
+ "1685": 46260322304.0,
+ "1690": 46260322304.0,
+ "1695": 46260322304.0,
+ "1700": 46260322304.0,
+ "1705": 46260322304.0,
+ "1710": 46260322304.0,
+ "1715": 46260322304.0,
+ "1720": 46260322304.0,
+ "1725": 46260322304.0,
+ "1730": 46260322304.0,
+ "1735": 46260322304.0,
+ "1740": 46260322304.0,
+ "1745": 46260322304.0,
+ "1750": 46260322304.0,
+ "1755": 46260322304.0,
+ "1760": 46260322304.0,
+ "1765": 46260322304.0,
+ "1770": 46260322304.0,
+ "1775": 46260322304.0,
+ "1780": 46260322304.0,
+ "1785": 46260322304.0,
+ "1790": 46260322304.0,
+ "1795": 46260322304.0,
+ "1800": 46260322304.0,
+ "1805": 46260322304.0,
+ "1810": 46260322304.0,
+ "1815": 46260322304.0,
+ "1820": 46260322304.0,
+ "1825": 46260322304.0,
+ "1830": 46260322304.0,
+ "1835": 46260322304.0,
+ "1840": 46260322304.0,
+ "1845": 46260322304.0,
+ "1850": 46260322304.0,
+ "1855": 46260322304.0,
+ "1860": 46260322304.0,
+ "1865": 46260322304.0,
+ "1870": 46260322304.0,
+ "1875": 46260322304.0,
+ "1880": 46260322304.0,
+ "1885": 46260322304.0,
+ "1890": 46260322304.0,
+ "1895": 46260322304.0,
+ "1900": 46260322304.0,
+ "1905": 46260322304.0,
+ "1910": 46260322304.0,
+ "1915": 46260322304.0,
+ "1920": 46260322304.0,
+ "1925": 46260322304.0,
+ "1930": 46260322304.0,
+ "1935": 46260322304.0,
+ "1940": 46260322304.0,
+ "1945": 46260322304.0,
+ "1950": 46260322304.0,
+ "1955": 46260322304.0,
+ "1960": 46260322304.0,
+ "1965": 46260322304.0,
+ "1970": 46260322304.0,
+ "1975": 46261714944.0,
+ "1980": 46261714944.0,
+ "1985": 46261714944.0,
+ "1990": 46261714944.0,
+ "1995": 46261714944.0,
+ "2000": 46261714944.0,
+ "2005": 46261714944.0,
+ "2010": 46261714944.0,
+ "2015": 46261714944.0,
+ "2020": 46261714944.0,
+ "2025": 46261714944.0,
+ "2030": 46261714944.0,
+ "2035": 46261714944.0,
+ "2040": 46261714944.0,
+ "2045": 46261714944.0,
+ "2050": 46261714944.0,
+ "2055": 46261714944.0,
+ "2060": 46261714944.0,
+ "2065": 46261714944.0,
+ "2070": 46261714944.0,
+ "2075": 46261714944.0,
+ "2080": 46261714944.0,
+ "2085": 46261714944.0,
+ "2090": 46261714944.0,
+ "2095": 46261714944.0,
+ "2100": 46261714944.0,
+ "2105": 46261714944.0,
+ "2110": 46261714944.0,
+ "2115": 46261714944.0,
+ "2120": 46261714944.0,
+ "2125": 46261714944.0,
+ "2130": 46261714944.0,
+ "2135": 46261714944.0,
+ "2140": 46261714944.0,
+ "2145": 46261714944.0,
+ "2150": 46261714944.0,
+ "2155": 46261714944.0,
+ "2160": 46261714944.0,
+ "2165": 46261714944.0,
+ "2170": 46261714944.0,
+ "2175": 46261714944.0,
+ "2180": 46261714944.0,
+ "2185": 46261714944.0,
+ "2190": 46261714944.0,
+ "2195": 46261714944.0,
+ "2200": 46261714944.0,
+ "2205": 46261714944.0,
+ "2210": 46261714944.0,
+ "2215": 46261714944.0,
+ "2220": 46261714944.0,
+ "2225": 46261714944.0,
+ "2230": 46261714944.0,
+ "2235": 46261714944.0,
+ "2240": 46261714944.0,
+ "2245": 46261714944.0,
+ "2250": 46261714944.0,
+ "2255": 46261714944.0,
+ "2260": 46261714944.0,
+ "2265": 46261714944.0,
+ "2270": 46261714944.0,
+ "2275": 46261714944.0,
+ "2280": 46261714944.0,
+ "2285": 46261714944.0,
+ "2290": 46261714944.0,
+ "2295": 46261714944.0,
+ "2300": 46261714944.0,
+ "2305": 46261714944.0,
+ "2310": 46261714944.0,
+ "2315": 46261714944.0,
+ "2320": 46261714944.0,
+ "2325": 46261714944.0,
+ "2330": 46261714944.0,
+ "2335": 46261714944.0,
+ "2340": 46261714944.0,
+ "2345": 46261714944.0,
+ "2350": 46261714944.0,
+ "2355": 46261714944.0,
+ "2360": 46261714944.0,
+ "2365": 46261714944.0,
+ "2370": 46261714944.0,
+ "2375": 46261714944.0,
+ "2380": 46261714944.0,
+ "2385": 46261714944.0,
+ "2390": 46261714944.0,
+ "2395": 46261714944.0,
+ "2400": 46261714944.0,
+ "2405": 46261714944.0,
+ "2410": 46261714944.0,
+ "2415": 46261714944.0,
+ "2420": 46261714944.0,
+ "2425": 46261714944.0,
+ "2430": 46261714944.0,
+ "2435": 46261714944.0,
+ "2440": 46261714944.0,
+ "2445": 46261714944.0,
+ "2450": 46261714944.0,
+ "2455": 46261714944.0,
+ "2460": 46261714944.0,
+ "2465": 46261714944.0,
+ "2470": 46261714944.0,
+ "2475": 46261714944.0,
+ "2480": 46261714944.0,
+ "2485": 46261714944.0,
+ "2490": 46261714944.0,
+ "2495": 46261714944.0,
+ "2500": 46261714944.0,
+ "2505": 46261714944.0,
+ "2510": 46261714944.0,
+ "2515": 46261714944.0,
+ "2520": 46261714944.0,
+ "2525": 46261714944.0,
+ "2530": 46261714944.0,
+ "2535": 46261714944.0,
+ "2540": 46261714944.0,
+ "2545": 46261714944.0,
+ "2550": 46261714944.0,
+ "2555": 46261714944.0,
+ "2560": 46261714944.0,
+ "2565": 46261714944.0,
+ "2570": 46261714944.0,
+ "2575": 46261714944.0,
+ "2580": 46261714944.0,
+ "2585": 46261714944.0,
+ "2590": 46261714944.0,
+ "2595": 46261714944.0,
+ "2600": 46261714944.0,
+ "2605": 46261714944.0,
+ "2610": 46261714944.0,
+ "2615": 46261714944.0,
+ "2620": 46261714944.0,
+ "2625": 46261714944.0,
+ "2630": 46261714944.0,
+ "2635": 46261714944.0,
+ "2640": 46261714944.0,
+ "2645": 46261714944.0,
+ "2650": 46261714944.0,
+ "2655": 46261714944.0,
+ "2660": 46261714944.0,
+ "2665": 46261714944.0,
+ "2670": 46261714944.0,
+ "2675": 46261714944.0,
+ "2680": 46261714944.0,
+ "2685": 46261714944.0,
+ "2690": 46261714944.0,
+ "2695": 46261714944.0,
+ "2700": 46261714944.0,
+ "2705": 46261714944.0,
+ "2710": 46261714944.0,
+ "2715": 46261714944.0,
+ "2720": 46261714944.0,
+ "2725": 46261714944.0,
+ "2730": 46261714944.0,
+ "2735": 46261714944.0,
+ "2740": 46261714944.0,
+ "2745": 46261714944.0,
+ "2750": 46261714944.0,
+ "2755": 46261714944.0,
+ "2760": 46261714944.0,
+ "2765": 46261714944.0,
+ "2770": 46261714944.0,
+ "2775": 46261714944.0,
+ "2780": 46261714944.0,
+ "2785": 46261714944.0,
+ "2790": 46261714944.0,
+ "2795": 46261714944.0,
+ "2800": 46261714944.0,
+ "2805": 46261714944.0,
+ "2810": 46261714944.0,
+ "2815": 46261714944.0,
+ "2820": 46261714944.0,
+ "2825": 46261714944.0,
+ "2830": 46261714944.0,
+ "2835": 46261714944.0,
+ "2840": 46261714944.0,
+ "2845": 46261714944.0,
+ "2850": 46261714944.0,
+ "2855": 46261714944.0,
+ "2860": 46261714944.0,
+ "2865": 46261714944.0,
+ "2870": 46261714944.0,
+ "2875": 46261714944.0,
+ "2880": 46261714944.0,
+ "2885": 46261714944.0,
+ "2890": 46261714944.0,
+ "2895": 46261714944.0,
+ "2900": 46261714944.0,
+ "2905": 46261714944.0,
+ "2910": 46261714944.0,
+ "2915": 46261714944.0,
+ "2920": 46261714944.0,
+ "2925": 46261714944.0,
+ "2930": 46261714944.0,
+ "2935": 46261714944.0,
+ "2940": 46261714944.0,
+ "2945": 46261714944.0,
+ "2950": 46261714944.0,
+ "2955": 46261714944.0,
+ "2960": 46261714944.0,
+ "2965": 46261714944.0,
+ "2970": 46261714944.0,
+ "2975": 46261714944.0,
+ "2980": 46261714944.0,
+ "2985": 45706711040.0,
+ "2990": 45883699200.0,
+ "2995": 46072287232.0,
+ "3000": 46072287232.0,
+ "3005": 46072287232.0,
+ "3010": 46072287232.0,
+ "3015": 46072287232.0,
+ "3020": 46072287232.0,
+ "3025": 46072287232.0,
+ "3030": 46072287232.0,
+ "3035": 46072287232.0,
+ "3040": 46072287232.0,
+ "3045": 46072287232.0,
+ "3050": 46072287232.0,
+ "3055": 46072287232.0,
+ "3060": 46072287232.0,
+ "3065": 46072287232.0,
+ "3070": 46072287232.0,
+ "3075": 46072287232.0,
+ "3080": 46072287232.0,
+ "3085": 46072287232.0,
+ "3090": 46072287232.0,
+ "3095": 46072287232.0,
+ "3100": 46072287232.0,
+ "3105": 46072287232.0,
+ "3110": 46072287232.0,
+ "3115": 46072287232.0,
+ "3120": 46072287232.0,
+ "3125": 46072287232.0,
+ "3130": 46072287232.0,
+ "3135": 46072287232.0,
+ "3140": 46072287232.0,
+ "3145": 46072287232.0,
+ "3150": 46072287232.0,
+ "3155": 46072287232.0,
+ "3160": 46072287232.0,
+ "3165": 46072287232.0,
+ "3170": 46072287232.0,
+ "3175": 46072287232.0,
+ "3180": 46072287232.0,
+ "3185": 46072287232.0,
+ "3190": 46072287232.0,
+ "3195": 46072287232.0,
+ "3200": 46072287232.0,
+ "3205": 46072287232.0,
+ "3210": 46072287232.0,
+ "3215": 46072287232.0,
+ "3220": 46072287232.0,
+ "3225": 46072287232.0,
+ "3230": 46072287232.0,
+ "3235": 46072287232.0,
+ "3240": 46072287232.0,
+ "3245": 46072287232.0,
+ "3250": 46072287232.0,
+ "3255": 46072287232.0,
+ "3260": 46072287232.0,
+ "3265": 46072287232.0,
+ "3270": 46072287232.0,
+ "3275": 46072287232.0,
+ "3280": 46072287232.0,
+ "3285": 46072287232.0,
+ "3290": 46072287232.0,
+ "3295": 46072287232.0,
+ "3300": 46072287232.0,
+ "3305": 46072287232.0,
+ "3310": 46072287232.0,
+ "3315": 46072287232.0,
+ "3320": 46072287232.0,
+ "3325": 46072287232.0,
+ "3330": 46072287232.0,
+ "3335": 46072287232.0,
+ "3340": 46072287232.0,
+ "3345": 46072287232.0,
+ "3350": 46072287232.0,
+ "3355": 46072287232.0,
+ "3360": 46072287232.0,
+ "3365": 46072287232.0,
+ "3370": 46072287232.0,
+ "3375": 46072287232.0,
+ "3380": 46072287232.0,
+ "3385": 46072287232.0,
+ "3390": 46072287232.0,
+ "3395": 46072287232.0,
+ "3400": 46072287232.0,
+ "3405": 46072287232.0,
+ "3410": 46072287232.0,
+ "3415": 46072287232.0,
+ "3420": 46072287232.0,
+ "3425": 46072672256.0,
+ "3430": 46072672256.0,
+ "3435": 46072672256.0,
+ "3440": 46072672256.0,
+ "3445": 46072672256.0,
+ "3450": 46072672256.0,
+ "3455": 46072672256.0,
+ "3460": 46072672256.0,
+ "3465": 46072672256.0,
+ "3470": 46072672256.0,
+ "3475": 46072672256.0,
+ "3480": 46072672256.0,
+ "3485": 46095564800.0,
+ "3490": 46095564800.0,
+ "3495": 46095564800.0,
+ "3500": 46095564800.0,
+ "3505": 46095564800.0,
+ "3510": 46095564800.0,
+ "3515": 46095564800.0,
+ "3520": 46095564800.0,
+ "3525": 46095564800.0,
+ "3530": 46095564800.0,
+ "3535": 46095564800.0,
+ "3540": 46095564800.0,
+ "3545": 46095564800.0,
+ "3550": 46191697920.0,
+ "3555": 46191697920.0,
+ "3560": 46191697920.0,
+ "3565": 46191697920.0,
+ "3570": 46191697920.0,
+ "3575": 46191697920.0,
+ "3580": 46191697920.0,
+ "3585": 46191697920.0,
+ "3590": 46191697920.0,
+ "3595": 46191697920.0,
+ "3600": 46191697920.0,
+ "3605": 46191697920.0,
+ "3610": 46191697920.0,
+ "3615": 46191697920.0,
+ "3620": 46191697920.0,
+ "3625": 46191697920.0,
+ "3630": 46191697920.0,
+ "3635": 46191697920.0,
+ "3640": 46191697920.0,
+ "3645": 46191697920.0,
+ "3650": 46191697920.0,
+ "3655": 46191697920.0,
+ "3660": 46191697920.0,
+ "3665": 46191697920.0,
+ "3670": 46191697920.0,
+ "3675": 46191697920.0,
+ "3680": 46191697920.0,
+ "3685": 46191697920.0,
+ "3690": 46191697920.0,
+ "3695": 46191697920.0,
+ "3700": 46191697920.0,
+ "3705": 46191697920.0,
+ "3710": 46191697920.0,
+ "3715": 46191697920.0,
+ "3720": 46191697920.0,
+ "3725": 46191697920.0,
+ "3730": 46191697920.0,
+ "3735": 46191697920.0,
+ "3740": 46191697920.0,
+ "3745": 46191697920.0,
+ "3750": 46191697920.0,
+ "3755": 46191697920.0,
+ "3760": 46191697920.0,
+ "3765": 46191697920.0,
+ "3770": 46191697920.0,
+ "3775": 46191697920.0,
+ "3780": 46191697920.0,
+ "3785": 46191697920.0,
+ "3790": 46191697920.0,
+ "3795": 46191697920.0,
+ "3800": 46191697920.0,
+ "3805": 46191697920.0,
+ "3810": 46191697920.0,
+ "3815": 46191697920.0,
+ "3820": 46191697920.0,
+ "3825": 46191697920.0,
+ "3830": 46191697920.0,
+ "3835": 46191697920.0,
+ "3840": 46191697920.0,
+ "3845": 46191697920.0,
+ "3850": 46191697920.0,
+ "3855": 46191697920.0,
+ "3860": 46191697920.0,
+ "3865": 46191697920.0,
+ "3870": 46191697920.0,
+ "3875": 46191697920.0,
+ "3880": 46191697920.0,
+ "3885": 46191697920.0,
+ "3890": 46191697920.0,
+ "3895": 46191697920.0,
+ "3900": 46191697920.0,
+ "3905": 46191697920.0,
+ "3910": 46191697920.0,
+ "3915": 46191697920.0,
+ "3920": 46191697920.0,
+ "3925": 46191697920.0,
+ "3930": 46191697920.0,
+ "3935": 46191697920.0,
+ "3940": 46191697920.0,
+ "3945": 46191697920.0,
+ "3950": 46191697920.0,
+ "3955": 46191697920.0,
+ "3960": 46191697920.0,
+ "3965": 46191697920.0,
+ "3970": 46191697920.0,
+ "3975": 46191697920.0,
+ "3980": 46191697920.0,
+ "3985": 46191697920.0,
+ "3990": 46191697920.0,
+ "3995": 46191697920.0,
+ "4000": 45840449536.0,
+ "4005": 45869191168.0,
+ "4010": 45897973760.0,
+ "4015": 45897973760.0,
+ "4020": 45940301824.0,
+ "4025": 45940301824.0,
+ "4030": 45940301824.0,
+ "4035": 45940301824.0,
+ "4040": 45940301824.0,
+ "4045": 45940301824.0,
+ "4050": 45940301824.0,
+ "4055": 45940301824.0,
+ "4060": 45940301824.0,
+ "4065": 45940301824.0,
+ "4070": 45940301824.0,
+ "4075": 45940301824.0,
+ "4080": 45940301824.0,
+ "4085": 46009651200.0,
+ "4090": 46009651200.0,
+ "4095": 46009651200.0,
+ "4100": 46009651200.0,
+ "4105": 46009651200.0,
+ "4110": 46009651200.0,
+ "4115": 46009651200.0,
+ "4120": 46009651200.0,
+ "4125": 46009651200.0,
+ "4130": 46009651200.0,
+ "4135": 46009651200.0,
+ "4140": 46009651200.0,
+ "4145": 46009651200.0,
+ "4150": 46009651200.0,
+ "4155": 46009651200.0,
+ "4160": 46009651200.0,
+ "4165": 46009651200.0,
+ "4170": 46009651200.0,
+ "4175": 46009651200.0,
+ "4180": 46009651200.0,
+ "4185": 46009651200.0,
+ "4190": 46009651200.0,
+ "4195": 46009651200.0,
+ "4200": 46009651200.0,
+ "4205": 46009651200.0,
+ "4210": 46009651200.0,
+ "4215": 46009651200.0,
+ "4220": 46009651200.0,
+ "4225": 46064635904.0,
+ "4230": 46064635904.0,
+ "4235": 46064635904.0,
+ "4240": 46064635904.0,
+ "4245": 46064635904.0,
+ "4250": 46064635904.0,
+ "4255": 46064635904.0,
+ "4260": 46064635904.0,
+ "4265": 46064635904.0,
+ "4270": 46064635904.0,
+ "4275": 46064635904.0,
+ "4280": 46064635904.0,
+ "4285": 46064635904.0,
+ "4290": 46064635904.0,
+ "4295": 46064635904.0,
+ "4300": 46064635904.0,
+ "4305": 46064635904.0,
+ "4310": 46064635904.0,
+ "4315": 46064635904.0,
+ "4320": 46064635904.0,
+ "4325": 46064635904.0,
+ "4330": 46064635904.0,
+ "4335": 46064635904.0,
+ "4340": 46064635904.0,
+ "4345": 46064635904.0,
+ "4350": 46064635904.0,
+ "4355": 46064635904.0,
+ "4360": 46064635904.0,
+ "4365": 46064635904.0,
+ "4370": 46064635904.0,
+ "4375": 46064635904.0,
+ "4380": 46064635904.0,
+ "4385": 46064635904.0,
+ "4390": 46064635904.0,
+ "4395": 46064635904.0,
+ "4400": 46064635904.0,
+ "4405": 46064635904.0,
+ "4410": 46064635904.0,
+ "4415": 46064635904.0,
+ "4420": 46064635904.0,
+ "4425": 46064635904.0,
+ "4430": 46064635904.0,
+ "4435": 46064635904.0,
+ "4440": 46064635904.0,
+ "4445": 46064635904.0,
+ "4450": 46064635904.0,
+ "4455": 46064635904.0,
+ "4460": 46080573440.0,
+ "4465": 46080573440.0,
+ "4470": 46080573440.0,
+ "4475": 46080573440.0,
+ "4480": 46080573440.0,
+ "4485": 46080573440.0,
+ "4490": 46080573440.0,
+ "4495": 46080573440.0,
+ "4500": 46080573440.0,
+ "4505": 46080573440.0,
+ "4510": 46080573440.0,
+ "4515": 46080573440.0,
+ "4520": 46080573440.0,
+ "4525": 46080573440.0,
+ "4530": 46080573440.0,
+ "4535": 46080573440.0,
+ "4540": 46080573440.0,
+ "4545": 46080573440.0,
+ "4550": 46080573440.0,
+ "4555": 46080573440.0,
+ "4560": 46080573440.0,
+ "4565": 46080573440.0,
+ "4570": 46080573440.0,
+ "4575": 46080573440.0,
+ "4580": 46080573440.0,
+ "4585": 46080573440.0,
+ "4590": 46080573440.0,
+ "4595": 46080573440.0,
+ "4600": 46080573440.0,
+ "4605": 46080573440.0,
+ "4610": 46080573440.0,
+ "4615": 46343888896.0,
+ "4620": 46343888896.0,
+ "4625": 46343888896.0,
+ "4630": 46343888896.0,
+ "4635": 46343888896.0,
+ "4640": 46343888896.0,
+ "4645": 46343888896.0,
+ "4650": 46343888896.0,
+ "4655": 46343888896.0,
+ "4660": 46343888896.0,
+ "4665": 46343888896.0,
+ "4670": 46343888896.0,
+ "4675": 46343888896.0,
+ "4680": 46343888896.0,
+ "4685": 46343888896.0,
+ "4690": 46343888896.0,
+ "4695": 46343888896.0,
+ "4700": 46343888896.0,
+ "4705": 46343888896.0,
+ "4710": 46343888896.0,
+ "4715": 46343888896.0,
+ "4720": 46343888896.0,
+ "4725": 46343888896.0,
+ "4730": 46343888896.0,
+ "4735": 46343888896.0,
+ "4740": 46343888896.0,
+ "4745": 46343888896.0,
+ "4750": 46343888896.0,
+ "4755": 46343888896.0,
+ "4760": 46343888896.0,
+ "4765": 46343888896.0,
+ "4770": 46343888896.0,
+ "4775": 46343888896.0,
+ "4780": 46343888896.0,
+ "4785": 46343888896.0,
+ "4790": 46343888896.0,
+ "4795": 46343888896.0,
+ "4800": 46343888896.0,
+ "4805": 46343888896.0,
+ "4810": 46343888896.0,
+ "4815": 46343888896.0,
+ "4820": 46343888896.0,
+ "4825": 46343888896.0,
+ "4830": 46343888896.0,
+ "4835": 46343888896.0,
+ "4840": 46343888896.0,
+ "4845": 46343888896.0,
+ "4850": 46343888896.0,
+ "4855": 46343888896.0,
+ "4860": 46343888896.0,
+ "4865": 46343888896.0,
+ "4870": 46343888896.0,
+ "4875": 46343888896.0,
+ "4880": 46343888896.0,
+ "4885": 46343888896.0,
+ "4890": 46343888896.0,
+ "4895": 46343888896.0,
+ "4900": 46343888896.0,
+ "4905": 46343888896.0,
+ "4910": 46343888896.0,
+ "4915": 46343888896.0,
+ "4920": 46343888896.0,
+ "4925": 46343888896.0,
+ "4930": 46343888896.0,
+ "4935": 46343888896.0,
+ "4940": 46343888896.0,
+ "4945": 46343888896.0,
+ "4950": 46343888896.0,
+ "4955": 46343888896.0,
+ "4960": 46343888896.0,
+ "4965": 46343888896.0,
+ "4970": 46343888896.0,
+ "4975": 46343888896.0,
+ "4980": 46343888896.0,
+ "4985": 46343888896.0,
+ "4990": 46343888896.0,
+ "4995": 46343888896.0,
+ "5000": 46343888896.0,
+ "5005": 46199529472.0,
+ "5010": 46199529472.0,
+ "5015": 45764182016.0,
+ "5020": 45878784000.0,
+ "5025": 45878784000.0,
+ "5030": 45878784000.0,
+ "5035": 45878784000.0,
+ "5040": 45992685568.0,
+ "5045": 45992685568.0,
+ "5050": 45992685568.0,
+ "5055": 45992685568.0,
+ "5060": 45992685568.0,
+ "5065": 45992685568.0,
+ "5070": 45992685568.0,
+ "5075": 45992685568.0,
+ "5080": 45992685568.0,
+ "5085": 45992685568.0,
+ "5090": 45992685568.0,
+ "5095": 46014451712.0,
+ "5100": 46014451712.0,
+ "5105": 46014451712.0,
+ "5110": 46014451712.0,
+ "5115": 46014451712.0,
+ "5120": 46014451712.0,
+ "5125": 46014451712.0,
+ "5130": 46014451712.0,
+ "5135": 46014451712.0,
+ "5140": 46014451712.0,
+ "5145": 46014451712.0,
+ "5150": 46014451712.0,
+ "5155": 46014451712.0,
+ "5160": 46014451712.0,
+ "5165": 46014451712.0,
+ "5170": 46014451712.0,
+ "5175": 46014451712.0,
+ "5180": 46014451712.0,
+ "5185": 46014451712.0,
+ "5190": 46014451712.0,
+ "5195": 46014451712.0,
+ "5200": 46139572224.0,
+ "5205": 46139572224.0,
+ "5210": 46139572224.0,
+ "5215": 46139572224.0,
+ "5220": 46168403968.0,
+ "5225": 46168403968.0,
+ "5230": 46168403968.0,
+ "5235": 46168403968.0,
+ "5240": 46168403968.0,
+ "5245": 46168403968.0,
+ "5250": 46168403968.0,
+ "5255": 46168403968.0,
+ "5260": 46168403968.0,
+ "5265": 46168403968.0,
+ "5270": 46168403968.0,
+ "5275": 46168403968.0,
+ "5280": 46168403968.0,
+ "5285": 46168403968.0,
+ "5290": 46168403968.0,
+ "5295": 46168403968.0,
+ "5300": 46168403968.0,
+ "5305": 46168403968.0,
+ "5310": 46168403968.0,
+ "5315": 46168403968.0,
+ "5320": 46168403968.0,
+ "5325": 46168403968.0,
+ "5330": 46168403968.0,
+ "5335": 46168403968.0,
+ "5340": 46168403968.0,
+ "5345": 46168403968.0,
+ "5350": 46168403968.0,
+ "5355": 46168403968.0,
+ "5360": 46168403968.0,
+ "5365": 46168403968.0,
+ "5370": 46168403968.0,
+ "5375": 46168403968.0,
+ "5380": 46168403968.0,
+ "5385": 46168403968.0,
+ "5390": 46168403968.0,
+ "5395": 46168403968.0,
+ "5400": 46168403968.0,
+ "5405": 46168403968.0,
+ "5410": 46168403968.0,
+ "5415": 46168403968.0,
+ "5420": 46168403968.0,
+ "5425": 46168403968.0,
+ "5430": 46168403968.0,
+ "5435": 46168403968.0,
+ "5440": 46168403968.0,
+ "5445": 46168403968.0,
+ "5450": 46168403968.0,
+ "5455": 46168403968.0,
+ "5460": 46168403968.0,
+ "5465": 46168403968.0,
+ "5470": 46168403968.0,
+ "5475": 46168403968.0,
+ "5480": 46168403968.0,
+ "5485": 46168403968.0,
+ "5490": 46168403968.0,
+ "5495": 46168403968.0,
+ "5500": 46168403968.0,
+ "5505": 46168403968.0,
+ "5510": 46168403968.0,
+ "5515": 46168403968.0,
+ "5520": 46168403968.0,
+ "5525": 46168403968.0,
+ "5530": 46168403968.0,
+ "5535": 46168403968.0,
+ "5540": 46168403968.0,
+ "5545": 46168403968.0,
+ "5550": 46168403968.0,
+ "5555": 46168403968.0,
+ "5560": 46168403968.0,
+ "5565": 46168403968.0,
+ "5570": 46168403968.0,
+ "5575": 46168403968.0,
+ "5580": 46168403968.0,
+ "5585": 46168403968.0,
+ "5590": 46168403968.0,
+ "5595": 46168403968.0,
+ "5600": 46168403968.0,
+ "5605": 46226247680.0,
+ "5610": 46226247680.0,
+ "5615": 46226247680.0,
+ "5620": 46226247680.0,
+ "5625": 46226247680.0,
+ "5630": 46226247680.0,
+ "5635": 46226247680.0,
+ "5640": 46226247680.0,
+ "5645": 46226247680.0,
+ "5650": 46226247680.0,
+ "5655": 46226247680.0,
+ "5660": 46226247680.0,
+ "5665": 46226247680.0,
+ "5670": 46226247680.0,
+ "5675": 46226247680.0,
+ "5680": 46226247680.0,
+ "5685": 46226247680.0,
+ "5690": 46226247680.0,
+ "5695": 46226247680.0,
+ "5700": 46226247680.0,
+ "5705": 46226247680.0,
+ "5710": 46226247680.0,
+ "5715": 46226247680.0,
+ "5720": 46226247680.0,
+ "5725": 46226247680.0,
+ "5730": 46226247680.0,
+ "5735": 46226247680.0,
+ "5740": 46226247680.0,
+ "5745": 46226247680.0,
+ "5750": 46226247680.0,
+ "5755": 46226247680.0,
+ "5760": 46226247680.0,
+ "5765": 46226247680.0,
+ "5770": 46226247680.0,
+ "5775": 46226247680.0,
+ "5780": 46226247680.0,
+ "5785": 46226247680.0,
+ "5790": 46226247680.0,
+ "5795": 46226247680.0,
+ "5800": 46226247680.0,
+ "5805": 46226247680.0,
+ "5810": 46226247680.0,
+ "5815": 46226247680.0,
+ "5820": 46226247680.0,
+ "5825": 46226247680.0,
+ "5830": 46226247680.0,
+ "5835": 46226247680.0,
+ "5840": 46226247680.0,
+ "5845": 46226247680.0,
+ "5850": 46226247680.0,
+ "5855": 46226247680.0,
+ "5860": 46226247680.0,
+ "5865": 46226247680.0,
+ "5870": 46226247680.0,
+ "5875": 46226247680.0,
+ "5880": 46226247680.0,
+ "5885": 46226247680.0,
+ "5890": 46226247680.0,
+ "5895": 46226247680.0,
+ "5900": 46226247680.0,
+ "5905": 46226247680.0,
+ "5910": 46226247680.0,
+ "5915": 46226247680.0,
+ "5920": 46226247680.0,
+ "5925": 46226247680.0,
+ "5930": 46226247680.0,
+ "5935": 46226247680.0,
+ "5940": 46226247680.0,
+ "5945": 46226247680.0,
+ "5950": 46226247680.0,
+ "5955": 46226247680.0,
+ "5960": 46226247680.0,
+ "5965": 46226247680.0,
+ "5970": 46226247680.0,
+ "5975": 46226247680.0,
+ "5980": 46226247680.0,
+ "5985": 46226247680.0,
+ "5990": 46226247680.0,
+ "5995": 46226247680.0,
+ "6000": 46226247680.0,
+ "6005": 46226247680.0,
+ "6010": 46226247680.0,
+ "6015": 46226247680.0,
+ "6020": 46226247680.0,
+ "6025": 46226247680.0,
+ "6030": 45912186880.0,
+ "6035": 45912186880.0,
+ "6040": 45995683840.0,
+ "6045": 45995683840.0,
+ "6050": 45995683840.0,
+ "6055": 45995683840.0,
+ "6060": 45995683840.0,
+ "6065": 45995683840.0,
+ "6070": 45995683840.0,
+ "6075": 46014836736.0,
+ "6080": 46014836736.0,
+ "6085": 46014836736.0,
+ "6090": 46014836736.0,
+ "6095": 46014836736.0,
+ "6100": 46014836736.0,
+ "6105": 46014836736.0,
+ "6110": 46014836736.0,
+ "6115": 46014836736.0,
+ "6120": 46014836736.0,
+ "6125": 46014836736.0,
+ "6130": 46014836736.0,
+ "6135": 46014836736.0,
+ "6140": 46014836736.0,
+ "6145": 46014836736.0,
+ "6150": 46014836736.0,
+ "6155": 46014836736.0,
+ "6160": 46014836736.0,
+ "6165": 46025334784.0,
+ "6170": 46025334784.0,
+ "6175": 46025334784.0,
+ "6180": 46025334784.0,
+ "6185": 46035255296.0,
+ "6190": 46035255296.0,
+ "6195": 46035255296.0,
+ "6200": 46035255296.0,
+ "6205": 46035255296.0,
+ "6210": 46035255296.0,
+ "6215": 46035255296.0,
+ "6220": 46035255296.0,
+ "6225": 46035255296.0,
+ "6230": 46035255296.0,
+ "6235": 46035255296.0,
+ "6240": 46035255296.0,
+ "6245": 46035255296.0,
+ "6250": 46035255296.0,
+ "6255": 46035255296.0,
+ "6260": 46035255296.0,
+ "6265": 46035255296.0,
+ "6270": 46035255296.0,
+ "6275": 46035255296.0,
+ "6280": 46035255296.0,
+ "6285": 46035255296.0,
+ "6290": 46035255296.0,
+ "6295": 46035255296.0,
+ "6300": 46035255296.0,
+ "6305": 46035255296.0,
+ "6310": 46035255296.0,
+ "6315": 46035255296.0,
+ "6320": 46035255296.0,
+ "6325": 46035255296.0,
+ "6330": 46035255296.0,
+ "6335": 46035255296.0,
+ "6340": 46035255296.0,
+ "6345": 46035255296.0,
+ "6350": 46035255296.0,
+ "6355": 46035255296.0,
+ "6360": 46035255296.0,
+ "6365": 46035255296.0,
+ "6370": 46035255296.0,
+ "6375": 46035255296.0,
+ "6380": 46035255296.0,
+ "6385": 46035255296.0,
+ "6390": 46035255296.0,
+ "6395": 46035255296.0,
+ "6400": 46035255296.0,
+ "6405": 46035255296.0,
+ "6410": 46035255296.0,
+ "6415": 46035255296.0,
+ "6420": 46035255296.0,
+ "6425": 46035255296.0,
+ "6430": 46035255296.0,
+ "6435": 46035255296.0,
+ "6440": 46035255296.0,
+ "6445": 46035255296.0,
+ "6450": 46035255296.0,
+ "6455": 46035255296.0,
+ "6460": 46035255296.0,
+ "6465": 46035255296.0,
+ "6470": 46035255296.0,
+ "6475": 46035255296.0,
+ "6480": 46035255296.0,
+ "6485": 46035255296.0,
+ "6490": 46035255296.0,
+ "6495": 46035255296.0,
+ "6500": 46035255296.0,
+ "6505": 46064041984.0,
+ "6510": 46064041984.0,
+ "6515": 46064041984.0,
+ "6520": 46064041984.0,
+ "6525": 46064041984.0,
+ "6530": 46064041984.0,
+ "6535": 46064041984.0,
+ "6540": 46064041984.0,
+ "6545": 46064041984.0,
+ "6550": 46064041984.0,
+ "6555": 46064041984.0,
+ "6560": 46064041984.0,
+ "6565": 46064041984.0,
+ "6570": 46064041984.0,
+ "6575": 46064041984.0,
+ "6580": 46064041984.0,
+ "6585": 46064041984.0,
+ "6590": 46064041984.0,
+ "6595": 46064041984.0,
+ "6600": 46064041984.0,
+ "6605": 46064041984.0,
+ "6610": 46064041984.0,
+ "6615": 46064041984.0,
+ "6620": 46064041984.0,
+ "6625": 46064041984.0,
+ "6630": 46064041984.0,
+ "6635": 46064041984.0,
+ "6640": 46064041984.0,
+ "6645": 46064041984.0,
+ "6650": 46064041984.0,
+ "6655": 46064041984.0,
+ "6660": 46064041984.0,
+ "6665": 46064041984.0,
+ "6670": 46064041984.0,
+ "6675": 46064041984.0,
+ "6680": 46064041984.0,
+ "6685": 46064041984.0,
+ "6690": 46064041984.0,
+ "6695": 46064041984.0,
+ "6700": 46064041984.0,
+ "6705": 46064041984.0,
+ "6710": 46064041984.0,
+ "6715": 46064041984.0,
+ "6720": 46064041984.0,
+ "6725": 46064041984.0,
+ "6730": 46064041984.0,
+ "6735": 46064041984.0,
+ "6740": 46064041984.0,
+ "6745": 46064041984.0,
+ "6750": 46064041984.0,
+ "6755": 46064041984.0,
+ "6760": 46064041984.0,
+ "6765": 46064041984.0,
+ "6770": 46064041984.0,
+ "6775": 46064041984.0,
+ "6780": 46064041984.0,
+ "6785": 46064041984.0,
+ "6790": 46064041984.0,
+ "6795": 46064041984.0,
+ "6800": 46064041984.0,
+ "6805": 46064041984.0,
+ "6810": 46064041984.0,
+ "6815": 46064041984.0,
+ "6820": 46064041984.0,
+ "6825": 46064041984.0,
+ "6830": 46064041984.0,
+ "6835": 46064041984.0,
+ "6840": 46064041984.0,
+ "6845": 46064041984.0,
+ "6850": 46064041984.0,
+ "6855": 46064041984.0,
+ "6860": 46064041984.0,
+ "6865": 46064041984.0,
+ "6870": 46064041984.0,
+ "6875": 46064041984.0,
+ "6880": 46064041984.0,
+ "6885": 46064041984.0,
+ "6890": 46064041984.0,
+ "6895": 46064041984.0,
+ "6900": 46064041984.0,
+ "6905": 46064041984.0,
+ "6910": 46064041984.0,
+ "6915": 46064041984.0,
+ "6920": 46064041984.0,
+ "6925": 46064041984.0,
+ "6930": 46064041984.0,
+ "6935": 46064041984.0,
+ "6940": 46064041984.0,
+ "6945": 46064041984.0,
+ "6950": 46064041984.0,
+ "6955": 46064041984.0,
+ "6960": 46064041984.0,
+ "6965": 46064041984.0,
+ "6970": 46064041984.0,
+ "6975": 46064041984.0,
+ "6980": 46064041984.0,
+ "6985": 46064041984.0,
+ "6990": 46064041984.0,
+ "6995": 46064041984.0,
+ "7000": 46064041984.0,
+ "7005": 46064041984.0,
+ "7010": 46064041984.0,
+ "7015": 46064041984.0,
+ "7020": 46064041984.0,
+ "7025": 46064041984.0,
+ "7030": 46108979200.0,
+ "7035": 46108979200.0,
+ "7040": 46108979200.0,
+ "7045": 46108979200.0,
+ "7050": 46065532928.0,
+ "7055": 46065532928.0,
+ "7060": 46065532928.0,
+ "7065": 46065532928.0,
+ "7070": 46065532928.0,
+ "7075": 46065532928.0,
+ "7080": 46065532928.0,
+ "7085": 46065532928.0,
+ "7090": 46065532928.0,
+ "7095": 46065532928.0,
+ "7100": 46065532928.0,
+ "7105": 46065532928.0,
+ "7110": 46065532928.0,
+ "7115": 46065532928.0,
+ "7120": 46065532928.0,
+ "7125": 46065532928.0,
+ "7130": 46065532928.0,
+ "7135": 46065532928.0,
+ "7140": 46065532928.0,
+ "7145": 46065532928.0,
+ "7150": 46065532928.0,
+ "7155": 46065532928.0,
+ "7160": 46065532928.0,
+ "7165": 46065532928.0,
+ "7170": 46065532928.0,
+ "7175": 46065532928.0,
+ "7180": 46065532928.0,
+ "7185": 46065532928.0,
+ "7190": 46065532928.0,
+ "7195": 46065532928.0,
+ "7200": 46065532928.0,
+ "7205": 46065532928.0,
+ "7210": 46065532928.0,
+ "7215": 46065532928.0,
+ "7220": 46065532928.0,
+ "7225": 46065532928.0,
+ "7230": 46065532928.0,
+ "7235": 46065532928.0,
+ "7240": 46065532928.0,
+ "7245": 46065532928.0,
+ "7250": 46065532928.0,
+ "7255": 46065532928.0,
+ "7260": 46065532928.0,
+ "7265": 46065532928.0,
+ "7270": 46065532928.0,
+ "7275": 46065532928.0,
+ "7280": 46065532928.0,
+ "7285": 46065532928.0,
+ "7290": 46065532928.0,
+ "7295": 46065532928.0,
+ "7300": 46065532928.0,
+ "7305": 46065532928.0,
+ "7310": 46065532928.0,
+ "7315": 46065532928.0,
+ "7320": 46065532928.0,
+ "7325": 46065532928.0,
+ "7330": 46065532928.0,
+ "7335": 46065532928.0,
+ "7340": 46065532928.0,
+ "7345": 46065532928.0,
+ "7350": 46065532928.0,
+ "7355": 46065532928.0,
+ "7360": 46065532928.0,
+ "7365": 46065532928.0,
+ "7370": 46065532928.0,
+ "7375": 46065532928.0,
+ "7380": 46065532928.0,
+ "7385": 46065532928.0,
+ "7390": 46065532928.0,
+ "7395": 46065532928.0,
+ "7400": 46065532928.0,
+ "7405": 46065532928.0,
+ "7410": 46065532928.0,
+ "7415": 46065532928.0,
+ "7420": 46065532928.0,
+ "7425": 46065532928.0,
+ "7430": 46065532928.0,
+ "7435": 46065532928.0,
+ "7440": 46065532928.0,
+ "7445": 46065532928.0,
+ "7450": 46065532928.0,
+ "7455": 46065532928.0,
+ "7460": 46065532928.0,
+ "7465": 46065532928.0,
+ "7470": 46065532928.0,
+ "7475": 46065532928.0,
+ "7480": 46065532928.0,
+ "7485": 46065532928.0,
+ "7490": 46065532928.0,
+ "7495": 46065532928.0,
+ "7500": 46065532928.0,
+ "7505": 46065532928.0,
+ "7510": 46065532928.0,
+ "7515": 46065532928.0,
+ "7520": 45618061312.0,
+ "7525": 45747933184.0,
+ "7530": 45825024000.0,
+ "7535": 45825024000.0,
+ "7540": 45825024000.0,
+ "7545": 45910597632.0,
+ "7550": 45910597632.0,
+ "7555": 45910597632.0,
+ "7560": 45910597632.0,
+ "7565": 45910597632.0,
+ "7570": 45910597632.0,
+ "7575": 45910597632.0,
+ "7580": 45910597632.0,
+ "7585": 45910597632.0,
+ "7590": 45910597632.0,
+ "7595": 45916950528.0,
+ "7600": 45924253696.0,
+ "7605": 45924253696.0,
+ "7610": 45924253696.0,
+ "7615": 45924253696.0,
+ "7620": 45924253696.0,
+ "7625": 45924253696.0,
+ "7630": 45924253696.0,
+ "7635": 45924253696.0,
+ "7640": 45924253696.0,
+ "7645": 45944950784.0,
+ "7650": 45944950784.0,
+ "7655": 45944950784.0,
+ "7660": 45944950784.0,
+ "7665": 45944950784.0,
+ "7670": 45944950784.0,
+ "7675": 45944950784.0,
+ "7680": 45944950784.0,
+ "7685": 45944950784.0,
+ "7690": 45944950784.0,
+ "7695": 45944950784.0,
+ "7700": 45944950784.0,
+ "7705": 45944950784.0,
+ "7710": 45944950784.0,
+ "7715": 45944950784.0,
+ "7720": 45944950784.0,
+ "7725": 45944950784.0,
+ "7730": 45944950784.0,
+ "7735": 45944950784.0,
+ "7740": 45944950784.0,
+ "7745": 45944950784.0,
+ "7750": 45944950784.0,
+ "7755": 45944950784.0,
+ "7760": 45944950784.0,
+ "7765": 45944950784.0,
+ "7770": 45944950784.0,
+ "7775": 45944950784.0,
+ "7780": 45944950784.0,
+ "7785": 45944950784.0,
+ "7790": 45944950784.0,
+ "7795": 45944950784.0,
+ "7800": 45944950784.0,
+ "7805": 45944950784.0,
+ "7810": 45944950784.0,
+ "7815": 45944950784.0,
+ "7820": 45944950784.0,
+ "7825": 45944950784.0,
+ "7830": 45944950784.0,
+ "7835": 45944950784.0,
+ "7840": 45973135360.0,
+ "7845": 45973135360.0,
+ "7850": 46089904128.0,
+ "7855": 46089904128.0,
+ "7860": 46089904128.0,
+ "7865": 46089904128.0,
+ "7870": 46089904128.0,
+ "7875": 46089904128.0,
+ "7880": 46089904128.0,
+ "7885": 46089904128.0,
+ "7890": 46089904128.0,
+ "7895": 46089904128.0,
+ "7900": 46089904128.0,
+ "7905": 46089904128.0,
+ "7910": 46089904128.0,
+ "7915": 46089904128.0,
+ "7920": 46089904128.0,
+ "7925": 46089904128.0,
+ "7930": 46089904128.0,
+ "7935": 46089904128.0,
+ "7940": 46089904128.0,
+ "7945": 46089904128.0,
+ "7950": 46089904128.0,
+ "7955": 46089904128.0,
+ "7960": 46089904128.0,
+ "7965": 46089904128.0,
+ "7970": 46089904128.0,
+ "7975": 46089904128.0,
+ "7980": 46089904128.0,
+ "7985": 46089904128.0,
+ "7990": 46089904128.0,
+ "7995": 46089904128.0,
+ "8000": 46089904128.0,
+ "8005": 46089904128.0,
+ "8010": 46089904128.0,
+ "8015": 46089904128.0,
+ "8020": 46089904128.0,
+ "8025": 46089904128.0,
+ "8030": 46089904128.0,
+ "8035": 46089904128.0,
+ "8040": 46089904128.0,
+ "8045": 46089904128.0,
+ "8050": 46089904128.0,
+ "8055": 46089904128.0,
+ "8060": 46089904128.0,
+ "8065": 46089904128.0,
+ "8070": 46089904128.0,
+ "8075": 46089904128.0,
+ "8080": 46089904128.0,
+ "8085": 46089904128.0,
+ "8090": 46089904128.0,
+ "8095": 46089904128.0,
+ "8100": 46089904128.0,
+ "8105": 46089904128.0,
+ "8110": 46089904128.0,
+ "8115": 46089904128.0,
+ "8120": 46089904128.0,
+ "8125": 46089904128.0,
+ "8130": 46089904128.0,
+ "8135": 46089904128.0,
+ "8140": 46089904128.0,
+ "8145": 46089904128.0,
+ "8150": 46089904128.0,
+ "8155": 46089904128.0,
+ "8160": 46089904128.0,
+ "8165": 46089904128.0,
+ "8170": 46089904128.0,
+ "8175": 46089904128.0,
+ "8180": 46089904128.0,
+ "8185": 46089904128.0,
+ "8190": 46089904128.0,
+ "8195": 46089904128.0,
+ "8200": 46089904128.0,
+ "8205": 46089904128.0,
+ "8210": 46089904128.0,
+ "8215": 46089904128.0,
+ "8220": 46089904128.0,
+ "8225": 46089904128.0,
+ "8230": 46089904128.0,
+ "8235": 46089904128.0,
+ "8240": 46089904128.0,
+ "8245": 46089904128.0,
+ "8250": 46089904128.0,
+ "8255": 46089904128.0,
+ "8260": 46089904128.0,
+ "8265": 46089904128.0,
+ "8270": 46089904128.0,
+ "8275": 46089904128.0,
+ "8280": 46089904128.0,
+ "8285": 46089904128.0,
+ "8290": 46089904128.0,
+ "8295": 46089904128.0,
+ "8300": 46089904128.0,
+ "8305": 46089904128.0,
+ "8310": 46089904128.0,
+ "8315": 46089904128.0,
+ "8320": 46089904128.0,
+ "8325": 46089904128.0,
+ "8330": 46089904128.0,
+ "8335": 46089904128.0,
+ "8340": 46089904128.0,
+ "8345": 46089904128.0,
+ "8350": 46089904128.0,
+ "8355": 46089904128.0,
+ "8360": 46089904128.0,
+ "8365": 46089904128.0,
+ "8370": 46089904128.0,
+ "8375": 46089904128.0,
+ "8380": 46089904128.0,
+ "8385": 46089904128.0,
+ "8390": 46089904128.0,
+ "8395": 46089904128.0,
+ "8400": 46089904128.0,
+ "8405": 46089904128.0,
+ "8410": 46089904128.0,
+ "8415": 46089904128.0,
+ "8420": 46089904128.0,
+ "8425": 46089904128.0,
+ "8430": 46089904128.0,
+ "8435": 46089904128.0,
+ "8440": 46089904128.0,
+ "8445": 46089904128.0,
+ "8450": 46089904128.0,
+ "8455": 46089904128.0,
+ "8460": 46089904128.0,
+ "8465": 46089904128.0,
+ "8470": 46089904128.0,
+ "8475": 46089904128.0,
+ "8480": 46089904128.0,
+ "8485": 46089904128.0,
+ "8490": 46089904128.0,
+ "8495": 46089904128.0,
+ "8500": 46089904128.0,
+ "8505": 46089904128.0,
+ "8510": 46089904128.0,
+ "8515": 46089904128.0,
+ "8520": 46089904128.0,
+ "8525": 46089904128.0,
+ "8530": 45938114560.0,
+ "8535": 45938114560.0,
+ "8540": 45938114560.0,
+ "8545": 45938114560.0,
+ "8550": 45938114560.0,
+ "8555": 45938114560.0,
+ "8560": 45938114560.0,
+ "8565": 45938114560.0,
+ "8570": 45938114560.0,
+ "8575": 45938114560.0,
+ "8580": 45938114560.0,
+ "8585": 45938114560.0,
+ "8590": 45950377984.0,
+ "8595": 45950377984.0,
+ "8600": 45950377984.0,
+ "8605": 45950377984.0,
+ "8610": 45950377984.0,
+ "8615": 45950377984.0,
+ "8620": 45950377984.0,
+ "8625": 45950377984.0,
+ "8630": 45950377984.0,
+ "8635": 45950377984.0,
+ "8640": 45950377984.0,
+ "8645": 45950377984.0,
+ "8650": 45950377984.0,
+ "8655": 45950377984.0,
+ "8660": 45950377984.0,
+ "8665": 45950377984.0,
+ "8670": 45955510272.0,
+ "8675": 45955510272.0,
+ "8680": 45955510272.0,
+ "8685": 45955510272.0,
+ "8690": 45991550976.0,
+ "8695": 45991550976.0,
+ "8700": 45991550976.0,
+ "8705": 45991550976.0,
+ "8710": 45991550976.0,
+ "8715": 45991550976.0,
+ "8720": 45991550976.0,
+ "8725": 45991550976.0,
+ "8730": 45991550976.0,
+ "8735": 45991550976.0,
+ "8740": 46068584448.0,
+ "8745": 46068584448.0,
+ "8750": 46068584448.0,
+ "8755": 46068584448.0,
+ "8760": 46068584448.0,
+ "8765": 46068584448.0,
+ "8770": 46068584448.0,
+ "8775": 46068584448.0,
+ "8780": 46068584448.0,
+ "8785": 46068584448.0,
+ "8790": 46068584448.0,
+ "8795": 46068584448.0,
+ "8800": 46068584448.0,
+ "8805": 46068584448.0,
+ "8810": 46068584448.0,
+ "8815": 46068584448.0,
+ "8820": 46068584448.0,
+ "8825": 46068584448.0,
+ "8830": 46068584448.0,
+ "8835": 46068584448.0,
+ "8840": 46068584448.0,
+ "8845": 46068584448.0,
+ "8850": 46068584448.0,
+ "8855": 46184767488.0,
+ "8860": 46184767488.0,
+ "8865": 46184767488.0,
+ "8870": 46184767488.0,
+ "8875": 46184767488.0,
+ "8880": 46184767488.0,
+ "8885": 46184767488.0,
+ "8890": 46184767488.0,
+ "8895": 46184767488.0,
+ "8900": 46184767488.0,
+ "8905": 46184767488.0,
+ "8910": 46184767488.0,
+ "8915": 46184767488.0,
+ "8920": 46184767488.0,
+ "8925": 46184767488.0,
+ "8930": 46184767488.0,
+ "8935": 46184767488.0,
+ "8940": 46184767488.0,
+ "8945": 46184767488.0,
+ "8950": 46184767488.0,
+ "8955": 46184767488.0,
+ "8960": 46184767488.0,
+ "8965": 46184767488.0,
+ "8970": 46184767488.0,
+ "8975": 46184767488.0,
+ "8980": 46184767488.0,
+ "8985": 46184767488.0,
+ "8990": 46184767488.0,
+ "8995": 46184767488.0,
+ "9000": 46184767488.0,
+ "9005": 46184767488.0,
+ "9010": 46184767488.0,
+ "9015": 46184767488.0,
+ "9020": 46184767488.0,
+ "9025": 46184767488.0,
+ "9030": 46184767488.0,
+ "9035": 46184767488.0,
+ "9040": 46184767488.0,
+ "9045": 46184767488.0,
+ "9050": 46184767488.0,
+ "9055": 46184767488.0,
+ "9060": 46184767488.0,
+ "9065": 46184767488.0,
+ "9070": 46184767488.0,
+ "9075": 46184767488.0,
+ "9080": 46184767488.0,
+ "9085": 46184767488.0,
+ "9090": 46184767488.0,
+ "9095": 46184767488.0,
+ "9100": 46184767488.0,
+ "9105": 46184767488.0,
+ "9110": 46184767488.0,
+ "9115": 46184767488.0,
+ "9120": 46184767488.0,
+ "9125": 46184767488.0,
+ "9130": 46184767488.0,
+ "9135": 46184767488.0,
+ "9140": 46184767488.0,
+ "9145": 46184767488.0,
+ "9150": 46184767488.0,
+ "9155": 46184767488.0,
+ "9160": 46184767488.0,
+ "9165": 46184767488.0,
+ "9170": 46184767488.0,
+ "9175": 46184767488.0,
+ "9180": 46184767488.0,
+ "9185": 46184767488.0,
+ "9190": 46184767488.0,
+ "9195": 46184767488.0,
+ "9200": 46184767488.0,
+ "9205": 46184767488.0,
+ "9210": 46184767488.0,
+ "9215": 46184767488.0,
+ "9220": 46184767488.0,
+ "9225": 46184767488.0,
+ "9230": 46184767488.0,
+ "9235": 46184767488.0,
+ "9240": 46184767488.0,
+ "9245": 46184767488.0,
+ "9250": 46184767488.0,
+ "9255": 46184767488.0,
+ "9260": 46184767488.0,
+ "9265": 46184767488.0,
+ "9270": 46184767488.0,
+ "9275": 46184767488.0,
+ "9280": 46184767488.0,
+ "9285": 46184767488.0,
+ "9290": 46184767488.0,
+ "9295": 46184767488.0,
+ "9300": 46184767488.0,
+ "9305": 46184767488.0,
+ "9310": 46184767488.0,
+ "9315": 46184767488.0,
+ "9320": 46184767488.0,
+ "9325": 46184767488.0,
+ "9330": 46184767488.0,
+ "9335": 46184767488.0,
+ "9340": 46184767488.0,
+ "9345": 46184767488.0,
+ "9350": 46184767488.0,
+ "9355": 46184767488.0,
+ "9360": 46184767488.0,
+ "9365": 46184767488.0,
+ "9370": 46184767488.0,
+ "9375": 46184767488.0,
+ "9380": 46184767488.0,
+ "9385": 46184767488.0,
+ "9390": 46184767488.0,
+ "9395": 46184767488.0,
+ "9400": 46184767488.0,
+ "9405": 46184767488.0,
+ "9410": 46184767488.0,
+ "9415": 46184767488.0,
+ "9420": 46184767488.0,
+ "9425": 46184767488.0,
+ "9430": 46184767488.0,
+ "9435": 46184767488.0,
+ "9440": 46184767488.0,
+ "9445": 46184767488.0,
+ "9450": 46184767488.0,
+ "9455": 46184767488.0,
+ "9460": 46184767488.0,
+ "9465": 46184767488.0,
+ "9470": 46184767488.0,
+ "9475": 46184767488.0,
+ "9480": 46184767488.0,
+ "9485": 46184767488.0,
+ "9490": 46184767488.0,
+ "9495": 46184767488.0,
+ "9500": 46184767488.0,
+ "9505": 46184767488.0,
+ "9510": 46184767488.0,
+ "9515": 46184767488.0,
+ "9520": 46184767488.0,
+ "9525": 46184767488.0,
+ "9530": 46184767488.0,
+ "9535": 46184767488.0
+ }
+ },
+ "mtp_1 loss": {
+ "start_step": 1,
+ "end_step": 9535,
+ "step_interval": 5,
+ "values": {
+ "1": 13.88878,
+ "5": 13.88979,
+ "10": 13.88767,
+ "15": 13.88576,
+ "20": 13.88068,
+ "25": 13.87774,
+ "30": 13.85566,
+ "35": 13.84855,
+ "40": 13.84546,
+ "45": 13.82693,
+ "50": 13.74828,
+ "55": 13.7249,
+ "60": 13.70841,
+ "65": 13.67571,
+ "70": 13.63981,
+ "75": 13.44327,
+ "80": 13.36054,
+ "85": 13.2835,
+ "90": 13.18641,
+ "95": 13.0505,
+ "100": 12.90733,
+ "105": 12.74689,
+ "110": 12.48525,
+ "115": 12.26801,
+ "120": 12.04358,
+ "125": 11.87011,
+ "130": 11.74911,
+ "135": 11.5841,
+ "140": 11.3494,
+ "145": 11.26997,
+ "150": 11.11919,
+ "155": 11.0211,
+ "160": 10.88133,
+ "165": 10.75162,
+ "170": 10.65694,
+ "175": 10.59566,
+ "180": 10.43546,
+ "185": 10.42441,
+ "190": 10.27183,
+ "195": 10.2539,
+ "200": 10.12718,
+ "205": 9.97472,
+ "210": 9.94271,
+ "215": 9.92122,
+ "220": 9.78944,
+ "225": 9.77014,
+ "230": 9.73,
+ "235": 9.64372,
+ "240": 9.57366,
+ "245": 9.50499,
+ "250": 9.43776,
+ "255": 9.37037,
+ "260": 9.29579,
+ "265": 9.2411,
+ "270": 9.15629,
+ "275": 9.12851,
+ "280": 9.10516,
+ "285": 9.09815,
+ "290": 9.01068,
+ "295": 8.94828,
+ "300": 8.83207,
+ "305": 8.80663,
+ "310": 8.74389,
+ "315": 8.71813,
+ "320": 8.68425,
+ "325": 8.58706,
+ "330": 8.56208,
+ "335": 8.53307,
+ "340": 8.52937,
+ "345": 8.41091,
+ "350": 8.39973,
+ "355": 8.29759,
+ "360": 8.38348,
+ "365": 8.28981,
+ "370": 8.2833,
+ "375": 8.22588,
+ "380": 8.18359,
+ "385": 8.16998,
+ "390": 8.1467,
+ "395": 8.09789,
+ "400": 8.01583,
+ "405": 8.01349,
+ "410": 8.00377,
+ "415": 7.95012,
+ "420": 7.93109,
+ "425": 7.88677,
+ "430": 7.81895,
+ "435": 7.82989,
+ "440": 7.77278,
+ "445": 7.7493,
+ "450": 7.67877,
+ "455": 7.7063,
+ "460": 7.6532,
+ "465": 7.6329,
+ "470": 7.59885,
+ "475": 7.61277,
+ "480": 7.48436,
+ "485": 7.53153,
+ "490": 7.48574,
+ "495": 7.4714,
+ "500": 7.41282,
+ "505": 7.41932,
+ "510": 7.38698,
+ "515": 7.35645,
+ "520": 7.35102,
+ "525": 7.32559,
+ "530": 7.32588,
+ "535": 7.30357,
+ "540": 7.2179,
+ "545": 7.24022,
+ "550": 7.27618,
+ "555": 7.30238,
+ "560": 7.23984,
+ "565": 7.16321,
+ "570": 7.17228,
+ "575": 7.18898,
+ "580": 7.11497,
+ "585": 7.11901,
+ "590": 7.06121,
+ "595": 7.04317,
+ "600": 7.06682,
+ "605": 7.06137,
+ "610": 7.01939,
+ "615": 7.078,
+ "620": 6.98113,
+ "625": 6.95612,
+ "630": 6.96104,
+ "635": 6.98871,
+ "640": 6.96819,
+ "645": 6.95817,
+ "650": 7.00625,
+ "655": 7.00242,
+ "660": 6.89823,
+ "665": 6.88159,
+ "670": 6.84888,
+ "675": 6.93827,
+ "680": 6.89638,
+ "685": 6.85679,
+ "690": 6.83445,
+ "695": 6.79719,
+ "700": 6.79183,
+ "705": 6.78625,
+ "710": 6.82275,
+ "715": 6.82665,
+ "720": 6.71137,
+ "725": 6.76643,
+ "730": 6.75579,
+ "735": 6.75515,
+ "740": 6.70045,
+ "745": 6.67565,
+ "750": 6.73564,
+ "755": 6.65767,
+ "760": 6.66496,
+ "765": 6.65951,
+ "770": 6.68075,
+ "775": 6.65453,
+ "780": 6.62427,
+ "785": 6.64321,
+ "790": 6.59399,
+ "795": 6.59812,
+ "800": 6.5878,
+ "805": 6.65391,
+ "810": 6.51946,
+ "815": 6.5419,
+ "820": 6.55134,
+ "825": 6.55855,
+ "830": 6.57041,
+ "835": 6.52603,
+ "840": 6.49033,
+ "845": 6.54438,
+ "850": 6.49874,
+ "855": 6.49335,
+ "860": 6.49024,
+ "865": 6.49642,
+ "870": 6.46222,
+ "875": 6.51054,
+ "880": 6.4748,
+ "885": 6.43786,
+ "890": 6.51246,
+ "895": 6.39629,
+ "900": 6.41895,
+ "905": 6.44341,
+ "910": 6.40617,
+ "915": 6.38978,
+ "920": 6.38772,
+ "925": 6.37391,
+ "930": 6.40825,
+ "935": 6.39755,
+ "940": 6.34172,
+ "945": 6.36869,
+ "950": 6.3953,
+ "955": 6.34893,
+ "960": 6.35406,
+ "965": 6.25416,
+ "970": 6.32381,
+ "975": 6.31262,
+ "980": 6.28797,
+ "985": 6.29222,
+ "990": 6.34527,
+ "995": 6.26326,
+ "1000": 6.28434,
+ "1005": 6.23155,
+ "1010": 6.26712,
+ "1015": 6.29352,
+ "1020": 6.20454,
+ "1025": 6.21082,
+ "1030": 6.20913,
+ "1035": 6.29924,
+ "1040": 6.22531,
+ "1045": 6.19943,
+ "1050": 6.2267,
+ "1055": 6.21777,
+ "1060": 6.1673,
+ "1065": 6.15758,
+ "1070": 6.19281,
+ "1075": 6.19093,
+ "1080": 6.19319,
+ "1085": 6.19606,
+ "1090": 6.17796,
+ "1095": 6.181,
+ "1100": 6.1397,
+ "1105": 6.11513,
+ "1110": 6.17787,
+ "1115": 6.11231,
+ "1120": 6.05286,
+ "1125": 6.08699,
+ "1130": 6.14167,
+ "1135": 6.09531,
+ "1140": 6.08221,
+ "1145": 6.06731,
+ "1150": 6.09458,
+ "1155": 6.06298,
+ "1160": 6.04607,
+ "1165": 6.09676,
+ "1170": 6.07336,
+ "1175": 6.04568,
+ "1180": 6.05058,
+ "1185": 6.04124,
+ "1190": 6.04961,
+ "1195": 6.02949,
+ "1200": 5.97329,
+ "1205": 6.07601,
+ "1210": 5.93751,
+ "1215": 5.98403,
+ "1220": 6.06306,
+ "1225": 5.95152,
+ "1230": 5.99877,
+ "1235": 5.95912,
+ "1240": 5.99322,
+ "1245": 5.97187,
+ "1250": 5.95299,
+ "1255": 5.94742,
+ "1260": 5.95227,
+ "1265": 5.93352,
+ "1270": 5.90818,
+ "1275": 5.96805,
+ "1280": 5.90416,
+ "1285": 5.92308,
+ "1290": 5.90725,
+ "1295": 5.92,
+ "1300": 5.9267,
+ "1305": 5.90057,
+ "1310": 5.83908,
+ "1315": 5.8992,
+ "1320": 5.89614,
+ "1325": 5.8271,
+ "1330": 5.88462,
+ "1335": 5.8531,
+ "1340": 5.91994,
+ "1345": 5.86667,
+ "1350": 5.84738,
+ "1355": 5.84415,
+ "1360": 5.85216,
+ "1365": 5.84478,
+ "1370": 5.79663,
+ "1375": 5.80667,
+ "1380": 5.86219,
+ "1385": 5.81826,
+ "1390": 5.81231,
+ "1395": 5.8299,
+ "1400": 5.83135,
+ "1405": 5.82032,
+ "1410": 5.78518,
+ "1415": 5.77017,
+ "1420": 5.8049,
+ "1425": 5.79565,
+ "1430": 5.83189,
+ "1435": 5.74562,
+ "1440": 5.76408,
+ "1445": 5.8071,
+ "1450": 5.78859,
+ "1455": 5.80534,
+ "1460": 5.75975,
+ "1465": 5.76379,
+ "1470": 5.8044,
+ "1475": 5.76985,
+ "1480": 5.77563,
+ "1485": 5.72396,
+ "1490": 5.72354,
+ "1495": 5.74538,
+ "1500": 5.75109,
+ "1505": 5.72321,
+ "1510": 5.74832,
+ "1515": 5.67052,
+ "1520": 5.70302,
+ "1525": 5.67385,
+ "1530": 5.69497,
+ "1535": 5.68565,
+ "1540": 5.672,
+ "1545": 5.7178,
+ "1550": 5.72274,
+ "1555": 5.70942,
+ "1560": 5.65211,
+ "1565": 5.69926,
+ "1570": 5.71179,
+ "1575": 5.6613,
+ "1580": 5.69275,
+ "1585": 5.67221,
+ "1590": 5.66087,
+ "1595": 5.63673,
+ "1600": 5.70849,
+ "1605": 5.64113,
+ "1610": 5.64353,
+ "1615": 5.63334,
+ "1620": 5.65496,
+ "1625": 5.64982,
+ "1630": 5.62727,
+ "1635": 5.67706,
+ "1640": 5.62761,
+ "1645": 5.6449,
+ "1650": 5.63803,
+ "1655": 5.62499,
+ "1660": 5.61278,
+ "1665": 5.60116,
+ "1670": 5.61214,
+ "1675": 5.62193,
+ "1680": 5.56155,
+ "1685": 5.57098,
+ "1690": 5.55098,
+ "1695": 5.55521,
+ "1700": 5.60178,
+ "1705": 5.57706,
+ "1710": 5.58407,
+ "1715": 5.54721,
+ "1720": 5.52704,
+ "1725": 5.56718,
+ "1730": 5.53148,
+ "1735": 5.58307,
+ "1740": 5.52337,
+ "1745": 5.55772,
+ "1750": 5.53213,
+ "1755": 5.5301,
+ "1760": 5.55304,
+ "1765": 5.5132,
+ "1770": 5.522,
+ "1775": 5.52704,
+ "1780": 5.53997,
+ "1785": 5.48896,
+ "1790": 5.52187,
+ "1795": 5.52448,
+ "1800": 5.4698,
+ "1805": 5.46326,
+ "1810": 5.47869,
+ "1815": 5.48464,
+ "1820": 5.48466,
+ "1825": 5.48352,
+ "1830": 5.46909,
+ "1835": 5.46355,
+ "1840": 5.46633,
+ "1845": 5.44723,
+ "1850": 5.42996,
+ "1855": 5.4834,
+ "1860": 5.43502,
+ "1865": 5.44302,
+ "1870": 5.43258,
+ "1875": 5.42823,
+ "1880": 5.491,
+ "1885": 5.45039,
+ "1890": 5.44132,
+ "1895": 5.38084,
+ "1900": 5.42123,
+ "1905": 5.41299,
+ "1910": 5.43539,
+ "1915": 5.4013,
+ "1920": 5.37729,
+ "1925": 5.4085,
+ "1930": 5.37579,
+ "1935": 5.39731,
+ "1940": 5.3727,
+ "1945": 5.4174,
+ "1950": 5.45899,
+ "1955": 5.39197,
+ "1960": 5.39342,
+ "1965": 5.34213,
+ "1970": 5.34023,
+ "1975": 5.40413,
+ "1980": 5.35398,
+ "1985": 5.37376,
+ "1990": 5.39658,
+ "1995": 5.37398,
+ "2000": 5.38469,
+ "2005": 5.42838,
+ "2010": 5.32884,
+ "2015": 5.32047,
+ "2020": 5.32991,
+ "2025": 5.37403,
+ "2030": 5.31228,
+ "2035": 5.33119,
+ "2040": 5.29466,
+ "2045": 5.38332,
+ "2050": 5.35716,
+ "2055": 5.33062,
+ "2060": 5.32903,
+ "2065": 5.29751,
+ "2070": 5.29985,
+ "2075": 5.32708,
+ "2080": 5.29709,
+ "2085": 5.32918,
+ "2090": 5.24905,
+ "2095": 5.29587,
+ "2100": 5.25777,
+ "2105": 5.28625,
+ "2110": 5.28042,
+ "2115": 5.28102,
+ "2120": 5.2839,
+ "2125": 5.24699,
+ "2130": 5.25602,
+ "2135": 5.25599,
+ "2140": 5.26607,
+ "2145": 5.22772,
+ "2150": 5.24774,
+ "2155": 5.22588,
+ "2160": 5.24123,
+ "2165": 5.22937,
+ "2170": 5.26626,
+ "2175": 5.2603,
+ "2180": 5.24294,
+ "2185": 5.24675,
+ "2190": 5.22691,
+ "2195": 5.20127,
+ "2200": 5.20409,
+ "2205": 5.2127,
+ "2210": 5.25738,
+ "2215": 5.30103,
+ "2220": 5.24446,
+ "2225": 5.2194,
+ "2230": 5.21789,
+ "2235": 5.25766,
+ "2240": 5.16329,
+ "2245": 5.1607,
+ "2250": 5.18607,
+ "2255": 5.19635,
+ "2260": 5.13701,
+ "2265": 5.21276,
+ "2270": 5.14278,
+ "2275": 5.19722,
+ "2280": 5.17159,
+ "2285": 5.18798,
+ "2290": 5.17456,
+ "2295": 5.18141,
+ "2300": 5.17912,
+ "2305": 5.15551,
+ "2310": 5.1834,
+ "2315": 5.12144,
+ "2320": 5.17039,
+ "2325": 5.14984,
+ "2330": 5.15156,
+ "2335": 5.13195,
+ "2340": 5.13852,
+ "2345": 5.18732,
+ "2350": 5.12945,
+ "2355": 5.11891,
+ "2360": 5.10445,
+ "2365": 5.11898,
+ "2370": 5.10258,
+ "2375": 5.11122,
+ "2380": 5.05395,
+ "2385": 5.09747,
+ "2390": 5.11702,
+ "2395": 5.1322,
+ "2400": 5.07944,
+ "2405": 5.06236,
+ "2410": 5.11554,
+ "2415": 5.09106,
+ "2420": 5.10878,
+ "2425": 5.06863,
+ "2430": 5.09273,
+ "2435": 5.08666,
+ "2440": 5.07515,
+ "2445": 5.08608,
+ "2450": 5.04943,
+ "2455": 5.09523,
+ "2460": 5.04536,
+ "2465": 5.08334,
+ "2470": 5.07644,
+ "2475": 5.11246,
+ "2480": 5.02872,
+ "2485": 5.05906,
+ "2490": 5.05297,
+ "2495": 5.04377,
+ "2500": 5.04447,
+ "2505": 5.05124,
+ "2510": 5.0909,
+ "2515": 5.08005,
+ "2520": 5.02414,
+ "2525": 5.03617,
+ "2530": 5.05281,
+ "2535": 5.04127,
+ "2540": 5.04342,
+ "2545": 5.05498,
+ "2550": 4.99288,
+ "2555": 5.05988,
+ "2560": 5.03403,
+ "2565": 5.00279,
+ "2570": 5.02524,
+ "2575": 4.98811,
+ "2580": 5.00235,
+ "2585": 4.98259,
+ "2590": 5.00195,
+ "2595": 4.95577,
+ "2600": 4.99616,
+ "2605": 5.01565,
+ "2610": 5.00846,
+ "2615": 4.9779,
+ "2620": 4.96,
+ "2625": 4.99167,
+ "2630": 4.92069,
+ "2635": 5.00179,
+ "2640": 5.00217,
+ "2645": 4.95857,
+ "2650": 4.98056,
+ "2655": 4.97276,
+ "2660": 4.91658,
+ "2665": 5.00931,
+ "2670": 4.95271,
+ "2675": 4.92627,
+ "2680": 4.95939,
+ "2685": 4.9606,
+ "2690": 4.92299,
+ "2695": 4.99925,
+ "2700": 4.90798,
+ "2705": 4.92161,
+ "2710": 4.9625,
+ "2715": 4.94083,
+ "2720": 4.97062,
+ "2725": 4.91977,
+ "2730": 4.9445,
+ "2735": 4.9369,
+ "2740": 4.92939,
+ "2745": 4.89678,
+ "2750": 4.93832,
+ "2755": 4.94144,
+ "2760": 4.94244,
+ "2765": 4.91315,
+ "2770": 4.95527,
+ "2775": 4.90029,
+ "2780": 4.93753,
+ "2785": 4.91159,
+ "2790": 4.93952,
+ "2795": 4.89812,
+ "2800": 4.84327,
+ "2805": 4.89103,
+ "2810": 4.88284,
+ "2815": 4.89434,
+ "2820": 4.93504,
+ "2825": 4.92479,
+ "2830": 4.90086,
+ "2835": 4.90451,
+ "2840": 4.89553,
+ "2845": 4.87238,
+ "2850": 4.90777,
+ "2855": 4.83628,
+ "2860": 4.89239,
+ "2865": 4.90134,
+ "2870": 4.89048,
+ "2875": 4.90822,
+ "2880": 4.82774,
+ "2885": 4.8758,
+ "2890": 4.84909,
+ "2895": 4.88906,
+ "2900": 4.84436,
+ "2905": 4.85096,
+ "2910": 4.84745,
+ "2915": 4.89554,
+ "2920": 4.87192,
+ "2925": 4.84408,
+ "2930": 4.83304,
+ "2935": 4.83856,
+ "2940": 4.8364,
+ "2945": 4.80087,
+ "2950": 4.79094,
+ "2955": 4.79257,
+ "2960": 4.81394,
+ "2965": 4.82244,
+ "2970": 4.83033,
+ "2975": 4.843,
+ "2980": 4.78708,
+ "2985": 4.83546,
+ "2990": 4.84632,
+ "2995": 4.79479,
+ "3000": 4.79957,
+ "3005": 4.7852,
+ "3010": 4.81747,
+ "3015": 4.77707,
+ "3020": 4.79613,
+ "3025": 4.80689,
+ "3030": 4.81521,
+ "3035": 4.81107,
+ "3040": 4.83014,
+ "3045": 4.81253,
+ "3050": 4.78854,
+ "3055": 4.79109,
+ "3060": 4.77291,
+ "3065": 4.80026,
+ "3070": 4.82011,
+ "3075": 4.75177,
+ "3080": 4.78059,
+ "3085": 4.7825,
+ "3090": 4.76596,
+ "3095": 4.80833,
+ "3100": 4.79656,
+ "3105": 4.77177,
+ "3110": 4.76085,
+ "3115": 4.71609,
+ "3120": 4.78235,
+ "3125": 4.74714,
+ "3130": 4.75497,
+ "3135": 4.75435,
+ "3140": 4.7318,
+ "3145": 4.71606,
+ "3150": 4.74842,
+ "3155": 4.78313,
+ "3160": 4.765,
+ "3165": 4.75911,
+ "3170": 4.7541,
+ "3175": 4.746,
+ "3180": 4.73371,
+ "3185": 4.70655,
+ "3190": 4.70906,
+ "3195": 4.70876,
+ "3200": 4.67795,
+ "3205": 4.72527,
+ "3210": 4.67973,
+ "3215": 4.71138,
+ "3220": 4.67941,
+ "3225": 4.71501,
+ "3230": 4.698,
+ "3235": 4.73415,
+ "3240": 4.68214,
+ "3245": 4.6954,
+ "3250": 4.64543,
+ "3255": 4.69551,
+ "3260": 4.67926,
+ "3265": 4.72582,
+ "3270": 4.70744,
+ "3275": 4.65457,
+ "3280": 4.68021,
+ "3285": 4.69583,
+ "3290": 4.66845,
+ "3295": 4.67202,
+ "3300": 4.66858,
+ "3305": 4.67172,
+ "3310": 4.66314,
+ "3315": 4.70829,
+ "3320": 4.64885,
+ "3325": 4.65812,
+ "3330": 4.64245,
+ "3335": 4.65293,
+ "3340": 4.62608,
+ "3345": 4.64548,
+ "3350": 4.65071,
+ "3355": 4.65765,
+ "3360": 4.64823,
+ "3365": 4.66194,
+ "3370": 4.63984,
+ "3375": 4.67722,
+ "3380": 4.61449,
+ "3385": 4.62869,
+ "3390": 4.60608,
+ "3395": 4.6967,
+ "3400": 4.64188,
+ "3405": 4.6721,
+ "3410": 4.60581,
+ "3415": 4.55337,
+ "3420": 4.61467,
+ "3425": 4.63228,
+ "3430": 4.66874,
+ "3435": 4.63419,
+ "3440": 4.65338,
+ "3445": 4.60093,
+ "3450": 4.59889,
+ "3455": 4.62429,
+ "3460": 4.58089,
+ "3465": 4.57689,
+ "3470": 4.59454,
+ "3475": 4.60079,
+ "3480": 4.59374,
+ "3485": 4.62356,
+ "3490": 4.60917,
+ "3495": 4.63221,
+ "3500": 4.59027,
+ "3505": 4.59844,
+ "3510": 4.59797,
+ "3515": 4.648,
+ "3520": 4.62554,
+ "3525": 4.57245,
+ "3530": 4.58587,
+ "3535": 4.58174,
+ "3540": 4.63653,
+ "3545": 4.56212,
+ "3550": 4.62056,
+ "3555": 4.55332,
+ "3560": 4.62414,
+ "3565": 4.55473,
+ "3570": 4.56696,
+ "3575": 4.53468,
+ "3580": 4.59878,
+ "3585": 4.58068,
+ "3590": 4.51872,
+ "3595": 4.58848,
+ "3600": 4.55395,
+ "3605": 4.53571,
+ "3610": 4.54008,
+ "3615": 4.56874,
+ "3620": 4.61691,
+ "3625": 4.55023,
+ "3630": 4.59867,
+ "3635": 4.50879,
+ "3640": 4.52782,
+ "3645": 4.56947,
+ "3650": 4.53552,
+ "3655": 4.54665,
+ "3660": 4.55228,
+ "3665": 4.58643,
+ "3670": 4.54047,
+ "3675": 4.55594,
+ "3680": 4.57348,
+ "3685": 4.49418,
+ "3690": 4.54299,
+ "3695": 4.49297,
+ "3700": 4.52866,
+ "3705": 4.50654,
+ "3710": 4.51966,
+ "3715": 4.53,
+ "3720": 4.50118,
+ "3725": 4.47886,
+ "3730": 4.4879,
+ "3735": 4.50546,
+ "3740": 4.49399,
+ "3745": 4.48041,
+ "3750": 4.51288,
+ "3755": 4.48915,
+ "3760": 4.50004,
+ "3765": 4.47669,
+ "3770": 4.48984,
+ "3775": 4.46969,
+ "3780": 4.45476,
+ "3785": 4.50898,
+ "3790": 4.42336,
+ "3795": 4.4846,
+ "3800": 4.46028,
+ "3805": 4.46023,
+ "3810": 4.42629,
+ "3815": 4.4806,
+ "3820": 4.4736,
+ "3825": 4.4803,
+ "3830": 4.46747,
+ "3835": 4.42638,
+ "3840": 4.52349,
+ "3845": 4.48225,
+ "3850": 4.42266,
+ "3855": 4.46223,
+ "3860": 4.48001,
+ "3865": 4.44144,
+ "3870": 4.50523,
+ "3875": 4.41439,
+ "3880": 4.42672,
+ "3885": 4.44983,
+ "3890": 4.43819,
+ "3895": 4.38007,
+ "3900": 4.43434,
+ "3905": 4.41283,
+ "3910": 4.42081,
+ "3915": 4.42082,
+ "3920": 4.41329,
+ "3925": 4.39336,
+ "3930": 4.41243,
+ "3935": 4.41903,
+ "3940": 4.41848,
+ "3945": 4.39397,
+ "3950": 4.46098,
+ "3955": 4.39087,
+ "3960": 4.43851,
+ "3965": 4.44901,
+ "3970": 4.39272,
+ "3975": 4.40242,
+ "3980": 4.37236,
+ "3985": 4.40832,
+ "3990": 4.40208,
+ "3995": 4.44335,
+ "4000": 4.38322,
+ "4005": 4.37255,
+ "4010": 4.40982,
+ "4015": 4.39813,
+ "4020": 4.43488,
+ "4025": 4.39111,
+ "4030": 4.44761,
+ "4035": 4.40548,
+ "4040": 4.43553,
+ "4045": 4.41155,
+ "4050": 4.40643,
+ "4055": 4.41393,
+ "4060": 4.40665,
+ "4065": 4.41291,
+ "4070": 4.34904,
+ "4075": 4.37708,
+ "4080": 4.35797,
+ "4085": 4.39736,
+ "4090": 4.37437,
+ "4095": 4.35826,
+ "4100": 4.37323,
+ "4105": 4.36208,
+ "4110": 4.32609,
+ "4115": 4.39421,
+ "4120": 4.31057,
+ "4125": 4.31168,
+ "4130": 4.39302,
+ "4135": 4.37289,
+ "4140": 4.31616,
+ "4145": 4.32788,
+ "4150": 4.37558,
+ "4155": 4.29766,
+ "4160": 4.35633,
+ "4165": 4.38157,
+ "4170": 4.32646,
+ "4175": 4.33285,
+ "4180": 4.32735,
+ "4185": 4.31953,
+ "4190": 4.31017,
+ "4195": 4.31525,
+ "4200": 4.31406,
+ "4205": 4.37,
+ "4210": 4.32695,
+ "4215": 4.3562,
+ "4220": 4.33701,
+ "4225": 4.32036,
+ "4230": 4.30579,
+ "4235": 4.35051,
+ "4240": 4.30872,
+ "4245": 4.31564,
+ "4250": 4.29999,
+ "4255": 4.31166,
+ "4260": 4.29019,
+ "4265": 4.30554,
+ "4270": 4.29954,
+ "4275": 4.36276,
+ "4280": 4.29798,
+ "4285": 4.33284,
+ "4290": 4.27741,
+ "4295": 4.30368,
+ "4300": 4.32594,
+ "4305": 4.29066,
+ "4310": 4.33408,
+ "4315": 4.3163,
+ "4320": 4.30571,
+ "4325": 4.32764,
+ "4330": 4.26525,
+ "4335": 4.30418,
+ "4340": 4.28838,
+ "4345": 4.23753,
+ "4350": 4.25927,
+ "4355": 4.33009,
+ "4360": 4.30543,
+ "4365": 4.30411,
+ "4370": 4.28149,
+ "4375": 4.24372,
+ "4380": 4.25559,
+ "4385": 4.23331,
+ "4390": 4.30895,
+ "4395": 4.27518,
+ "4400": 4.26254,
+ "4405": 4.23007,
+ "4410": 4.28048,
+ "4415": 4.26816,
+ "4420": 4.24916,
+ "4425": 4.29252,
+ "4430": 4.24244,
+ "4435": 4.29049,
+ "4440": 4.28601,
+ "4445": 4.24232,
+ "4450": 4.20719,
+ "4455": 4.26016,
+ "4460": 4.23459,
+ "4465": 4.25243,
+ "4470": 4.23841,
+ "4475": 4.2641,
+ "4480": 4.24909,
+ "4485": 4.23389,
+ "4490": 4.23593,
+ "4495": 4.17962,
+ "4500": 4.25444,
+ "4505": 4.22942,
+ "4510": 4.23965,
+ "4515": 4.19566,
+ "4520": 4.23113,
+ "4525": 4.19456,
+ "4530": 4.24001,
+ "4535": 4.20166,
+ "4540": 4.21127,
+ "4545": 4.23188,
+ "4550": 4.27088,
+ "4555": 4.2072,
+ "4560": 4.22378,
+ "4565": 4.15426,
+ "4570": 4.21606,
+ "4575": 4.1941,
+ "4580": 4.25747,
+ "4585": 4.22428,
+ "4590": 4.21266,
+ "4595": 4.17399,
+ "4600": 4.16313,
+ "4605": 4.2045,
+ "4610": 4.19939,
+ "4615": 4.24443,
+ "4620": 4.16447,
+ "4625": 4.19099,
+ "4630": 4.20991,
+ "4635": 4.18208,
+ "4640": 4.21078,
+ "4645": 4.20652,
+ "4650": 4.22758,
+ "4655": 4.19246,
+ "4660": 4.18248,
+ "4665": 4.193,
+ "4670": 4.23574,
+ "4675": 4.17989,
+ "4680": 4.20859,
+ "4685": 4.19688,
+ "4690": 4.1723,
+ "4695": 4.18485,
+ "4700": 4.16546,
+ "4705": 4.14067,
+ "4710": 4.20305,
+ "4715": 4.19002,
+ "4720": 4.14737,
+ "4725": 4.12216,
+ "4730": 4.17809,
+ "4735": 4.10178,
+ "4740": 4.14697,
+ "4745": 4.18779,
+ "4750": 4.13615,
+ "4755": 4.19424,
+ "4760": 4.1984,
+ "4765": 4.1461,
+ "4770": 4.14849,
+ "4775": 4.14773,
+ "4780": 4.15523,
+ "4785": 4.13664,
+ "4790": 4.19224,
+ "4795": 4.17628,
+ "4800": 4.13942,
+ "4805": 4.17839,
+ "4810": 4.1375,
+ "4815": 4.17167,
+ "4820": 4.12226,
+ "4825": 4.17474,
+ "4830": 4.16985,
+ "4835": 4.14976,
+ "4840": 4.15298,
+ "4845": 4.10968,
+ "4850": 4.17354,
+ "4855": 4.17639,
+ "4860": 4.11236,
+ "4865": 4.13759,
+ "4870": 4.13215,
+ "4875": 4.17643,
+ "4880": 4.1702,
+ "4885": 4.13029,
+ "4890": 4.1249,
+ "4895": 4.12403,
+ "4900": 4.09958,
+ "4905": 4.09173,
+ "4910": 4.09074,
+ "4915": 4.14665,
+ "4920": 4.12021,
+ "4925": 4.08814,
+ "4930": 4.09778,
+ "4935": 4.12094,
+ "4940": 4.04981,
+ "4945": 4.13369,
+ "4950": 4.07708,
+ "4955": 4.15684,
+ "4960": 4.11652,
+ "4965": 4.1151,
+ "4970": 4.09971,
+ "4975": 4.11736,
+ "4980": 4.12585,
+ "4985": 4.12754,
+ "4990": 4.09005,
+ "4995": 4.12916,
+ "5000": 4.05682,
+ "5005": 4.11701,
+ "5010": 4.10942,
+ "5015": 4.07584,
+ "5020": 4.05201,
+ "5025": 4.06082,
+ "5030": 4.10005,
+ "5035": 4.08177,
+ "5040": 4.0418,
+ "5045": 4.11064,
+ "5050": 4.06425,
+ "5055": 4.08995,
+ "5060": 4.03143,
+ "5065": 4.09666,
+ "5070": 4.07056,
+ "5075": 4.12386,
+ "5080": 4.07795,
+ "5085": 4.09595,
+ "5090": 4.07748,
+ "5095": 4.0424,
+ "5100": 4.0782,
+ "5105": 4.0809,
+ "5110": 4.08612,
+ "5115": 4.07663,
+ "5120": 4.09438,
+ "5125": 4.05976,
+ "5130": 4.06327,
+ "5135": 4.0488,
+ "5140": 4.06922,
+ "5145": 4.05942,
+ "5150": 4.07092,
+ "5155": 4.07553,
+ "5160": 4.05549,
+ "5165": 4.09766,
+ "5170": 3.96642,
+ "5175": 4.07515,
+ "5180": 4.03531,
+ "5185": 4.05861,
+ "5190": 4.08092,
+ "5195": 4.04601,
+ "5200": 4.06577,
+ "5205": 4.09747,
+ "5210": 4.01055,
+ "5215": 4.02373,
+ "5220": 4.02621,
+ "5225": 4.02349,
+ "5230": 4.06271,
+ "5235": 4.03585,
+ "5240": 4.02422,
+ "5245": 4.04177,
+ "5250": 4.04544,
+ "5255": 4.03173,
+ "5260": 4.04798,
+ "5265": 4.01495,
+ "5270": 3.98673,
+ "5275": 4.00519,
+ "5280": 4.02024,
+ "5285": 4.04277,
+ "5290": 4.00304,
+ "5295": 4.00093,
+ "5300": 4.02323,
+ "5305": 4.01012,
+ "5310": 4.0478,
+ "5315": 3.99571,
+ "5320": 4.03864,
+ "5325": 4.06497,
+ "5330": 3.99981,
+ "5335": 4.02122,
+ "5340": 3.9739,
+ "5345": 4.01424,
+ "5350": 4.0246,
+ "5355": 4.01714,
+ "5360": 3.9668,
+ "5365": 3.98455,
+ "5370": 4.02892,
+ "5375": 3.99384,
+ "5380": 3.98952,
+ "5385": 4.00787,
+ "5390": 3.99585,
+ "5395": 3.932,
+ "5400": 4.02192,
+ "5405": 3.94401,
+ "5410": 4.03103,
+ "5415": 3.94954,
+ "5420": 3.98108,
+ "5425": 3.96619,
+ "5430": 3.97462,
+ "5435": 4.00917,
+ "5440": 3.96082,
+ "5445": 3.96843,
+ "5450": 3.98078,
+ "5455": 3.96312,
+ "5460": 3.97781,
+ "5465": 4.03343,
+ "5470": 3.99301,
+ "5475": 3.92634,
+ "5480": 4.0001,
+ "5485": 3.96789,
+ "5490": 3.99381,
+ "5495": 3.99755,
+ "5500": 3.95394,
+ "5505": 3.9702,
+ "5510": 4.00139,
+ "5515": 3.97886,
+ "5520": 3.95723,
+ "5525": 4.01089,
+ "5530": 3.95723,
+ "5535": 3.99058,
+ "5540": 3.95888,
+ "5545": 3.97704,
+ "5550": 3.97005,
+ "5555": 3.93134,
+ "5560": 3.94203,
+ "5565": 3.98688,
+ "5570": 3.94409,
+ "5575": 3.97691,
+ "5580": 3.95423,
+ "5585": 3.89232,
+ "5590": 3.96662,
+ "5595": 3.91996,
+ "5600": 3.97099,
+ "5605": 3.87423,
+ "5610": 3.96509,
+ "5615": 3.9629,
+ "5620": 3.97882,
+ "5625": 3.95843,
+ "5630": 3.94884,
+ "5635": 3.92989,
+ "5640": 3.95308,
+ "5645": 3.91537,
+ "5650": 3.88759,
+ "5655": 3.91914,
+ "5660": 3.9101,
+ "5665": 3.92739,
+ "5670": 3.91107,
+ "5675": 3.94487,
+ "5680": 3.91238,
+ "5685": 3.92365,
+ "5690": 3.92517,
+ "5695": 3.953,
+ "5700": 3.88996,
+ "5705": 3.88995,
+ "5710": 3.87532,
+ "5715": 3.99623,
+ "5720": 3.94505,
+ "5725": 3.89527,
+ "5730": 3.94792,
+ "5735": 3.92817,
+ "5740": 3.92171,
+ "5745": 3.89897,
+ "5750": 3.92176,
+ "5755": 3.94672,
+ "5760": 3.92632,
+ "5765": 3.92024,
+ "5770": 3.95286,
+ "5775": 3.86965,
+ "5780": 3.91041,
+ "5785": 3.91605,
+ "5790": 3.9236,
+ "5795": 3.93068,
+ "5800": 3.86954,
+ "5805": 3.8764,
+ "5810": 3.92692,
+ "5815": 3.89083,
+ "5820": 3.84021,
+ "5825": 3.89285,
+ "5830": 3.85163,
+ "5835": 3.88292,
+ "5840": 3.89361,
+ "5845": 3.91293,
+ "5850": 3.90508,
+ "5855": 3.84956,
+ "5860": 3.87018,
+ "5865": 3.8979,
+ "5870": 3.85816,
+ "5875": 3.89604,
+ "5880": 3.88075,
+ "5885": 3.89965,
+ "5890": 3.90395,
+ "5895": 3.92339,
+ "5900": 3.85618,
+ "5905": 3.92033,
+ "5910": 3.88782,
+ "5915": 3.85158,
+ "5920": 3.88999,
+ "5925": 3.82174,
+ "5930": 3.88478,
+ "5935": 3.86887,
+ "5940": 3.89924,
+ "5945": 3.90324,
+ "5950": 3.88472,
+ "5955": 3.83758,
+ "5960": 3.91077,
+ "5965": 3.85295,
+ "5970": 3.90592,
+ "5975": 3.87131,
+ "5980": 3.94635,
+ "5985": 3.81828,
+ "5990": 3.91445,
+ "5995": 3.82666,
+ "6000": 3.86389,
+ "6005": 3.82737,
+ "6010": 3.84638,
+ "6015": 3.82528,
+ "6020": 3.84213,
+ "6025": 3.8812,
+ "6030": 3.82864,
+ "6035": 3.87549,
+ "6040": 3.85371,
+ "6045": 3.88892,
+ "6050": 3.86125,
+ "6055": 3.84398,
+ "6060": 3.86538,
+ "6065": 3.8955,
+ "6070": 3.844,
+ "6075": 3.79156,
+ "6080": 3.86497,
+ "6085": 3.82767,
+ "6090": 3.86054,
+ "6095": 3.85995,
+ "6100": 3.82399,
+ "6105": 3.87238,
+ "6110": 3.80525,
+ "6115": 3.87931,
+ "6120": 3.85374,
+ "6125": 3.85469,
+ "6130": 3.85122,
+ "6135": 3.82709,
+ "6140": 3.8225,
+ "6145": 3.81264,
+ "6150": 3.85853,
+ "6155": 3.83605,
+ "6160": 3.80232,
+ "6165": 3.82292,
+ "6170": 3.81513,
+ "6175": 3.80691,
+ "6180": 3.8071,
+ "6185": 3.84448,
+ "6190": 3.81178,
+ "6195": 3.78014,
+ "6200": 3.80543,
+ "6205": 3.81219,
+ "6210": 3.77002,
+ "6215": 3.82559,
+ "6220": 3.822,
+ "6225": 3.82598,
+ "6230": 3.76955,
+ "6235": 3.8072,
+ "6240": 3.73374,
+ "6245": 3.84624,
+ "6250": 3.80845,
+ "6255": 3.8223,
+ "6260": 3.7948,
+ "6265": 3.82819,
+ "6270": 3.75673,
+ "6275": 3.78492,
+ "6280": 3.80313,
+ "6285": 3.78154,
+ "6290": 3.79976,
+ "6295": 3.80168,
+ "6300": 3.80756,
+ "6305": 3.88253,
+ "6310": 3.7702,
+ "6315": 3.7633,
+ "6320": 3.81817,
+ "6325": 3.75526,
+ "6330": 3.82862,
+ "6335": 3.81943,
+ "6340": 3.76721,
+ "6345": 3.82391,
+ "6350": 3.76718,
+ "6355": 3.77414,
+ "6360": 3.75111,
+ "6365": 3.80986,
+ "6370": 3.81014,
+ "6375": 3.78548,
+ "6380": 3.8065,
+ "6385": 3.82336,
+ "6390": 3.78289,
+ "6395": 3.75935,
+ "6400": 3.76038,
+ "6405": 3.83749,
+ "6410": 3.83127,
+ "6415": 3.7623,
+ "6420": 3.82306,
+ "6425": 3.83219,
+ "6430": 3.81048,
+ "6435": 3.77764,
+ "6440": 3.76108,
+ "6445": 3.80173,
+ "6450": 3.73884,
+ "6455": 3.75156,
+ "6460": 3.77352,
+ "6465": 3.80905,
+ "6470": 3.78701,
+ "6475": 3.78176,
+ "6480": 3.81548,
+ "6485": 3.76414,
+ "6490": 3.71291,
+ "6495": 3.81407,
+ "6500": 3.79809,
+ "6505": 3.72741,
+ "6510": 3.7976,
+ "6515": 3.81938,
+ "6520": 3.73166,
+ "6525": 3.80464,
+ "6530": 3.76853,
+ "6535": 3.76159,
+ "6540": 3.82675,
+ "6545": 3.76261,
+ "6550": 3.76963,
+ "6555": 3.75505,
+ "6560": 3.71108,
+ "6565": 3.70887,
+ "6570": 3.7465,
+ "6575": 3.69338,
+ "6580": 3.81517,
+ "6585": 3.76239,
+ "6590": 3.72546,
+ "6595": 3.74461,
+ "6600": 3.73687,
+ "6605": 3.71668,
+ "6610": 3.72679,
+ "6615": 3.76079,
+ "6620": 3.70966,
+ "6625": 3.72313,
+ "6630": 3.72114,
+ "6635": 3.76232,
+ "6640": 3.73374,
+ "6645": 3.75061,
+ "6650": 3.77922,
+ "6655": 3.70627,
+ "6660": 3.73531,
+ "6665": 3.7573,
+ "6670": 3.71979,
+ "6675": 3.74124,
+ "6680": 3.73477,
+ "6685": 3.76436,
+ "6690": 3.74256,
+ "6695": 3.75545,
+ "6700": 3.74559,
+ "6705": 3.72882,
+ "6710": 3.72913,
+ "6715": 3.69291,
+ "6720": 3.77736,
+ "6725": 3.75737,
+ "6730": 3.73993,
+ "6735": 3.74082,
+ "6740": 3.73806,
+ "6745": 3.72041,
+ "6750": 3.74412,
+ "6755": 3.69337,
+ "6760": 3.68122,
+ "6765": 3.74232,
+ "6770": 3.69625,
+ "6775": 3.74604,
+ "6780": 3.70485,
+ "6785": 3.70942,
+ "6790": 3.73683,
+ "6795": 3.69846,
+ "6800": 3.71752,
+ "6805": 3.72172,
+ "6810": 3.73628,
+ "6815": 3.65876,
+ "6820": 3.70229,
+ "6825": 3.72745,
+ "6830": 3.70872,
+ "6835": 3.68623,
+ "6840": 3.67517,
+ "6845": 3.74818,
+ "6850": 3.70405,
+ "6855": 3.73713,
+ "6860": 3.6695,
+ "6865": 3.73585,
+ "6870": 3.6953,
+ "6875": 3.69781,
+ "6880": 3.70324,
+ "6885": 3.67727,
+ "6890": 3.69236,
+ "6895": 3.67848,
+ "6900": 3.68133,
+ "6905": 3.68771,
+ "6910": 3.72919,
+ "6915": 3.73359,
+ "6920": 3.68934,
+ "6925": 3.69022,
+ "6930": 3.68858,
+ "6935": 3.62056,
+ "6940": 3.68927,
+ "6945": 3.67777,
+ "6950": 3.68038,
+ "6955": 3.6771,
+ "6960": 3.68108,
+ "6965": 3.72225,
+ "6970": 3.64603,
+ "6975": 3.72781,
+ "6980": 3.68459,
+ "6985": 3.68985,
+ "6990": 3.7316,
+ "6995": 3.70495,
+ "7000": 3.63993,
+ "7005": 3.71744,
+ "7010": 3.69223,
+ "7015": 3.67561,
+ "7020": 3.72152,
+ "7025": 3.70969,
+ "7030": 3.70236,
+ "7035": 3.65723,
+ "7040": 3.61488,
+ "7045": 3.69518,
+ "7050": 3.71947,
+ "7055": 3.64991,
+ "7060": 3.69149,
+ "7065": 3.74261,
+ "7070": 3.67108,
+ "7075": 3.67419,
+ "7080": 3.71683,
+ "7085": 3.64191,
+ "7090": 3.66318,
+ "7095": 3.63818,
+ "7100": 3.68341,
+ "7105": 3.62024,
+ "7110": 3.68873,
+ "7115": 3.63797,
+ "7120": 3.68741,
+ "7125": 3.63499,
+ "7130": 3.65311,
+ "7135": 3.66196,
+ "7140": 3.66504,
+ "7145": 3.68183,
+ "7150": 3.62677,
+ "7155": 3.69052,
+ "7160": 3.62415,
+ "7165": 3.64241,
+ "7170": 3.68231,
+ "7175": 3.64603,
+ "7180": 3.67571,
+ "7185": 3.70721,
+ "7190": 3.663,
+ "7195": 3.66862,
+ "7200": 3.67265,
+ "7205": 3.65833,
+ "7210": 3.68834,
+ "7215": 3.67282,
+ "7220": 3.69117,
+ "7225": 3.66107,
+ "7230": 3.68593,
+ "7235": 3.64823,
+ "7240": 3.64663,
+ "7245": 3.66574,
+ "7250": 3.60447,
+ "7255": 3.62598,
+ "7260": 3.68023,
+ "7265": 3.60288,
+ "7270": 3.63936,
+ "7275": 3.64805,
+ "7280": 3.62623,
+ "7285": 3.65053,
+ "7290": 3.6735,
+ "7295": 3.66357,
+ "7300": 3.62393,
+ "7305": 3.62784,
+ "7310": 3.66312,
+ "7315": 3.67632,
+ "7320": 3.65015,
+ "7325": 3.65453,
+ "7330": 3.62344,
+ "7335": 3.62574,
+ "7340": 3.64422,
+ "7345": 3.60533,
+ "7350": 3.65727,
+ "7355": 3.64352,
+ "7360": 3.61779,
+ "7365": 3.63578,
+ "7370": 3.6188,
+ "7375": 3.59366,
+ "7380": 3.64743,
+ "7385": 3.67218,
+ "7390": 3.65876,
+ "7395": 3.60688,
+ "7400": 3.65695,
+ "7405": 3.64945,
+ "7410": 3.66151,
+ "7415": 3.64439,
+ "7420": 3.63591,
+ "7425": 3.6844,
+ "7430": 3.63181,
+ "7435": 3.61154,
+ "7440": 3.62564,
+ "7445": 3.60843,
+ "7450": 3.57301,
+ "7455": 3.64772,
+ "7460": 3.63452,
+ "7465": 3.63169,
+ "7470": 3.63744,
+ "7475": 3.64264,
+ "7480": 3.61171,
+ "7485": 3.57567,
+ "7490": 3.57599,
+ "7495": 3.5863,
+ "7500": 3.61565,
+ "7505": 3.59614,
+ "7510": 3.55707,
+ "7515": 3.61683,
+ "7520": 3.60991,
+ "7525": 3.56658,
+ "7530": 3.61196,
+ "7535": 3.62507,
+ "7540": 3.61046,
+ "7545": 3.64639,
+ "7550": 3.65882,
+ "7555": 3.58595,
+ "7560": 3.60212,
+ "7565": 3.59782,
+ "7570": 3.60603,
+ "7575": 3.57351,
+ "7580": 3.62111,
+ "7585": 3.60137,
+ "7590": 3.6026,
+ "7595": 3.66318,
+ "7600": 3.6076,
+ "7605": 3.59626,
+ "7610": 3.58483,
+ "7615": 3.58478,
+ "7620": 3.56787,
+ "7625": 3.62193,
+ "7630": 3.60469,
+ "7635": 3.5928,
+ "7640": 3.59019,
+ "7645": 3.62279,
+ "7650": 3.6259,
+ "7655": 3.66371,
+ "7660": 3.5305,
+ "7665": 3.60545,
+ "7670": 3.59796,
+ "7675": 3.58201,
+ "7680": 3.57701,
+ "7685": 3.64556,
+ "7690": 3.59102,
+ "7695": 3.57063,
+ "7700": 3.63352,
+ "7705": 3.58816,
+ "7710": 3.62048,
+ "7715": 3.5764,
+ "7720": 3.65561,
+ "7725": 3.55706,
+ "7730": 3.57614,
+ "7735": 3.61006,
+ "7740": 3.58168,
+ "7745": 3.58454,
+ "7750": 3.57422,
+ "7755": 3.59202,
+ "7760": 3.56089,
+ "7765": 3.58551,
+ "7770": 3.60104,
+ "7775": 3.57103,
+ "7780": 3.55457,
+ "7785": 3.57713,
+ "7790": 3.57042,
+ "7795": 3.58792,
+ "7800": 3.57997,
+ "7805": 3.58361,
+ "7810": 3.60683,
+ "7815": 3.57773,
+ "7820": 3.57578,
+ "7825": 3.61835,
+ "7830": 3.59192,
+ "7835": 3.52632,
+ "7840": 3.6194,
+ "7845": 3.55538,
+ "7850": 3.51354,
+ "7855": 3.56599,
+ "7860": 3.54645,
+ "7865": 3.60369,
+ "7870": 3.54114,
+ "7875": 3.55695,
+ "7880": 3.572,
+ "7885": 3.56229,
+ "7890": 3.60585,
+ "7895": 3.59334,
+ "7900": 3.60641,
+ "7905": 3.56339,
+ "7910": 3.58203,
+ "7915": 3.58298,
+ "7920": 3.59012,
+ "7925": 3.5681,
+ "7930": 3.59927,
+ "7935": 3.56169,
+ "7940": 3.60948,
+ "7945": 3.62723,
+ "7950": 3.53708,
+ "7955": 3.54481,
+ "7960": 3.53124,
+ "7965": 3.51862,
+ "7970": 3.52486,
+ "7975": 3.55975,
+ "7980": 3.56722,
+ "7985": 3.54114,
+ "7990": 3.54399,
+ "7995": 3.5186,
+ "8000": 3.57756,
+ "8005": 3.54643,
+ "8010": 3.53705,
+ "8015": 3.53445,
+ "8020": 3.53111,
+ "8025": 3.51514,
+ "8030": 3.54148,
+ "8035": 3.53478,
+ "8040": 3.52163,
+ "8045": 3.57586,
+ "8050": 3.57789,
+ "8055": 3.54866,
+ "8060": 3.5712,
+ "8065": 3.54757,
+ "8070": 3.53654,
+ "8075": 3.52629,
+ "8080": 3.57467,
+ "8085": 3.52928,
+ "8090": 3.53424,
+ "8095": 3.56313,
+ "8100": 3.51543,
+ "8105": 3.54752,
+ "8110": 3.5453,
+ "8115": 3.51645,
+ "8120": 3.52703,
+ "8125": 3.56437,
+ "8130": 3.52567,
+ "8135": 3.53994,
+ "8140": 3.52104,
+ "8145": 3.50389,
+ "8150": 3.52394,
+ "8155": 3.51178,
+ "8160": 3.56129,
+ "8165": 3.54328,
+ "8170": 3.5116,
+ "8175": 3.5057,
+ "8180": 3.57245,
+ "8185": 3.54733,
+ "8190": 3.58207,
+ "8195": 3.55001,
+ "8200": 3.52156,
+ "8205": 3.52888,
+ "8210": 3.53558,
+ "8215": 3.55713,
+ "8220": 3.5201,
+ "8225": 3.51201,
+ "8230": 3.53756,
+ "8235": 3.55814,
+ "8240": 3.54052,
+ "8245": 3.53652,
+ "8250": 3.5692,
+ "8255": 3.51844,
+ "8260": 3.52912,
+ "8265": 3.52072,
+ "8270": 3.52843,
+ "8275": 3.51526,
+ "8280": 3.50321,
+ "8285": 3.52669,
+ "8290": 3.5272,
+ "8295": 3.49645,
+ "8300": 3.51721,
+ "8305": 3.53958,
+ "8310": 3.5351,
+ "8315": 3.50396,
+ "8320": 3.53046,
+ "8325": 3.47885,
+ "8330": 3.44388,
+ "8335": 3.51457,
+ "8340": 3.54076,
+ "8345": 3.49873,
+ "8350": 3.51134,
+ "8355": 3.54342,
+ "8360": 3.51607,
+ "8365": 3.53716,
+ "8370": 3.53127,
+ "8375": 3.48696,
+ "8380": 3.4848,
+ "8385": 3.52879,
+ "8390": 3.49474,
+ "8395": 3.52721,
+ "8400": 3.49636,
+ "8405": 3.51685,
+ "8410": 3.57651,
+ "8415": 3.48228,
+ "8420": 3.45216,
+ "8425": 3.53401,
+ "8430": 3.53787,
+ "8435": 3.47534,
+ "8440": 3.55163,
+ "8445": 3.53658,
+ "8450": 3.50995,
+ "8455": 3.52875,
+ "8460": 3.53463,
+ "8465": 3.4708,
+ "8470": 3.4929,
+ "8475": 3.55004,
+ "8480": 3.47555,
+ "8485": 3.49487,
+ "8490": 3.48489,
+ "8495": 3.48023,
+ "8500": 3.52888,
+ "8505": 3.46749,
+ "8510": 3.54064,
+ "8515": 3.48982,
+ "8520": 3.49184,
+ "8525": 3.42254,
+ "8530": 3.50181,
+ "8535": 3.52351,
+ "8540": 3.47484,
+ "8545": 3.49944,
+ "8550": 3.46881,
+ "8555": 3.53517,
+ "8560": 3.5346,
+ "8565": 3.48792,
+ "8570": 3.48883,
+ "8575": 3.46414,
+ "8580": 3.50837,
+ "8585": 3.52994,
+ "8590": 3.51956,
+ "8595": 3.52409,
+ "8600": 3.50319,
+ "8605": 3.49079,
+ "8610": 3.49584,
+ "8615": 3.49483,
+ "8620": 3.46525,
+ "8625": 3.4875,
+ "8630": 3.49269,
+ "8635": 3.47742,
+ "8640": 3.46288,
+ "8645": 3.52844,
+ "8650": 3.45936,
+ "8655": 3.50294,
+ "8660": 3.51093,
+ "8665": 3.48996,
+ "8670": 3.50547,
+ "8675": 3.47414,
+ "8680": 3.4685,
+ "8685": 3.48029,
+ "8690": 3.51264,
+ "8695": 3.51367,
+ "8700": 3.48324,
+ "8705": 3.45351,
+ "8710": 3.50031,
+ "8715": 3.45042,
+ "8720": 3.52876,
+ "8725": 3.48819,
+ "8730": 3.47981,
+ "8735": 3.51018,
+ "8740": 3.46013,
+ "8745": 3.50108,
+ "8750": 3.50543,
+ "8755": 3.46564,
+ "8760": 3.48373,
+ "8765": 3.43955,
+ "8770": 3.50951,
+ "8775": 3.47313,
+ "8780": 3.45782,
+ "8785": 3.47628,
+ "8790": 3.4608,
+ "8795": 3.49675,
+ "8800": 3.46402,
+ "8805": 3.43267,
+ "8810": 3.45044,
+ "8815": 3.47281,
+ "8820": 3.43586,
+ "8825": 3.46906,
+ "8830": 3.44494,
+ "8835": 3.42402,
+ "8840": 3.4361,
+ "8845": 3.45772,
+ "8850": 3.48143,
+ "8855": 3.46505,
+ "8860": 3.53187,
+ "8865": 3.46882,
+ "8870": 3.44869,
+ "8875": 3.45286,
+ "8880": 3.45584,
+ "8885": 3.44986,
+ "8890": 3.47298,
+ "8895": 3.45131,
+ "8900": 3.47879,
+ "8905": 3.46796,
+ "8910": 3.45421,
+ "8915": 3.44293,
+ "8920": 3.43345,
+ "8925": 3.50917,
+ "8930": 3.49052,
+ "8935": 3.50073,
+ "8940": 3.47584,
+ "8945": 3.47848,
+ "8950": 3.45717,
+ "8955": 3.44615,
+ "8960": 3.43965,
+ "8965": 3.45818,
+ "8970": 3.47179,
+ "8975": 3.42177,
+ "8980": 3.42266,
+ "8985": 3.44671,
+ "8990": 3.50075,
+ "8995": 3.47255,
+ "9000": 3.41954,
+ "9005": 3.46563,
+ "9010": 3.51573,
+ "9015": 3.4185,
+ "9020": 3.43896,
+ "9025": 3.44768,
+ "9030": 3.4718,
+ "9035": 3.37943,
+ "9040": 3.45501,
+ "9045": 3.45466,
+ "9050": 3.49179,
+ "9055": 3.40312,
+ "9060": 3.49477,
+ "9065": 3.51349,
+ "9070": 3.44713,
+ "9075": 3.47746,
+ "9080": 3.47127,
+ "9085": 3.47459,
+ "9090": 3.46668,
+ "9095": 3.42167,
+ "9100": 3.4227,
+ "9105": 3.41261,
+ "9110": 3.45663,
+ "9115": 3.46481,
+ "9120": 3.51949,
+ "9125": 3.44245,
+ "9130": 3.43654,
+ "9135": 3.46008,
+ "9140": 3.47929,
+ "9145": 3.42408,
+ "9150": 3.44307,
+ "9155": 3.45089,
+ "9160": 3.44998,
+ "9165": 3.45651,
+ "9170": 3.47508,
+ "9175": 3.41133,
+ "9180": 3.45323,
+ "9185": 3.41086,
+ "9190": 3.46875,
+ "9195": 3.43315,
+ "9200": 3.44758,
+ "9205": 3.42373,
+ "9210": 3.45572,
+ "9215": 3.39585,
+ "9220": 3.42327,
+ "9225": 3.44665,
+ "9230": 3.37357,
+ "9235": 3.39456,
+ "9240": 3.42282,
+ "9245": 3.40683,
+ "9250": 3.40791,
+ "9255": 3.42077,
+ "9260": 3.39755,
+ "9265": 3.44216,
+ "9270": 3.40754,
+ "9275": 3.42864,
+ "9280": 3.44334,
+ "9285": 3.44087,
+ "9290": 3.45563,
+ "9295": 3.44456,
+ "9300": 3.39522,
+ "9305": 3.42638,
+ "9310": 3.41593,
+ "9315": 3.38278,
+ "9320": 3.3797,
+ "9325": 3.42046,
+ "9330": 3.47853,
+ "9335": 3.38962,
+ "9340": 3.4706,
+ "9345": 3.46224,
+ "9350": 3.42735,
+ "9355": 3.39326,
+ "9360": 3.4165,
+ "9365": 3.41212,
+ "9370": 3.46155,
+ "9375": 3.42622,
+ "9380": 3.36413,
+ "9385": 3.43469,
+ "9390": 3.44403,
+ "9395": 3.45465,
+ "9400": 3.41582,
+ "9405": 3.40031,
+ "9410": 3.43744,
+ "9415": 3.42574,
+ "9420": 3.40295,
+ "9425": 3.42063,
+ "9430": 3.3935,
+ "9435": 3.41529,
+ "9440": 3.40125,
+ "9445": 3.39961,
+ "9450": 3.39469,
+ "9455": 3.4008,
+ "9460": 3.46489,
+ "9465": 3.46303,
+ "9470": 3.40478,
+ "9475": 3.45335,
+ "9480": 3.40789,
+ "9485": 3.3998,
+ "9490": 3.41154,
+ "9495": 3.44387,
+ "9500": 3.40535,
+ "9505": 3.37735,
+ "9510": 3.41645,
+ "9515": 3.41113,
+ "9520": 3.43045,
+ "9525": 3.40102,
+ "9530": 3.40027,
+ "9535": 3.42216
+ }
+ },
+ "iteration-time": {
+ "start_step": 1,
+ "end_step": 9535,
+ "step_interval": 5,
+ "values": {
+ "1": 241.22832,
+ "5": 11.6467,
+ "10": 11.59177,
+ "15": 11.54982,
+ "20": 11.50554,
+ "25": 11.48401,
+ "30": 11.47019,
+ "35": 11.4638,
+ "40": 11.44621,
+ "45": 11.45505,
+ "50": 11.48551,
+ "55": 11.47505,
+ "60": 11.46559,
+ "65": 11.69276,
+ "70": 11.51491,
+ "75": 11.58841,
+ "80": 11.59402,
+ "85": 11.55505,
+ "90": 11.57827,
+ "95": 11.6084,
+ "100": 11.72328,
+ "105": 11.84735,
+ "110": 11.81445,
+ "115": 12.01469,
+ "120": 12.27052,
+ "125": 12.40894,
+ "130": 12.32306,
+ "135": 12.6537,
+ "140": 12.87941,
+ "145": 12.87274,
+ "150": 13.17646,
+ "155": 13.42132,
+ "160": 13.29203,
+ "165": 13.33468,
+ "170": 13.38365,
+ "175": 13.29143,
+ "180": 13.37704,
+ "185": 13.17491,
+ "190": 13.2207,
+ "195": 13.0407,
+ "200": 13.03378,
+ "205": 12.93499,
+ "210": 12.93302,
+ "215": 12.83429,
+ "220": 12.77504,
+ "225": 12.71437,
+ "230": 12.67462,
+ "235": 12.7241,
+ "240": 12.78341,
+ "245": 12.61372,
+ "250": 12.60968,
+ "255": 12.49502,
+ "260": 12.38655,
+ "265": 12.35372,
+ "270": 12.32939,
+ "275": 12.25213,
+ "280": 12.23412,
+ "285": 12.25047,
+ "290": 12.1386,
+ "295": 12.11066,
+ "300": 12.11487,
+ "305": 12.08746,
+ "310": 12.06842,
+ "315": 12.13334,
+ "320": 12.12044,
+ "325": 12.01351,
+ "330": 11.97276,
+ "335": 11.951,
+ "340": 11.97582,
+ "345": 11.94178,
+ "350": 11.90942,
+ "355": 11.9474,
+ "360": 11.94231,
+ "365": 11.91539,
+ "370": 11.89051,
+ "375": 11.87871,
+ "380": 11.8539,
+ "385": 11.81422,
+ "390": 11.82072,
+ "395": 11.85516,
+ "400": 11.8322,
+ "405": 11.81286,
+ "410": 11.81008,
+ "415": 11.76854,
+ "420": 11.7721,
+ "425": 11.7287,
+ "430": 11.80281,
+ "435": 11.76948,
+ "440": 11.78237,
+ "445": 11.81223,
+ "450": 11.76024,
+ "455": 11.83905,
+ "460": 11.86797,
+ "465": 11.88193,
+ "470": 11.94544,
+ "475": 12.03403,
+ "480": 11.8718,
+ "485": 11.96463,
+ "490": 11.9543,
+ "495": 11.99738,
+ "500": 12.06608,
+ "505": 12.04813,
+ "510": 12.09706,
+ "515": 12.14335,
+ "520": 12.36581,
+ "525": 12.19115,
+ "530": 12.1887,
+ "535": 12.25354,
+ "540": 12.27902,
+ "545": 12.32347,
+ "550": 12.44366,
+ "555": 12.25807,
+ "560": 12.22369,
+ "565": 12.28956,
+ "570": 12.31572,
+ "575": 12.28835,
+ "580": 12.33571,
+ "585": 12.26567,
+ "590": 12.30079,
+ "595": 12.29151,
+ "600": 12.30023,
+ "605": 12.45501,
+ "610": 12.27373,
+ "615": 12.217,
+ "620": 12.22334,
+ "625": 12.21274,
+ "630": 12.21904,
+ "635": 12.20277,
+ "640": 12.25538,
+ "645": 12.19988,
+ "650": 12.14026,
+ "655": 12.14302,
+ "660": 12.14678,
+ "665": 12.13972,
+ "670": 12.11485,
+ "675": 12.0282,
+ "680": 12.01901,
+ "685": 11.98462,
+ "690": 11.98742,
+ "695": 11.95917,
+ "700": 11.92521,
+ "705": 18.38779,
+ "710": 11.92438,
+ "715": 11.8274,
+ "720": 11.90138,
+ "725": 11.84998,
+ "730": 11.83009,
+ "735": 11.89248,
+ "740": 11.82364,
+ "745": 11.91839,
+ "750": 11.9577,
+ "755": 11.85056,
+ "760": 11.90523,
+ "765": 11.9116,
+ "770": 11.83717,
+ "775": 12.05864,
+ "780": 11.84895,
+ "785": 11.84375,
+ "790": 11.86493,
+ "795": 11.85763,
+ "800": 11.94365,
+ "805": 11.86899,
+ "810": 11.86748,
+ "815": 11.86393,
+ "820": 11.87992,
+ "825": 11.85259,
+ "830": 11.86886,
+ "835": 11.8517,
+ "840": 11.86254,
+ "845": 11.89508,
+ "850": 11.85613,
+ "855": 11.87434,
+ "860": 11.90703,
+ "865": 11.83224,
+ "870": 11.88246,
+ "875": 11.9305,
+ "880": 11.96022,
+ "885": 11.81651,
+ "890": 12.06642,
+ "895": 11.92653,
+ "900": 11.86469,
+ "905": 12.01767,
+ "910": 11.89635,
+ "915": 11.8254,
+ "920": 11.86106,
+ "925": 11.88434,
+ "930": 11.97059,
+ "935": 12.03718,
+ "940": 11.87698,
+ "945": 11.88008,
+ "950": 12.02071,
+ "955": 11.84843,
+ "960": 244.37245,
+ "965": 12.32084,
+ "970": 11.86341,
+ "975": 12.01988,
+ "980": 11.92166,
+ "985": 11.85411,
+ "990": 11.87753,
+ "995": 11.84786,
+ "1000": 11.89892,
+ "1005": 11.99759,
+ "1010": 11.91045,
+ "1015": 11.87038,
+ "1020": 11.85674,
+ "1025": 11.85567,
+ "1030": 11.86674,
+ "1035": 11.92499,
+ "1040": 11.85969,
+ "1045": 12.04929,
+ "1050": 11.82341,
+ "1055": 11.83111,
+ "1060": 11.87567,
+ "1065": 11.84584,
+ "1070": 11.93603,
+ "1075": 11.87121,
+ "1080": 11.85935,
+ "1085": 11.88667,
+ "1090": 11.86058,
+ "1095": 11.86482,
+ "1100": 11.82375,
+ "1105": 11.86482,
+ "1110": 11.89668,
+ "1115": 11.94941,
+ "1120": 11.84941,
+ "1125": 11.94466,
+ "1130": 11.90846,
+ "1135": 11.8602,
+ "1140": 11.86926,
+ "1145": 11.90365,
+ "1150": 11.88788,
+ "1155": 11.81781,
+ "1160": 11.88464,
+ "1165": 11.85344,
+ "1170": 11.8865,
+ "1175": 11.93361,
+ "1180": 11.89647,
+ "1185": 11.9031,
+ "1190": 11.89287,
+ "1195": 11.88683,
+ "1200": 11.85927,
+ "1205": 11.92471,
+ "1210": 11.85592,
+ "1215": 17.4276,
+ "1220": 11.87359,
+ "1225": 11.9296,
+ "1230": 11.95025,
+ "1235": 11.90738,
+ "1240": 11.86621,
+ "1245": 11.98001,
+ "1250": 12.003,
+ "1255": 11.91396,
+ "1260": 11.92279,
+ "1265": 11.85195,
+ "1270": 11.87463,
+ "1275": 11.90307,
+ "1280": 11.84637,
+ "1285": 11.95883,
+ "1290": 11.88039,
+ "1295": 11.8399,
+ "1300": 11.81976,
+ "1305": 11.89766,
+ "1310": 11.91584,
+ "1315": 12.12571,
+ "1320": 12.05556,
+ "1325": 11.84679,
+ "1330": 11.94985,
+ "1335": 11.94039,
+ "1340": 12.00572,
+ "1345": 11.98268,
+ "1350": 12.15927,
+ "1355": 12.04312,
+ "1360": 11.98816,
+ "1365": 11.95737,
+ "1370": 11.92395,
+ "1375": 11.89595,
+ "1380": 11.88635,
+ "1385": 11.96617,
+ "1390": 11.87421,
+ "1395": 12.02833,
+ "1400": 11.87415,
+ "1405": 11.85875,
+ "1410": 11.85419,
+ "1415": 11.8978,
+ "1420": 11.86309,
+ "1425": 11.87505,
+ "1430": 12.10339,
+ "1435": 11.88151,
+ "1440": 12.15068,
+ "1445": 11.98493,
+ "1450": 11.95438,
+ "1455": 12.03808,
+ "1460": 11.85293,
+ "1465": 11.93176,
+ "1470": 11.92246,
+ "1475": 11.90448,
+ "1480": 11.98959,
+ "1485": 11.93685,
+ "1490": 11.92389,
+ "1495": 11.95047,
+ "1500": 11.94526,
+ "1505": 11.9086,
+ "1510": 11.95225,
+ "1515": 11.87405,
+ "1520": 11.87975,
+ "1525": 11.88264,
+ "1530": 12.04989,
+ "1535": 12.02942,
+ "1540": 11.93089,
+ "1545": 11.89376,
+ "1550": 11.88596,
+ "1555": 11.95001,
+ "1560": 11.90239,
+ "1565": 11.89699,
+ "1570": 11.91441,
+ "1575": 11.87813,
+ "1580": 11.86939,
+ "1585": 11.8566,
+ "1590": 11.8665,
+ "1595": 11.90861,
+ "1600": 11.90425,
+ "1605": 11.82248,
+ "1610": 11.86531,
+ "1615": 11.8796,
+ "1620": 11.87587,
+ "1625": 11.88944,
+ "1630": 11.88839,
+ "1635": 11.8307,
+ "1640": 11.87082,
+ "1645": 11.84687,
+ "1650": 11.87887,
+ "1655": 11.85709,
+ "1660": 11.85167,
+ "1665": 11.90284,
+ "1670": 11.85205,
+ "1675": 12.00742,
+ "1680": 11.90754,
+ "1685": 11.97458,
+ "1690": 11.97016,
+ "1695": 11.9189,
+ "1700": 11.89709,
+ "1705": 11.88042,
+ "1710": 11.87879,
+ "1715": 12.06779,
+ "1720": 11.98631,
+ "1725": 12.01044,
+ "1730": 11.9924,
+ "1735": 11.87648,
+ "1740": 11.87455,
+ "1745": 11.93461,
+ "1750": 11.90235,
+ "1755": 11.97053,
+ "1760": 11.89545,
+ "1765": 11.8564,
+ "1770": 11.92635,
+ "1775": 11.91815,
+ "1780": 11.91235,
+ "1785": 11.85546,
+ "1790": 11.93087,
+ "1795": 11.91138,
+ "1800": 11.95901,
+ "1805": 12.0529,
+ "1810": 11.98858,
+ "1815": 12.13997,
+ "1820": 11.94798,
+ "1825": 11.97682,
+ "1830": 11.91244,
+ "1835": 11.94888,
+ "1840": 11.93666,
+ "1845": 11.87312,
+ "1850": 11.86327,
+ "1855": 11.94769,
+ "1860": 12.00187,
+ "1865": 12.06916,
+ "1870": 11.99528,
+ "1875": 11.89416,
+ "1880": 12.02292,
+ "1885": 12.04249,
+ "1890": 11.94094,
+ "1895": 11.93619,
+ "1900": 11.95301,
+ "1905": 11.85793,
+ "1910": 11.96264,
+ "1915": 11.92826,
+ "1920": 11.94216,
+ "1925": 12.01307,
+ "1930": 11.98891,
+ "1935": 11.95834,
+ "1940": 11.92143,
+ "1945": 11.98459,
+ "1950": 16.97099,
+ "1955": 11.89147,
+ "1960": 11.94643,
+ "1965": 11.92486,
+ "1970": 11.91542,
+ "1975": 13.09741,
+ "1980": 12.02148,
+ "1985": 11.92812,
+ "1990": 12.01102,
+ "1995": 11.94891,
+ "2000": 12.06741,
+ "2005": 11.94166,
+ "2010": 11.95871,
+ "2015": 12.00042,
+ "2020": 11.99101,
+ "2025": 11.95463,
+ "2030": 12.36755,
+ "2035": 11.96199,
+ "2040": 11.97863,
+ "2045": 12.01033,
+ "2050": 12.0643,
+ "2055": 11.96928,
+ "2060": 11.98383,
+ "2065": 11.92648,
+ "2070": 11.92379,
+ "2075": 11.97669,
+ "2080": 11.95508,
+ "2085": 11.94472,
+ "2090": 11.9663,
+ "2095": 11.93695,
+ "2100": 11.97178,
+ "2105": 11.98764,
+ "2110": 11.9516,
+ "2115": 11.9215,
+ "2120": 11.95207,
+ "2125": 11.95947,
+ "2130": 11.96722,
+ "2135": 11.97924,
+ "2140": 11.88777,
+ "2145": 11.95546,
+ "2150": 11.90266,
+ "2155": 11.97573,
+ "2160": 11.93275,
+ "2165": 11.98593,
+ "2170": 11.9842,
+ "2175": 12.00145,
+ "2180": 11.99219,
+ "2185": 11.96424,
+ "2190": 11.94313,
+ "2195": 11.93489,
+ "2200": 11.94356,
+ "2205": 12.00157,
+ "2210": 11.97153,
+ "2215": 11.9563,
+ "2220": 12.14117,
+ "2225": 11.97066,
+ "2230": 12.00037,
+ "2235": 11.95279,
+ "2240": 11.9544,
+ "2245": 11.97031,
+ "2250": 11.92229,
+ "2255": 11.98097,
+ "2260": 11.96529,
+ "2265": 11.98619,
+ "2270": 12.02117,
+ "2275": 11.94865,
+ "2280": 12.02569,
+ "2285": 11.98203,
+ "2290": 12.10479,
+ "2295": 11.95346,
+ "2300": 11.99961,
+ "2305": 11.96025,
+ "2310": 11.98746,
+ "2315": 11.95209,
+ "2320": 12.02644,
+ "2325": 11.95369,
+ "2330": 11.91985,
+ "2335": 11.93244,
+ "2340": 11.97061,
+ "2345": 11.90115,
+ "2350": 11.99136,
+ "2355": 12.0541,
+ "2360": 12.03728,
+ "2365": 11.95319,
+ "2370": 11.8917,
+ "2375": 11.94629,
+ "2380": 11.9087,
+ "2385": 11.91696,
+ "2390": 11.90123,
+ "2395": 11.87998,
+ "2400": 12.02954,
+ "2405": 11.97917,
+ "2410": 11.98456,
+ "2415": 11.9575,
+ "2420": 11.95917,
+ "2425": 11.95788,
+ "2430": 11.99944,
+ "2435": 12.00043,
+ "2440": 11.91339,
+ "2445": 11.97889,
+ "2450": 11.93997,
+ "2455": 11.91834,
+ "2460": 11.98321,
+ "2465": 11.94509,
+ "2470": 11.93387,
+ "2475": 11.9562,
+ "2480": 11.93148,
+ "2485": 11.94432,
+ "2490": 11.95477,
+ "2495": 11.94334,
+ "2500": 11.9284,
+ "2505": 11.93757,
+ "2510": 11.92289,
+ "2515": 11.97869,
+ "2520": 11.94858,
+ "2525": 11.96606,
+ "2530": 11.90894,
+ "2535": 11.95425,
+ "2540": 11.89136,
+ "2545": 11.94553,
+ "2550": 11.98026,
+ "2555": 11.93376,
+ "2560": 11.94866,
+ "2565": 11.92767,
+ "2570": 11.93583,
+ "2575": 11.97284,
+ "2580": 11.98911,
+ "2585": 11.95484,
+ "2590": 11.96399,
+ "2595": 11.96211,
+ "2600": 11.93906,
+ "2605": 11.9733,
+ "2610": 12.01872,
+ "2615": 11.99897,
+ "2620": 11.90926,
+ "2625": 11.93248,
+ "2630": 11.92842,
+ "2635": 11.94338,
+ "2640": 11.94678,
+ "2645": 11.95901,
+ "2650": 11.9296,
+ "2655": 12.02405,
+ "2660": 12.0166,
+ "2665": 12.01166,
+ "2670": 11.90595,
+ "2675": 11.98569,
+ "2680": 12.0118,
+ "2685": 11.92029,
+ "2690": 11.93111,
+ "2695": 12.00369,
+ "2700": 11.94818,
+ "2705": 11.99119,
+ "2710": 11.93978,
+ "2715": 11.9296,
+ "2720": 11.93044,
+ "2725": 11.94343,
+ "2730": 12.02248,
+ "2735": 11.95389,
+ "2740": 11.94611,
+ "2745": 11.92776,
+ "2750": 11.91647,
+ "2755": 11.9522,
+ "2760": 11.95012,
+ "2765": 11.96707,
+ "2770": 11.94892,
+ "2775": 11.9867,
+ "2780": 11.96897,
+ "2785": 11.97268,
+ "2790": 12.01936,
+ "2795": 11.97259,
+ "2800": 12.01028,
+ "2805": 11.94892,
+ "2810": 12.04828,
+ "2815": 11.93469,
+ "2820": 11.94568,
+ "2825": 11.92529,
+ "2830": 11.97458,
+ "2835": 11.99475,
+ "2840": 11.94984,
+ "2845": 11.93356,
+ "2850": 12.05796,
+ "2855": 11.99065,
+ "2860": 11.96077,
+ "2865": 11.9377,
+ "2870": 11.97627,
+ "2875": 11.97986,
+ "2880": 11.97201,
+ "2885": 11.91879,
+ "2890": 11.93586,
+ "2895": 12.00661,
+ "2900": 11.94616,
+ "2905": 11.94376,
+ "2910": 11.94168,
+ "2915": 11.94867,
+ "2920": 11.99355,
+ "2925": 11.94779,
+ "2930": 11.97133,
+ "2935": 11.96256,
+ "2940": 11.97787,
+ "2945": 11.93759,
+ "2950": 11.91863,
+ "2955": 11.98973,
+ "2960": 12.00486,
+ "2965": 11.91623,
+ "2970": 11.94846,
+ "2975": 11.91534,
+ "2980": 11.97787,
+ "2985": 12.385,
+ "2990": 11.88498,
+ "2995": 11.92173,
+ "3000": 11.90561,
+ "3005": 11.86795,
+ "3010": 11.88075,
+ "3015": 11.87833,
+ "3020": 11.98777,
+ "3025": 11.90078,
+ "3030": 11.98251,
+ "3035": 11.92211,
+ "3040": 11.91067,
+ "3045": 12.04371,
+ "3050": 11.91886,
+ "3055": 11.952,
+ "3060": 11.90649,
+ "3065": 11.86917,
+ "3070": 11.86601,
+ "3075": 11.92435,
+ "3080": 11.98092,
+ "3085": 11.94809,
+ "3090": 12.20304,
+ "3095": 11.87329,
+ "3100": 11.92696,
+ "3105": 11.85799,
+ "3110": 11.84125,
+ "3115": 11.82558,
+ "3120": 11.87566,
+ "3125": 11.89426,
+ "3130": 11.85869,
+ "3135": 11.92893,
+ "3140": 11.97022,
+ "3145": 11.84939,
+ "3150": 11.9785,
+ "3155": 11.92499,
+ "3160": 11.8889,
+ "3165": 11.87938,
+ "3170": 11.95555,
+ "3175": 11.91883,
+ "3180": 11.85842,
+ "3185": 11.9325,
+ "3190": 11.86061,
+ "3195": 11.90479,
+ "3200": 11.85963,
+ "3205": 11.91214,
+ "3210": 11.9243,
+ "3215": 11.8472,
+ "3220": 11.86665,
+ "3225": 11.89836,
+ "3230": 11.86299,
+ "3235": 11.89396,
+ "3240": 11.87482,
+ "3245": 11.86774,
+ "3250": 11.86673,
+ "3255": 11.88133,
+ "3260": 11.9014,
+ "3265": 11.92289,
+ "3270": 11.98401,
+ "3275": 11.95198,
+ "3280": 11.87392,
+ "3285": 11.89268,
+ "3290": 11.88963,
+ "3295": 11.91043,
+ "3300": 11.89803,
+ "3305": 11.87011,
+ "3310": 11.84465,
+ "3315": 11.84015,
+ "3320": 11.88334,
+ "3325": 11.93368,
+ "3330": 11.83472,
+ "3335": 11.86862,
+ "3340": 11.87575,
+ "3345": 11.94875,
+ "3350": 11.93528,
+ "3355": 11.81967,
+ "3360": 11.95954,
+ "3365": 11.88024,
+ "3370": 11.88333,
+ "3375": 11.85751,
+ "3380": 11.88742,
+ "3385": 11.9179,
+ "3390": 11.83242,
+ "3395": 11.96084,
+ "3400": 11.88213,
+ "3405": 11.86112,
+ "3410": 11.8407,
+ "3415": 11.92255,
+ "3420": 11.91997,
+ "3425": 11.88372,
+ "3430": 11.8672,
+ "3435": 11.85235,
+ "3440": 11.84935,
+ "3445": 11.93228,
+ "3450": 11.85166,
+ "3455": 11.9026,
+ "3460": 11.99596,
+ "3465": 11.88838,
+ "3470": 11.90065,
+ "3475": 11.92033,
+ "3480": 11.87265,
+ "3485": 11.89235,
+ "3490": 11.89267,
+ "3495": 11.97544,
+ "3500": 11.92819,
+ "3505": 11.82459,
+ "3510": 11.90756,
+ "3515": 11.92021,
+ "3520": 11.88124,
+ "3525": 11.86983,
+ "3530": 11.90548,
+ "3535": 11.94666,
+ "3540": 11.93322,
+ "3545": 11.90904,
+ "3550": 11.85224,
+ "3555": 11.886,
+ "3560": 11.93583,
+ "3565": 11.87294,
+ "3570": 11.86107,
+ "3575": 11.83618,
+ "3580": 11.94649,
+ "3585": 11.8886,
+ "3590": 12.01796,
+ "3595": 11.86065,
+ "3600": 11.96008,
+ "3605": 11.94154,
+ "3610": 11.91928,
+ "3615": 11.88551,
+ "3620": 11.8865,
+ "3625": 11.86807,
+ "3630": 11.98152,
+ "3635": 11.87685,
+ "3640": 11.89995,
+ "3645": 11.86485,
+ "3650": 11.94291,
+ "3655": 11.86472,
+ "3660": 11.84946,
+ "3665": 11.90789,
+ "3670": 11.86396,
+ "3675": 12.07226,
+ "3680": 11.8654,
+ "3685": 11.90154,
+ "3690": 11.87282,
+ "3695": 11.84993,
+ "3700": 11.92847,
+ "3705": 11.85848,
+ "3710": 11.86691,
+ "3715": 11.93176,
+ "3720": 11.86996,
+ "3725": 11.92665,
+ "3730": 11.90876,
+ "3735": 11.83597,
+ "3740": 11.8819,
+ "3745": 11.90119,
+ "3750": 11.90765,
+ "3755": 11.89791,
+ "3760": 11.91124,
+ "3765": 11.95606,
+ "3770": 11.93789,
+ "3775": 11.87152,
+ "3780": 11.89754,
+ "3785": 11.8704,
+ "3790": 11.88079,
+ "3795": 11.89363,
+ "3800": 11.88641,
+ "3805": 11.87724,
+ "3810": 11.86303,
+ "3815": 11.96793,
+ "3820": 11.97071,
+ "3825": 11.90678,
+ "3830": 11.84478,
+ "3835": 11.86339,
+ "3840": 11.84359,
+ "3845": 11.85381,
+ "3850": 11.89843,
+ "3855": 11.83659,
+ "3860": 11.8253,
+ "3865": 11.82796,
+ "3870": 11.93815,
+ "3875": 11.87584,
+ "3880": 11.85716,
+ "3885": 11.85848,
+ "3890": 11.84472,
+ "3895": 11.85001,
+ "3900": 11.90416,
+ "3905": 11.87723,
+ "3910": 11.90409,
+ "3915": 11.88375,
+ "3920": 11.9526,
+ "3925": 11.8796,
+ "3930": 11.92607,
+ "3935": 12.02111,
+ "3940": 11.89989,
+ "3945": 11.96829,
+ "3950": 11.92362,
+ "3955": 11.91298,
+ "3960": 11.93391,
+ "3965": 11.9977,
+ "3970": 11.91134,
+ "3975": 11.87698,
+ "3980": 11.84039,
+ "3985": 11.8296,
+ "3990": 11.8824,
+ "3995": 12.03103,
+ "4000": 12.53061,
+ "4005": 11.99032,
+ "4010": 11.94569,
+ "4015": 12.02459,
+ "4020": 12.05098,
+ "4025": 11.9408,
+ "4030": 11.9872,
+ "4035": 11.91882,
+ "4040": 11.91053,
+ "4045": 11.94764,
+ "4050": 11.96252,
+ "4055": 11.92924,
+ "4060": 11.95584,
+ "4065": 11.96477,
+ "4070": 11.95333,
+ "4075": 11.95009,
+ "4080": 11.94196,
+ "4085": 11.96679,
+ "4090": 12.09863,
+ "4095": 12.09521,
+ "4100": 11.99854,
+ "4105": 12.05345,
+ "4110": 11.99127,
+ "4115": 12.05731,
+ "4120": 11.95072,
+ "4125": 12.09249,
+ "4130": 12.04972,
+ "4135": 11.892,
+ "4140": 11.93048,
+ "4145": 11.92862,
+ "4150": 12.00088,
+ "4155": 11.95542,
+ "4160": 12.01499,
+ "4165": 11.90691,
+ "4170": 11.99204,
+ "4175": 12.02661,
+ "4180": 12.08762,
+ "4185": 11.93626,
+ "4190": 11.96513,
+ "4195": 11.9247,
+ "4200": 11.89449,
+ "4205": 11.95353,
+ "4210": 11.90984,
+ "4215": 11.92857,
+ "4220": 11.99809,
+ "4225": 12.01358,
+ "4230": 12.00065,
+ "4235": 11.95146,
+ "4240": 12.12674,
+ "4245": 11.99718,
+ "4250": 11.98808,
+ "4255": 11.95388,
+ "4260": 11.91437,
+ "4265": 11.97358,
+ "4270": 11.99013,
+ "4275": 11.95746,
+ "4280": 11.9273,
+ "4285": 11.92873,
+ "4290": 11.94103,
+ "4295": 11.93054,
+ "4300": 11.92986,
+ "4305": 12.11627,
+ "4310": 11.95471,
+ "4315": 11.96985,
+ "4320": 12.03911,
+ "4325": 12.01041,
+ "4330": 11.93084,
+ "4335": 11.95171,
+ "4340": 12.03209,
+ "4345": 11.94503,
+ "4350": 11.95426,
+ "4355": 12.08714,
+ "4360": 12.18212,
+ "4365": 11.94575,
+ "4370": 11.96598,
+ "4375": 12.00939,
+ "4380": 12.08808,
+ "4385": 11.9772,
+ "4390": 12.02704,
+ "4395": 12.01062,
+ "4400": 11.94619,
+ "4405": 11.98609,
+ "4410": 11.98025,
+ "4415": 11.99156,
+ "4420": 11.96913,
+ "4425": 12.02991,
+ "4430": 11.98417,
+ "4435": 12.07654,
+ "4440": 12.09429,
+ "4445": 11.9962,
+ "4450": 11.91032,
+ "4455": 11.99724,
+ "4460": 11.94549,
+ "4465": 11.92313,
+ "4470": 11.98709,
+ "4475": 11.9946,
+ "4480": 12.041,
+ "4485": 11.98684,
+ "4490": 12.00793,
+ "4495": 11.96519,
+ "4500": 11.91768,
+ "4505": 11.93855,
+ "4510": 11.96344,
+ "4515": 11.93266,
+ "4520": 11.99772,
+ "4525": 12.00265,
+ "4530": 12.00144,
+ "4535": 11.93099,
+ "4540": 11.9976,
+ "4545": 12.04415,
+ "4550": 11.92104,
+ "4555": 11.97762,
+ "4560": 12.05513,
+ "4565": 12.08413,
+ "4570": 12.00561,
+ "4575": 12.03402,
+ "4580": 12.07435,
+ "4585": 11.91157,
+ "4590": 11.93266,
+ "4595": 12.00575,
+ "4600": 11.98764,
+ "4605": 12.07608,
+ "4610": 11.98608,
+ "4615": 12.23058,
+ "4620": 11.96992,
+ "4625": 11.98931,
+ "4630": 11.92725,
+ "4635": 11.94909,
+ "4640": 11.94336,
+ "4645": 11.95955,
+ "4650": 11.99978,
+ "4655": 11.95199,
+ "4660": 11.97643,
+ "4665": 12.03686,
+ "4670": 12.0499,
+ "4675": 11.98439,
+ "4680": 12.00394,
+ "4685": 11.97515,
+ "4690": 11.95102,
+ "4695": 12.07552,
+ "4700": 11.9222,
+ "4705": 11.97387,
+ "4710": 11.99203,
+ "4715": 11.93004,
+ "4720": 11.97237,
+ "4725": 12.00277,
+ "4730": 12.00835,
+ "4735": 11.97435,
+ "4740": 11.98233,
+ "4745": 11.92423,
+ "4750": 11.95154,
+ "4755": 12.02084,
+ "4760": 11.94378,
+ "4765": 11.95313,
+ "4770": 11.92338,
+ "4775": 11.92352,
+ "4780": 12.00277,
+ "4785": 11.94768,
+ "4790": 11.97296,
+ "4795": 11.98757,
+ "4800": 12.26361,
+ "4805": 11.90736,
+ "4810": 11.9844,
+ "4815": 12.04212,
+ "4820": 11.98762,
+ "4825": 12.89959,
+ "4830": 11.9442,
+ "4835": 12.35106,
+ "4840": 11.93828,
+ "4845": 11.92418,
+ "4850": 11.96443,
+ "4855": 12.03431,
+ "4860": 12.04422,
+ "4865": 11.9646,
+ "4870": 11.91857,
+ "4875": 11.95672,
+ "4880": 11.9198,
+ "4885": 11.96783,
+ "4890": 11.94953,
+ "4895": 11.96692,
+ "4900": 12.04475,
+ "4905": 12.05877,
+ "4910": 12.15039,
+ "4915": 12.15039,
+ "4920": 11.95008,
+ "4925": 11.96843,
+ "4930": 11.958,
+ "4935": 11.98531,
+ "4940": 11.90874,
+ "4945": 11.95752,
+ "4950": 12.01284,
+ "4955": 11.97799,
+ "4960": 11.99989,
+ "4965": 11.9277,
+ "4970": 12.06095,
+ "4975": 11.95713,
+ "4980": 12.02719,
+ "4985": 11.96446,
+ "4990": 11.92043,
+ "4995": 11.99522,
+ "5000": 12.0792,
+ "5005": 11.95462,
+ "5010": 18.30939,
+ "5015": 12.57034,
+ "5020": 12.13652,
+ "5025": 11.95064,
+ "5030": 11.93538,
+ "5035": 12.01779,
+ "5040": 11.8639,
+ "5045": 11.89312,
+ "5050": 11.93054,
+ "5055": 11.89904,
+ "5060": 11.88635,
+ "5065": 11.89505,
+ "5070": 11.95957,
+ "5075": 11.96591,
+ "5080": 11.85594,
+ "5085": 11.87343,
+ "5090": 11.89162,
+ "5095": 11.9231,
+ "5100": 11.9213,
+ "5105": 11.9793,
+ "5110": 11.92942,
+ "5115": 11.87025,
+ "5120": 11.84167,
+ "5125": 11.92967,
+ "5130": 11.90523,
+ "5135": 11.8727,
+ "5140": 11.95822,
+ "5145": 11.97795,
+ "5150": 11.90614,
+ "5155": 11.88276,
+ "5160": 11.94188,
+ "5165": 11.91373,
+ "5170": 12.01192,
+ "5175": 11.85511,
+ "5180": 11.84375,
+ "5185": 11.88965,
+ "5190": 11.88542,
+ "5195": 11.85346,
+ "5200": 11.94188,
+ "5205": 11.92082,
+ "5210": 11.8821,
+ "5215": 11.92239,
+ "5220": 11.90608,
+ "5225": 11.8947,
+ "5230": 11.88619,
+ "5235": 11.8948,
+ "5240": 11.89599,
+ "5245": 11.88662,
+ "5250": 11.95415,
+ "5255": 11.96527,
+ "5260": 11.89009,
+ "5265": 11.87997,
+ "5270": 11.94016,
+ "5275": 11.89138,
+ "5280": 11.90447,
+ "5285": 11.86453,
+ "5290": 11.90845,
+ "5295": 11.89373,
+ "5300": 11.96084,
+ "5305": 12.00505,
+ "5310": 11.87874,
+ "5315": 11.94047,
+ "5320": 11.90115,
+ "5325": 11.8657,
+ "5330": 11.98456,
+ "5335": 11.89142,
+ "5340": 11.94056,
+ "5345": 11.88326,
+ "5350": 12.02941,
+ "5355": 11.94937,
+ "5360": 11.84158,
+ "5365": 11.85236,
+ "5370": 11.89414,
+ "5375": 11.92681,
+ "5380": 11.89983,
+ "5385": 11.93247,
+ "5390": 11.88545,
+ "5395": 11.85963,
+ "5400": 11.87187,
+ "5405": 11.92558,
+ "5410": 11.94364,
+ "5415": 11.9087,
+ "5420": 11.86332,
+ "5425": 11.92767,
+ "5430": 11.87425,
+ "5435": 11.91049,
+ "5440": 11.87699,
+ "5445": 11.93171,
+ "5450": 11.90161,
+ "5455": 11.921,
+ "5460": 11.88038,
+ "5465": 11.91315,
+ "5470": 11.89728,
+ "5475": 11.95689,
+ "5480": 11.98965,
+ "5485": 11.91576,
+ "5490": 11.89757,
+ "5495": 11.93064,
+ "5500": 11.88252,
+ "5505": 11.96073,
+ "5510": 11.86654,
+ "5515": 11.87886,
+ "5520": 11.90936,
+ "5525": 12.03373,
+ "5530": 11.90318,
+ "5535": 11.92154,
+ "5540": 11.90086,
+ "5545": 11.89022,
+ "5550": 11.90225,
+ "5555": 11.83513,
+ "5560": 11.91062,
+ "5565": 11.87125,
+ "5570": 11.87145,
+ "5575": 11.86357,
+ "5580": 11.91841,
+ "5585": 11.92436,
+ "5590": 11.9023,
+ "5595": 11.86709,
+ "5600": 11.91375,
+ "5605": 11.90872,
+ "5610": 11.8916,
+ "5615": 11.95578,
+ "5620": 11.89294,
+ "5625": 11.90784,
+ "5630": 11.92391,
+ "5635": 11.89956,
+ "5640": 11.89869,
+ "5645": 11.91776,
+ "5650": 11.9431,
+ "5655": 11.89517,
+ "5660": 11.88968,
+ "5665": 11.89529,
+ "5670": 11.91051,
+ "5675": 11.91888,
+ "5680": 11.90991,
+ "5685": 11.93985,
+ "5690": 11.90708,
+ "5695": 11.8876,
+ "5700": 11.95923,
+ "5705": 11.93355,
+ "5710": 11.87364,
+ "5715": 11.9268,
+ "5720": 11.98226,
+ "5725": 11.87678,
+ "5730": 11.83368,
+ "5735": 11.89468,
+ "5740": 11.90674,
+ "5745": 11.88476,
+ "5750": 11.86646,
+ "5755": 11.88929,
+ "5760": 11.85649,
+ "5765": 11.85565,
+ "5770": 11.93646,
+ "5775": 11.90704,
+ "5780": 12.04897,
+ "5785": 11.91885,
+ "5790": 11.90414,
+ "5795": 11.92795,
+ "5800": 11.9484,
+ "5805": 11.9947,
+ "5810": 11.88562,
+ "5815": 11.89893,
+ "5820": 11.86069,
+ "5825": 11.85602,
+ "5830": 11.90577,
+ "5835": 11.90369,
+ "5840": 11.95291,
+ "5845": 11.93547,
+ "5850": 11.89776,
+ "5855": 11.89365,
+ "5860": 11.88809,
+ "5865": 11.89502,
+ "5870": 11.90093,
+ "5875": 11.89463,
+ "5880": 11.85877,
+ "5885": 11.91775,
+ "5890": 11.9362,
+ "5895": 11.90238,
+ "5900": 11.89416,
+ "5905": 11.9161,
+ "5910": 11.91617,
+ "5915": 11.89704,
+ "5920": 11.86193,
+ "5925": 11.94942,
+ "5930": 11.85147,
+ "5935": 11.87033,
+ "5940": 11.9311,
+ "5945": 11.96348,
+ "5950": 11.96932,
+ "5955": 11.90137,
+ "5960": 11.87563,
+ "5965": 11.86128,
+ "5970": 11.99512,
+ "5975": 11.92846,
+ "5980": 11.83738,
+ "5985": 11.88075,
+ "5990": 11.89265,
+ "5995": 11.92537,
+ "6000": 11.88009,
+ "6005": 11.9523,
+ "6010": 11.93509,
+ "6015": 11.89766,
+ "6020": 11.88045,
+ "6025": 11.87641,
+ "6030": 246.60413,
+ "6035": 12.33879,
+ "6040": 11.91607,
+ "6045": 11.95709,
+ "6050": 11.93381,
+ "6055": 11.91355,
+ "6060": 11.91286,
+ "6065": 11.97819,
+ "6070": 11.93373,
+ "6075": 11.85049,
+ "6080": 11.96747,
+ "6085": 11.93318,
+ "6090": 11.93239,
+ "6095": 11.8622,
+ "6100": 11.88525,
+ "6105": 11.97899,
+ "6110": 11.91577,
+ "6115": 11.92755,
+ "6120": 11.92296,
+ "6125": 11.99725,
+ "6130": 11.97753,
+ "6135": 11.92108,
+ "6140": 11.91607,
+ "6145": 11.9071,
+ "6150": 11.92499,
+ "6155": 11.91611,
+ "6160": 12.01604,
+ "6165": 11.89838,
+ "6170": 11.90254,
+ "6175": 11.96493,
+ "6180": 11.84452,
+ "6185": 11.91052,
+ "6190": 11.8712,
+ "6195": 11.90582,
+ "6200": 11.90605,
+ "6205": 11.98397,
+ "6210": 11.92035,
+ "6215": 11.96579,
+ "6220": 11.99275,
+ "6225": 11.88749,
+ "6230": 11.89369,
+ "6235": 11.95748,
+ "6240": 11.93057,
+ "6245": 11.94912,
+ "6250": 11.9372,
+ "6255": 11.90439,
+ "6260": 11.92527,
+ "6265": 11.95201,
+ "6270": 11.9095,
+ "6275": 11.97821,
+ "6280": 11.94458,
+ "6285": 11.90287,
+ "6290": 11.89278,
+ "6295": 11.96073,
+ "6300": 11.90554,
+ "6305": 11.88653,
+ "6310": 11.8962,
+ "6315": 11.93036,
+ "6320": 11.95396,
+ "6325": 11.94894,
+ "6330": 12.04569,
+ "6335": 11.88055,
+ "6340": 11.91066,
+ "6345": 11.89024,
+ "6350": 11.89994,
+ "6355": 11.92221,
+ "6360": 11.92333,
+ "6365": 11.91761,
+ "6370": 11.97313,
+ "6375": 11.90689,
+ "6380": 12.08922,
+ "6385": 11.94942,
+ "6390": 11.91702,
+ "6395": 11.90139,
+ "6400": 11.89012,
+ "6405": 11.9541,
+ "6410": 12.00044,
+ "6415": 11.89967,
+ "6420": 11.86695,
+ "6425": 11.87294,
+ "6430": 11.89524,
+ "6435": 11.94881,
+ "6440": 11.91361,
+ "6445": 11.91243,
+ "6450": 11.90246,
+ "6455": 11.88301,
+ "6460": 11.94133,
+ "6465": 11.95353,
+ "6470": 11.93545,
+ "6475": 11.91767,
+ "6480": 11.904,
+ "6485": 11.97366,
+ "6490": 11.9268,
+ "6495": 11.92497,
+ "6500": 12.05293,
+ "6505": 11.83715,
+ "6510": 11.86732,
+ "6515": 11.90038,
+ "6520": 11.86776,
+ "6525": 11.86971,
+ "6530": 11.85789,
+ "6535": 11.88616,
+ "6540": 11.85825,
+ "6545": 11.82803,
+ "6550": 11.89596,
+ "6555": 11.89246,
+ "6560": 11.87827,
+ "6565": 11.87369,
+ "6570": 11.88103,
+ "6575": 11.86696,
+ "6580": 11.90165,
+ "6585": 11.85113,
+ "6590": 11.85101,
+ "6595": 11.80896,
+ "6600": 11.90596,
+ "6605": 11.87406,
+ "6610": 11.8658,
+ "6615": 11.86475,
+ "6620": 11.88848,
+ "6625": 11.85675,
+ "6630": 11.84722,
+ "6635": 11.83752,
+ "6640": 11.8855,
+ "6645": 11.91332,
+ "6650": 11.86288,
+ "6655": 11.89588,
+ "6660": 11.8071,
+ "6665": 11.84093,
+ "6670": 11.88653,
+ "6675": 11.88047,
+ "6680": 11.87018,
+ "6685": 11.8411,
+ "6690": 11.82244,
+ "6695": 11.86596,
+ "6700": 11.85423,
+ "6705": 11.86228,
+ "6710": 11.86517,
+ "6715": 11.87189,
+ "6720": 11.84138,
+ "6725": 11.88097,
+ "6730": 11.90906,
+ "6735": 11.91578,
+ "6740": 11.88058,
+ "6745": 11.88169,
+ "6750": 12.03575,
+ "6755": 11.84511,
+ "6760": 11.84038,
+ "6765": 11.83499,
+ "6770": 11.87927,
+ "6775": 11.81349,
+ "6780": 13.01048,
+ "6785": 11.81032,
+ "6790": 11.93614,
+ "6795": 11.97801,
+ "6800": 11.86,
+ "6805": 11.83039,
+ "6810": 11.8441,
+ "6815": 11.89187,
+ "6820": 11.87841,
+ "6825": 11.86012,
+ "6830": 11.83442,
+ "6835": 11.85081,
+ "6840": 11.83799,
+ "6845": 11.82691,
+ "6850": 11.89092,
+ "6855": 11.82022,
+ "6860": 11.8279,
+ "6865": 11.79814,
+ "6870": 11.83217,
+ "6875": 11.90136,
+ "6880": 11.85295,
+ "6885": 11.84058,
+ "6890": 11.84482,
+ "6895": 11.82768,
+ "6900": 11.88337,
+ "6905": 11.84656,
+ "6910": 11.90272,
+ "6915": 11.8005,
+ "6920": 11.93804,
+ "6925": 12.00166,
+ "6930": 11.88293,
+ "6935": 11.9479,
+ "6940": 11.85228,
+ "6945": 11.86242,
+ "6950": 11.83582,
+ "6955": 11.81523,
+ "6960": 11.75894,
+ "6965": 11.81699,
+ "6970": 11.85282,
+ "6975": 11.84727,
+ "6980": 11.84729,
+ "6985": 12.01189,
+ "6990": 11.86887,
+ "6995": 11.88713,
+ "7000": 11.85612,
+ "7005": 11.86648,
+ "7010": 11.8888,
+ "7015": 11.84573,
+ "7020": 11.77395,
+ "7025": 11.85096,
+ "7030": 11.86323,
+ "7035": 11.84315,
+ "7040": 11.82293,
+ "7045": 11.81241,
+ "7050": 11.85808,
+ "7055": 11.86593,
+ "7060": 11.87475,
+ "7065": 11.90707,
+ "7070": 11.9358,
+ "7075": 11.84297,
+ "7080": 11.80853,
+ "7085": 11.88178,
+ "7090": 11.87836,
+ "7095": 11.85532,
+ "7100": 11.89414,
+ "7105": 11.85379,
+ "7110": 11.89642,
+ "7115": 11.85858,
+ "7120": 11.90327,
+ "7125": 11.89711,
+ "7130": 11.89177,
+ "7135": 11.88659,
+ "7140": 11.85757,
+ "7145": 11.87756,
+ "7150": 11.88577,
+ "7155": 11.86153,
+ "7160": 11.92297,
+ "7165": 11.88396,
+ "7170": 11.85778,
+ "7175": 11.91483,
+ "7180": 11.86232,
+ "7185": 11.87476,
+ "7190": 11.8982,
+ "7195": 11.88516,
+ "7200": 11.88158,
+ "7205": 11.88444,
+ "7210": 11.89206,
+ "7215": 11.87279,
+ "7220": 11.90742,
+ "7225": 11.85079,
+ "7230": 11.8483,
+ "7235": 11.90312,
+ "7240": 11.87181,
+ "7245": 11.91535,
+ "7250": 11.87908,
+ "7255": 11.92293,
+ "7260": 11.84549,
+ "7265": 11.8901,
+ "7270": 11.84322,
+ "7275": 11.848,
+ "7280": 11.8967,
+ "7285": 11.89986,
+ "7290": 11.95382,
+ "7295": 11.90753,
+ "7300": 11.86218,
+ "7305": 11.85436,
+ "7310": 11.85753,
+ "7315": 11.9134,
+ "7320": 11.90034,
+ "7325": 11.83407,
+ "7330": 11.85974,
+ "7335": 11.90032,
+ "7340": 11.88835,
+ "7345": 11.88443,
+ "7350": 11.85147,
+ "7355": 11.86003,
+ "7360": 11.88911,
+ "7365": 11.88721,
+ "7370": 11.94597,
+ "7375": 11.88507,
+ "7380": 11.8675,
+ "7385": 11.88615,
+ "7390": 11.85493,
+ "7395": 11.9078,
+ "7400": 11.89976,
+ "7405": 11.94755,
+ "7410": 11.86216,
+ "7415": 11.81832,
+ "7420": 11.89699,
+ "7425": 11.90201,
+ "7430": 11.88324,
+ "7435": 11.84242,
+ "7440": 11.89387,
+ "7445": 11.85554,
+ "7450": 11.927,
+ "7455": 11.89196,
+ "7460": 11.93241,
+ "7465": 11.89671,
+ "7470": 11.8633,
+ "7475": 11.85785,
+ "7480": 11.86619,
+ "7485": 11.90047,
+ "7490": 11.93453,
+ "7495": 11.89595,
+ "7500": 11.92255,
+ "7505": 11.86705,
+ "7510": 11.86492,
+ "7515": 11.83778,
+ "7520": 12.43308,
+ "7525": 11.94046,
+ "7530": 12.11911,
+ "7535": 11.95645,
+ "7540": 12.01144,
+ "7545": 11.94459,
+ "7550": 12.00989,
+ "7555": 11.95308,
+ "7560": 12.02894,
+ "7565": 12.00926,
+ "7570": 11.88032,
+ "7575": 11.94986,
+ "7580": 11.94673,
+ "7585": 11.92777,
+ "7590": 11.96311,
+ "7595": 11.90291,
+ "7600": 11.96776,
+ "7605": 11.91009,
+ "7610": 11.98945,
+ "7615": 11.943,
+ "7620": 11.97203,
+ "7625": 11.87696,
+ "7630": 11.92313,
+ "7635": 11.9056,
+ "7640": 11.89922,
+ "7645": 11.93063,
+ "7650": 11.89735,
+ "7655": 11.93078,
+ "7660": 11.95494,
+ "7665": 11.91011,
+ "7670": 11.97093,
+ "7675": 11.97514,
+ "7680": 11.93177,
+ "7685": 11.8992,
+ "7690": 11.94571,
+ "7695": 11.92277,
+ "7700": 11.94906,
+ "7705": 11.92727,
+ "7710": 11.93604,
+ "7715": 11.92305,
+ "7720": 11.93766,
+ "7725": 11.95622,
+ "7730": 11.90603,
+ "7735": 11.91132,
+ "7740": 11.97695,
+ "7745": 11.96601,
+ "7750": 11.88967,
+ "7755": 11.93644,
+ "7760": 11.96688,
+ "7765": 11.92672,
+ "7770": 23.39259,
+ "7775": 23.06567,
+ "7780": 11.93112,
+ "7785": 11.93477,
+ "7790": 11.94106,
+ "7795": 11.94556,
+ "7800": 12.0002,
+ "7805": 11.97342,
+ "7810": 11.95163,
+ "7815": 11.96208,
+ "7820": 11.96513,
+ "7825": 11.93368,
+ "7830": 11.91708,
+ "7835": 11.89017,
+ "7840": 11.94549,
+ "7845": 11.96002,
+ "7850": 11.95829,
+ "7855": 11.92186,
+ "7860": 11.93832,
+ "7865": 11.889,
+ "7870": 11.96191,
+ "7875": 12.05703,
+ "7880": 11.97288,
+ "7885": 11.91666,
+ "7890": 11.93728,
+ "7895": 11.96047,
+ "7900": 11.9818,
+ "7905": 11.92242,
+ "7910": 11.97684,
+ "7915": 11.91154,
+ "7920": 11.96828,
+ "7925": 11.94506,
+ "7930": 11.93465,
+ "7935": 11.90216,
+ "7940": 11.91383,
+ "7945": 11.91481,
+ "7950": 11.96693,
+ "7955": 11.94446,
+ "7960": 11.92358,
+ "7965": 11.94155,
+ "7970": 11.95822,
+ "7975": 12.03469,
+ "7980": 11.94102,
+ "7985": 11.94681,
+ "7990": 11.92459,
+ "7995": 11.92763,
+ "8000": 11.96299,
+ "8005": 11.9788,
+ "8010": 11.96826,
+ "8015": 12.02982,
+ "8020": 11.94329,
+ "8025": 11.98105,
+ "8030": 12.01501,
+ "8035": 11.96502,
+ "8040": 11.97586,
+ "8045": 11.96948,
+ "8050": 11.92611,
+ "8055": 11.93414,
+ "8060": 11.93961,
+ "8065": 11.9262,
+ "8070": 11.9178,
+ "8075": 11.90325,
+ "8080": 11.93833,
+ "8085": 11.97936,
+ "8090": 11.99724,
+ "8095": 11.94796,
+ "8100": 11.9625,
+ "8105": 11.94798,
+ "8110": 11.92353,
+ "8115": 11.96357,
+ "8120": 11.92451,
+ "8125": 11.89352,
+ "8130": 11.97563,
+ "8135": 11.97236,
+ "8140": 11.9723,
+ "8145": 11.92641,
+ "8150": 11.89834,
+ "8155": 11.94876,
+ "8160": 11.95465,
+ "8165": 11.95874,
+ "8170": 11.93402,
+ "8175": 11.96745,
+ "8180": 11.91172,
+ "8185": 11.91331,
+ "8190": 11.95504,
+ "8195": 11.94346,
+ "8200": 11.95192,
+ "8205": 11.9973,
+ "8210": 11.95023,
+ "8215": 12.03521,
+ "8220": 11.96486,
+ "8225": 11.95464,
+ "8230": 11.96151,
+ "8235": 11.95994,
+ "8240": 11.97909,
+ "8245": 11.92928,
+ "8250": 11.92518,
+ "8255": 11.94881,
+ "8260": 11.907,
+ "8265": 11.93185,
+ "8270": 11.9211,
+ "8275": 11.86366,
+ "8280": 12.00914,
+ "8285": 11.97086,
+ "8290": 11.98208,
+ "8295": 11.92309,
+ "8300": 11.94129,
+ "8305": 11.99302,
+ "8310": 11.97601,
+ "8315": 11.88862,
+ "8320": 11.96454,
+ "8325": 11.89961,
+ "8330": 11.99534,
+ "8335": 11.91687,
+ "8340": 11.96466,
+ "8345": 11.93152,
+ "8350": 11.94368,
+ "8355": 11.92235,
+ "8360": 11.99578,
+ "8365": 11.90045,
+ "8370": 11.91744,
+ "8375": 11.92667,
+ "8380": 11.90428,
+ "8385": 11.94828,
+ "8390": 11.93507,
+ "8395": 11.9473,
+ "8400": 11.94267,
+ "8405": 11.93414,
+ "8410": 11.90959,
+ "8415": 11.92941,
+ "8420": 11.91201,
+ "8425": 11.91625,
+ "8430": 11.9332,
+ "8435": 11.99456,
+ "8440": 11.8869,
+ "8445": 11.90729,
+ "8450": 11.93362,
+ "8455": 11.96619,
+ "8460": 12.01359,
+ "8465": 11.9429,
+ "8470": 11.99594,
+ "8475": 11.95465,
+ "8480": 11.92489,
+ "8485": 11.92415,
+ "8490": 11.97388,
+ "8495": 11.89913,
+ "8500": 11.95945,
+ "8505": 11.91567,
+ "8510": 11.91482,
+ "8515": 11.93548,
+ "8520": 11.95743,
+ "8525": 11.94743,
+ "8530": 12.42097,
+ "8535": 11.9272,
+ "8540": 12.09436,
+ "8545": 12.04967,
+ "8550": 11.9651,
+ "8555": 12.03857,
+ "8560": 11.97265,
+ "8565": 11.91082,
+ "8570": 11.95406,
+ "8575": 11.94802,
+ "8580": 11.9942,
+ "8585": 11.96288,
+ "8590": 11.95701,
+ "8595": 11.97786,
+ "8600": 11.89715,
+ "8605": 11.93644,
+ "8610": 11.98611,
+ "8615": 11.91557,
+ "8620": 11.92076,
+ "8625": 11.96113,
+ "8630": 11.99266,
+ "8635": 11.93916,
+ "8640": 12.02781,
+ "8645": 11.99006,
+ "8650": 11.91164,
+ "8655": 11.91924,
+ "8660": 11.95194,
+ "8665": 12.00021,
+ "8670": 11.90972,
+ "8675": 11.96086,
+ "8680": 11.95175,
+ "8685": 11.95495,
+ "8690": 12.00198,
+ "8695": 12.07659,
+ "8700": 11.96371,
+ "8705": 11.91845,
+ "8710": 11.97745,
+ "8715": 11.93805,
+ "8720": 11.9173,
+ "8725": 11.91035,
+ "8730": 12.01393,
+ "8735": 11.98447,
+ "8740": 11.97475,
+ "8745": 11.96291,
+ "8750": 11.9361,
+ "8755": 11.96838,
+ "8760": 11.93695,
+ "8765": 12.00162,
+ "8770": 11.92599,
+ "8775": 12.0012,
+ "8780": 12.03738,
+ "8785": 11.94909,
+ "8790": 11.90577,
+ "8795": 11.97012,
+ "8800": 11.93035,
+ "8805": 11.99893,
+ "8810": 11.94421,
+ "8815": 11.98191,
+ "8820": 11.99062,
+ "8825": 11.92267,
+ "8830": 11.95194,
+ "8835": 11.937,
+ "8840": 11.97075,
+ "8845": 11.95007,
+ "8850": 12.02522,
+ "8855": 11.94712,
+ "8860": 11.96728,
+ "8865": 11.89285,
+ "8870": 11.94189,
+ "8875": 11.92065,
+ "8880": 11.98822,
+ "8885": 11.98285,
+ "8890": 11.99582,
+ "8895": 11.96596,
+ "8900": 11.94354,
+ "8905": 11.95473,
+ "8910": 11.99259,
+ "8915": 11.96618,
+ "8920": 11.93587,
+ "8925": 11.99413,
+ "8930": 12.00638,
+ "8935": 11.93,
+ "8940": 11.95031,
+ "8945": 11.91928,
+ "8950": 11.9941,
+ "8955": 11.94031,
+ "8960": 11.96914,
+ "8965": 11.95062,
+ "8970": 11.95268,
+ "8975": 12.03161,
+ "8980": 11.97245,
+ "8985": 12.01027,
+ "8990": 11.9446,
+ "8995": 11.96843,
+ "9000": 11.9429,
+ "9005": 11.94091,
+ "9010": 11.93667,
+ "9015": 11.95344,
+ "9020": 11.93207,
+ "9025": 11.91998,
+ "9030": 11.92651,
+ "9035": 11.97131,
+ "9040": 11.92008,
+ "9045": 11.9777,
+ "9050": 11.93287,
+ "9055": 11.96682,
+ "9060": 11.982,
+ "9065": 11.9763,
+ "9070": 11.92703,
+ "9075": 11.95149,
+ "9080": 11.94863,
+ "9085": 11.92217,
+ "9090": 11.92326,
+ "9095": 11.9586,
+ "9100": 11.93403,
+ "9105": 11.97708,
+ "9110": 11.97248,
+ "9115": 11.91899,
+ "9120": 11.98175,
+ "9125": 12.0043,
+ "9130": 11.98361,
+ "9135": 11.95811,
+ "9140": 11.89116,
+ "9145": 11.92833,
+ "9150": 11.96999,
+ "9155": 11.95682,
+ "9160": 11.93898,
+ "9165": 11.98676,
+ "9170": 11.96776,
+ "9175": 11.91735,
+ "9180": 11.96488,
+ "9185": 11.93801,
+ "9190": 11.93829,
+ "9195": 11.96444,
+ "9200": 11.91924,
+ "9205": 11.99554,
+ "9210": 11.91977,
+ "9215": 11.99739,
+ "9220": 11.92053,
+ "9225": 11.93702,
+ "9230": 11.95815,
+ "9235": 12.05346,
+ "9240": 11.9596,
+ "9245": 11.97173,
+ "9250": 11.94092,
+ "9255": 11.94632,
+ "9260": 12.00354,
+ "9265": 11.96854,
+ "9270": 11.91621,
+ "9275": 11.94709,
+ "9280": 11.93375,
+ "9285": 11.92465,
+ "9290": 11.93047,
+ "9295": 11.93184,
+ "9300": 11.95538,
+ "9305": 11.96102,
+ "9310": 11.93874,
+ "9315": 11.94123,
+ "9320": 11.95854,
+ "9325": 11.98961,
+ "9330": 11.87394,
+ "9335": 11.97986,
+ "9340": 12.02583,
+ "9345": 11.94202,
+ "9350": 12.00113,
+ "9355": 11.97405,
+ "9360": 11.96746,
+ "9365": 11.96018,
+ "9370": 11.9475,
+ "9375": 11.94327,
+ "9380": 11.92135,
+ "9385": 12.01574,
+ "9390": 11.95494,
+ "9395": 11.93529,
+ "9400": 11.96463,
+ "9405": 11.9807,
+ "9410": 11.92926,
+ "9415": 11.95919,
+ "9420": 11.94796,
+ "9425": 11.94261,
+ "9430": 11.94968,
+ "9435": 11.9655,
+ "9440": 11.94016,
+ "9445": 11.98541,
+ "9450": 11.94602,
+ "9455": 11.96365,
+ "9460": 11.9884,
+ "9465": 11.93962,
+ "9470": 11.93471,
+ "9475": 11.91073,
+ "9480": 11.92557,
+ "9485": 11.93537,
+ "9490": 11.97267,
+ "9495": 11.93521,
+ "9500": 11.92542,
+ "9505": 12.00627,
+ "9510": 11.9749,
+ "9515": 11.97511,
+ "9520": 11.88493,
+ "9525": 11.91739,
+ "9530": 11.92418,
+ "9535": 11.97024
+ }
+ }
+}
\ No newline at end of file
diff --git a/tests/functional_tests/test_cases/mixtral/deepseekv3_proxy_flex_tp1pp4emp16etp1cp1_release_sm/model_config.yaml b/tests/functional_tests/test_cases/mixtral/deepseekv3_proxy_flex_tp1pp4emp16etp1cp1_release_sm/model_config.yaml
new file mode 100644
index 00000000000..8bab921aa04
--- /dev/null
+++ b/tests/functional_tests/test_cases/mixtral/deepseekv3_proxy_flex_tp1pp4emp16etp1cp1_release_sm/model_config.yaml
@@ -0,0 +1,166 @@
+# The proxy model is used for local code quality check.
+# The proxy model should contain all the necessary components and settings but fewer parameters.
+ENV_VARS:
+ TORCH_NCCL_AVOID_RECORD_STREAMS: 0
+ NVTE_ALLOW_NONDETERMINISTIC_ALGO: 1
+ PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True
+ NCCL_NVLS_ENABLE: 0
+ NVTE_FUSED_ATTN: 1
+ NVTE_NORM_FWD_USE_CUDNN: 1
+ NVTE_NORM_BWD_USE_CUDNN: 1
+ PYTHONWARNINGS: ignore
+ NCCL_DEBUG: VERSION
+ NON_DETERMINSTIC_RESULTS: 1
+ NVSHMEM_IB_ENABLE_IBGDA: 0
+ CUDA_DEVICE_MAX_CONNECTIONS: 1
+TEST_TYPE: "release"
+MODEL_ARGS:
+ # Distributed args
+ --distributed-timeout-minutes: 60
+ --tensor-model-parallel-size: 2
+ --pipeline-model-parallel-size: 4
+ --pipeline-model-parallel-layout: Et*2\\|\\(tt\\|\\)*5t\\|tmL # Et*2|(tt|)*5t|tmL
+ --expert-model-parallel-size: 16
+ --context-parallel-size: 1
+ --expert-tensor-parallel-size: 1
+ --use-distributed-optimizer: true
+ --overlap-grad-reduce: true
+ --overlap-param-gather: true
+
+ # Training args
+ --use-mcore-models: true
+ --sequence-parallel: true
+ --use-flash-attn: true
+ --disable-bias-linear: true
+ --micro-batch-size: 1
+ --global-batch-size: 512
+ --train-samples: 24414062
+ --exit-duration-in-mins: 220
+ --no-check-for-nan-in-loss-and-grad: true
+ --cross-entropy-loss-fusion: true
+ --cross-entropy-fusion-impl: te
+ --manual-gc: true
+ --manual-gc-interval: 10
+
+ # Transformer Engine args
+ --transformer-impl: transformer_engine
+
+ # Data args
+ --seq-length: 4096
+ --data-cache-path: ${DATA_CACHE_PATH}
+ --tokenizer-type: GPTSentencePieceTokenizer
+ --tokenizer-model: ${DATA_PATH}/utils/nemotron_2_256k.model
+ --data-path: $DATA_BLEND
+ --split: 99,1,0
+ --no-mmap-bin-files: true
+ --no-create-attention-mask-in-dataloader: true
+ --num-workers: 6
+
+ # Add network size args
+ --num-layers: 14 # original 61 layers
+ --hidden-size: 7168
+ --ffn-hidden-size: 18432
+ --num-attention-heads: 128
+ --kv-channels: 128
+ --max-position-embeddings: 4096
+ --position-embedding-type: rope
+ --rotary-base: 10000
+ --make-vocab-size-divisible-by: 3232
+ --normalization: RMSNorm
+ --norm-epsilon: 1e-6
+ --swiglu: true
+ --untie-embeddings-and-output-weights: true
+ --multi-latent-attention: true
+ --mtp-num-layers: 1
+ --mtp-loss-scaling-factor: 0.1
+
+ # Add regularization args
+ --attention-dropout: 0.0
+ --hidden-dropout: 0.0
+ --clip-grad: 1.0
+ --weight-decay: 0.1
+ --qk-layernorm: true
+
+ # Add learning rate args
+ --lr-decay-samples: 24413696
+ --lr-warmup-samples: 1536000
+ --lr-warmup-init: 1e-7
+ --lr: 1e-5
+ --min-lr: 1e-6
+ --lr-decay-style: cosine
+ --adam-beta1: 0.9
+ --adam-beta2: 0.95
+
+ # Add MoE args
+ --num-experts: 64 # local 4 + 1 shared, EP16
+ --moe-layer-freq: ([0]*3+[1]*11)
+ --moe-ffn-hidden-size: 2048
+ --moe-shared-expert-intermediate-size: 2048
+ --moe-router-load-balancing-type: seq_aux_loss
+ --moe-router-topk: 8
+ --moe-token-dispatcher-type: flex
+ --moe-enable-deepep: true
+ --moe-router-pre-softmax: true
+ --moe-grouped-gemm: true
+ --moe-aux-loss-coeff: 1e-4
+ --moe-router-group-topk: 4
+ --moe-router-num-groups: 8
+ --moe-router-topk-scaling-factor: 2.5
+ --moe-router-score-function: sigmoid
+ --moe-router-enable-expert-bias: true
+ --moe-router-bias-update-rate: 1e-3
+ --moe-router-dtype: fp32
+ --moe-permute-fusion: true
+
+ # Add MLA args
+ --q-lora-rank: 1536
+ --kv-lora-rank: 512
+ --qk-head-dim: 128
+ --qk-pos-emb-head-dim: 64
+ --v-head-dim: 128
+ --rotary-scaling-factor: 40
+ --mscale: 1.0
+ --mscale-all-dim: 1.0
+
+ # Add validation args
+ --eval-iters: 32
+ --eval-interval: 200
+
+ # Add checkpointing args
+ --auto-detect-ckpt-format:
+ true
+ # Add checkpointing args
+ --save: ${CHECKPOINT_SAVE_PATH}
+ --load: ${CHECKPOINT_LOAD_PATH}
+ --save-interval: 500
+ --save-retain-interval: 10000
+ --dist-ckpt-strictness: log_all
+
+ # Add initialization args
+ --init-method-std: 0.02
+
+ # Add logging args
+ --log-timers-to-tensorboard: true
+ --log-memory-to-tensorboard: true
+ --log-num-zeros-in-grad: true
+ --log-params-norm: true
+ --log-validation-ppl-to-tensorboard: true
+ --log-throughput: true
+ --log-interval: 1
+ --logging-level: 40
+ --tensorboard-dir: ${TENSORBOARD_PATH}
+ --wandb-project: megatron-core-release-runs
+ --wandb-exp-name: ${WANDB_EXPERIMENT}
+ --wandb-save-dir: ${WANDB_SAVE_PATH}
+
+ # Add mixed precision args
+ --bf16: true
+
+ # enable experimental
+ --enable-experimental: true
+ --exit-interval: 9536
+METRICS:
+ - "iteration-time"
+ - "lm loss"
+ - "mem-allocated-bytes"
+ - "mem-max-allocated-bytes"
diff --git a/tests/functional_tests/test_cases/moe/gpt3_mcore_te_tp2_pp2_ep4_etp1_memory_speed/golden_values_dev_dgx_h100.json b/tests/functional_tests/test_cases/moe/gpt3_mcore_te_tp2_pp2_ep4_etp1_memory_speed/golden_values_dev_dgx_h100.json
index 221abd48c74..a47b94faa75 100644
--- a/tests/functional_tests/test_cases/moe/gpt3_mcore_te_tp2_pp2_ep4_etp1_memory_speed/golden_values_dev_dgx_h100.json
+++ b/tests/functional_tests/test_cases/moe/gpt3_mcore_te_tp2_pp2_ep4_etp1_memory_speed/golden_values_dev_dgx_h100.json
@@ -6,54 +6,54 @@
"values": {
"1": 11.04733,
"2": 11.03572,
- "3": 9.5878,
- "4": 9.25791,
- "5": 9.51585,
- "6": 9.91425,
- "7": 9.49022,
- "8": 8.94619,
- "9": 8.65195,
- "10": 9.06313,
- "11": 8.49654,
- "12": 8.52749,
- "13": 8.45919,
- "14": 7.99341,
- "15": 8.05353,
- "16": 8.08327,
- "17": 8.10021,
- "18": 7.77408,
- "19": 8.14992,
- "20": 7.89646,
- "21": 7.60027,
- "22": 7.55248,
- "23": 7.43137,
- "24": 7.43223,
- "25": 7.68057,
- "26": 7.07422,
- "27": 7.62201,
- "28": 7.33353,
- "29": 7.49795,
- "30": 7.64414,
- "31": 7.39519,
- "32": 7.59013,
- "33": 7.64569,
- "34": 7.70593,
- "35": 7.2143,
- "36": 7.08788,
- "37": 7.43168,
- "38": 7.19723,
- "39": 7.55557,
- "40": 7.54844,
- "41": 7.49611,
- "42": 7.25383,
- "43": 7.23801,
- "44": 7.42036,
- "45": 7.19742,
- "46": 6.90447,
- "47": 7.30251,
- "48": 7.14379,
- "49": 7.59525,
- "50": 7.04023
+ "3": 9.58776,
+ "4": 9.25801,
+ "5": 9.53164,
+ "6": 9.90992,
+ "7": 9.48661,
+ "8": 8.93947,
+ "9": 8.65725,
+ "10": 9.0567,
+ "11": 8.49436,
+ "12": 8.52422,
+ "13": 8.45295,
+ "14": 7.97674,
+ "15": 8.04629,
+ "16": 8.08024,
+ "17": 8.08398,
+ "18": 7.76141,
+ "19": 8.15001,
+ "20": 7.89339,
+ "21": 7.58212,
+ "22": 7.54491,
+ "23": 7.43428,
+ "24": 7.42622,
+ "25": 7.67267,
+ "26": 7.07291,
+ "27": 7.61503,
+ "28": 7.31789,
+ "29": 7.48965,
+ "30": 7.64357,
+ "31": 7.3927,
+ "32": 7.58407,
+ "33": 7.63624,
+ "34": 7.69746,
+ "35": 7.21377,
+ "36": 7.08367,
+ "37": 7.4245,
+ "38": 7.18783,
+ "39": 7.5498,
+ "40": 7.54133,
+ "41": 7.48816,
+ "42": 7.24677,
+ "43": 7.23194,
+ "44": 7.41471,
+ "45": 7.18838,
+ "46": 6.89674,
+ "47": 7.29904,
+ "48": 7.13855,
+ "49": 7.58882,
+ "50": 7.03386
}
},
"num-zeros": {
@@ -61,56 +61,56 @@
"end_step": 50,
"step_interval": 1,
"values": {
- "1": 38802616.0,
- "2": 38543540.0,
- "3": 38741560.0,
- "4": 273652640.0,
- "5": 246619984.0,
- "6": 255713984.0,
- "7": 585904576.0,
- "8": 775188544.0,
- "9": 683552384.0,
- "10": 678184384.0,
- "11": 709420544.0,
- "12": 771913024.0,
- "13": 884572992.0,
- "14": 805905152.0,
- "15": 771490816.0,
- "16": 932248832.0,
- "17": 721261824.0,
- "18": 683711296.0,
- "19": 963724352.0,
- "20": 998655872.0,
- "21": 756360320.0,
- "22": 969720704.0,
- "23": 762708416.0,
- "24": 889305088.0,
- "25": 865191296.0,
- "26": 828440320.0,
- "27": 806905024.0,
- "28": 837449408.0,
- "29": 783497856.0,
- "30": 772494272.0,
- "31": 793774528.0,
- "32": 774902528.0,
- "33": 752992128.0,
- "34": 721632000.0,
- "35": 728225216.0,
- "36": 542603008.0,
- "37": 723530816.0,
- "38": 677573184.0,
- "39": 686397568.0,
- "40": 651324224.0,
- "41": 604614656.0,
- "42": 582812544.0,
- "43": 564189760.0,
- "44": 569972864.0,
- "45": 536820928.0,
- "46": 334504672.0,
- "47": 494444000.0,
- "48": 504118016.0,
- "49": 475199808.0,
- "50": 350261056.0
+ "1": 38802552.0,
+ "2": 38543496.0,
+ "3": 38742496.0,
+ "4": 276808768.0,
+ "5": 252900224.0,
+ "6": 262014400.0,
+ "7": 604765376.0,
+ "8": 778329280.0,
+ "9": 664674944.0,
+ "10": 728521920.0,
+ "11": 718868480.0,
+ "12": 787622592.0,
+ "13": 900296192.0,
+ "14": 831151488.0,
+ "15": 762029184.0,
+ "16": 938532864.0,
+ "17": 633234048.0,
+ "18": 708920704.0,
+ "19": 976315584.0,
+ "20": 986060288.0,
+ "21": 781551744.0,
+ "22": 762139648.0,
+ "23": 888477824.0,
+ "24": 851552512.0,
+ "25": 827443072.0,
+ "26": 812721088.0,
+ "27": 806914304.0,
+ "28": 802850496.0,
+ "29": 748894592.0,
+ "30": 731604672.0,
+ "31": 752878144.0,
+ "32": 762315520.0,
+ "33": 737258304.0,
+ "34": 746789888.0,
+ "35": 734508928.0,
+ "36": 674695808.0,
+ "37": 673198208.0,
+ "38": 633526912.0,
+ "39": 620340928.0,
+ "40": 613575552.0,
+ "41": 566869312.0,
+ "42": 557646592.0,
+ "43": 554752576.0,
+ "44": 547950784.0,
+ "45": 527374464.0,
+ "46": 347107200.0,
+ "47": 497586496.0,
+ "48": 497828864.0,
+ "49": 465758912.0,
+ "50": 450885792.0
}
},
"mem-allocated-bytes": {
@@ -175,56 +175,56 @@
"end_step": 50,
"step_interval": 1,
"values": {
- "1": 55051542528.0,
- "2": 57803964416.0,
- "3": 57918414848.0,
- "4": 57918414848.0,
- "5": 57918414848.0,
- "6": 57918414848.0,
- "7": 57918414848.0,
- "8": 57918414848.0,
- "9": 57918414848.0,
- "10": 57918414848.0,
- "11": 57918414848.0,
- "12": 57918414848.0,
- "13": 57918414848.0,
- "14": 57918414848.0,
- "15": 57918414848.0,
- "16": 57918414848.0,
- "17": 57918414848.0,
- "18": 57918414848.0,
- "19": 57918414848.0,
- "20": 57918414848.0,
- "21": 57918414848.0,
- "22": 57918414848.0,
- "23": 57918414848.0,
- "24": 57918414848.0,
- "25": 57918414848.0,
- "26": 57918414848.0,
- "27": 57918414848.0,
- "28": 57918414848.0,
- "29": 57918414848.0,
- "30": 57918414848.0,
- "31": 57918414848.0,
- "32": 57918414848.0,
- "33": 57918414848.0,
- "34": 57918414848.0,
- "35": 57918414848.0,
- "36": 57918414848.0,
- "37": 57918414848.0,
- "38": 57918414848.0,
- "39": 57918414848.0,
- "40": 57918414848.0,
- "41": 57918414848.0,
- "42": 57918414848.0,
- "43": 57918414848.0,
- "44": 57981075456.0,
- "45": 58164338688.0,
- "46": 58164338688.0,
- "47": 58164338688.0,
- "48": 58164338688.0,
- "49": 58164338688.0,
- "50": 58164338688.0
+ "1": 54204293120.0,
+ "2": 56956715008.0,
+ "3": 57074692096.0,
+ "4": 57074692096.0,
+ "5": 57074692096.0,
+ "6": 57074692096.0,
+ "7": 57074692096.0,
+ "8": 57074692096.0,
+ "9": 57074692096.0,
+ "10": 57074692096.0,
+ "11": 57074692096.0,
+ "12": 57074692096.0,
+ "13": 57074692096.0,
+ "14": 57074692096.0,
+ "15": 57074692096.0,
+ "16": 57074692096.0,
+ "17": 57074692096.0,
+ "18": 57074692096.0,
+ "19": 57074692096.0,
+ "20": 57074692096.0,
+ "21": 57074692096.0,
+ "22": 57074692096.0,
+ "23": 57074692096.0,
+ "24": 57074692096.0,
+ "25": 57074692096.0,
+ "26": 57211289600.0,
+ "27": 57211289600.0,
+ "28": 57211289600.0,
+ "29": 57368535040.0,
+ "30": 57742073856.0,
+ "31": 57742073856.0,
+ "32": 57742073856.0,
+ "33": 57742073856.0,
+ "34": 57744101376.0,
+ "35": 58293194752.0,
+ "36": 58293194752.0,
+ "37": 58293194752.0,
+ "38": 58293194752.0,
+ "39": 58293194752.0,
+ "40": 58293194752.0,
+ "41": 58293194752.0,
+ "42": 58293194752.0,
+ "43": 58293194752.0,
+ "44": 58293194752.0,
+ "45": 58293194752.0,
+ "46": 58293194752.0,
+ "47": 58293194752.0,
+ "48": 58293194752.0,
+ "49": 58293194752.0,
+ "50": 58293194752.0
}
},
"mtp_1 loss": {
@@ -234,54 +234,54 @@
"values": {
"1": 11.0765,
"2": 11.07404,
- "3": 10.5387,
- "4": 10.09807,
- "5": 9.81158,
- "6": 10.07371,
- "7": 9.79765,
- "8": 9.06972,
- "9": 8.86823,
- "10": 9.12665,
- "11": 8.49944,
- "12": 8.5346,
- "13": 8.42954,
- "14": 7.8522,
- "15": 7.99476,
- "16": 8.05407,
- "17": 8.0055,
- "18": 7.73795,
- "19": 8.11808,
- "20": 7.83141,
- "21": 7.53056,
- "22": 7.50549,
- "23": 7.37363,
- "24": 7.37845,
- "25": 7.62115,
- "26": 7.02061,
- "27": 7.5605,
- "28": 7.2695,
- "29": 7.44668,
- "30": 7.58971,
- "31": 7.32847,
- "32": 7.50861,
- "33": 7.57687,
- "34": 7.63939,
- "35": 7.15634,
- "36": 7.02394,
- "37": 7.35539,
- "38": 7.13177,
- "39": 7.49132,
- "40": 7.47677,
- "41": 7.42456,
- "42": 7.1802,
- "43": 7.16487,
- "44": 7.34808,
- "45": 7.12903,
- "46": 6.83012,
- "47": 7.2395,
- "48": 7.08268,
- "49": 7.51404,
- "50": 6.97693
+ "3": 10.53863,
+ "4": 10.0981,
+ "5": 9.81152,
+ "6": 10.0744,
+ "7": 9.79944,
+ "8": 9.07176,
+ "9": 8.87116,
+ "10": 9.12759,
+ "11": 8.49894,
+ "12": 8.53114,
+ "13": 8.42531,
+ "14": 7.84784,
+ "15": 7.99147,
+ "16": 8.05102,
+ "17": 8.00126,
+ "18": 7.73217,
+ "19": 8.11102,
+ "20": 7.83055,
+ "21": 7.52608,
+ "22": 7.49979,
+ "23": 7.37315,
+ "24": 7.37265,
+ "25": 7.61392,
+ "26": 7.01833,
+ "27": 7.55877,
+ "28": 7.26822,
+ "29": 7.44363,
+ "30": 7.58581,
+ "31": 7.3265,
+ "32": 7.50876,
+ "33": 7.57264,
+ "34": 7.63783,
+ "35": 7.15428,
+ "36": 7.02086,
+ "37": 7.35313,
+ "38": 7.12909,
+ "39": 7.48882,
+ "40": 7.47518,
+ "41": 7.42231,
+ "42": 7.17726,
+ "43": 7.16243,
+ "44": 7.34345,
+ "45": 7.12344,
+ "46": 6.8279,
+ "47": 7.23665,
+ "48": 7.08061,
+ "49": 7.51184,
+ "50": 6.9731
}
},
"iteration-time": {
@@ -289,56 +289,56 @@
"end_step": 50,
"step_interval": 1,
"values": {
- "1": 90.94511,
- "2": 1.54793,
- "3": 1.33035,
- "4": 2.25969,
- "5": 1.82487,
- "6": 1.71972,
- "7": 2.15404,
- "8": 1.61956,
- "9": 1.77326,
- "10": 1.72086,
- "11": 1.01952,
- "12": 1.02588,
- "13": 1.02874,
- "14": 1.02703,
- "15": 1.03114,
- "16": 1.03244,
- "17": 1.03532,
- "18": 1.04017,
- "19": 1.03111,
- "20": 1.03139,
- "21": 1.03293,
- "22": 1.03136,
- "23": 1.03187,
- "24": 1.0297,
- "25": 1.03561,
- "26": 1.5512,
- "27": 1.03857,
- "28": 1.02247,
- "29": 1.03252,
- "30": 1.02351,
- "31": 1.02701,
- "32": 1.0267,
- "33": 1.02921,
- "34": 1.02405,
- "35": 1.02405,
- "36": 1.04177,
- "37": 1.0449,
- "38": 1.04688,
- "39": 1.05181,
- "40": 1.04378,
- "41": 1.0421,
- "42": 1.04502,
- "43": 1.0336,
- "44": 1.05112,
- "45": 1.04838,
- "46": 1.03386,
- "47": 1.04806,
- "48": 1.04195,
- "49": 1.04121,
- "50": 1.03797
+ "1": 97.95665,
+ "2": 1.66988,
+ "3": 1.35644,
+ "4": 2.24552,
+ "5": 2.14285,
+ "6": 1.60272,
+ "7": 1.5113,
+ "8": 2.10932,
+ "9": 1.69738,
+ "10": 1.0561,
+ "11": 1.04064,
+ "12": 1.0335,
+ "13": 1.03186,
+ "14": 1.03406,
+ "15": 1.05897,
+ "16": 1.03516,
+ "17": 1.04396,
+ "18": 1.08073,
+ "19": 1.06079,
+ "20": 1.04178,
+ "21": 1.03726,
+ "22": 1.03706,
+ "23": 1.03878,
+ "24": 1.04111,
+ "25": 1.04952,
+ "26": 1.04497,
+ "27": 1.04672,
+ "28": 1.03793,
+ "29": 1.03092,
+ "30": 1.04813,
+ "31": 1.03205,
+ "32": 1.03729,
+ "33": 1.02557,
+ "34": 1.03623,
+ "35": 1.04247,
+ "36": 1.03261,
+ "37": 1.03911,
+ "38": 1.04764,
+ "39": 1.0376,
+ "40": 1.04918,
+ "41": 1.03907,
+ "42": 1.05227,
+ "43": 1.04186,
+ "44": 1.04266,
+ "45": 1.03786,
+ "46": 1.04673,
+ "47": 1.05766,
+ "48": 1.04958,
+ "49": 1.05312,
+ "50": 1.05239
}
}
}
\ No newline at end of file
diff --git a/tests/functional_tests/test_cases/moe/gpt3_mcore_te_tp2_pp2_ep4_etp1_mtp_resume_torch_dist_fp8/golden_values_dev_dgx_h100.json b/tests/functional_tests/test_cases/moe/gpt3_mcore_te_tp2_pp2_ep4_etp1_mtp_resume_torch_dist_fp8/golden_values_dev_dgx_h100.json
index ef8ee741272..a76d8667ec6 100644
--- a/tests/functional_tests/test_cases/moe/gpt3_mcore_te_tp2_pp2_ep4_etp1_mtp_resume_torch_dist_fp8/golden_values_dev_dgx_h100.json
+++ b/tests/functional_tests/test_cases/moe/gpt3_mcore_te_tp2_pp2_ep4_etp1_mtp_resume_torch_dist_fp8/golden_values_dev_dgx_h100.json
@@ -2,109 +2,343 @@
"lm loss": {
"start_step": 1,
"end_step": 50,
- "step_interval": 5,
+ "step_interval": 1,
"values": {
- "1": 11.0475,
- "5": 9.43078,
- "10": 8.89238,
- "15": 7.93732,
- "20": 7.77942,
- "25": 7.61408,
- "30": 7.57234,
- "35": 7.15189,
- "40": 7.48085,
- "45": 7.12056,
- "50": 6.96054
+ "1": 11.0474,
+ "2": 11.03765,
+ "3": 9.6074,
+ "4": 9.2648,
+ "5": 9.42291,
+ "6": 9.09511,
+ "7": 9.12753,
+ "8": 8.75686,
+ "9": 8.61627,
+ "10": 8.89295,
+ "11": 8.37933,
+ "12": 8.39932,
+ "13": 8.32626,
+ "14": 7.81437,
+ "15": 7.93661,
+ "16": 7.99492,
+ "17": 7.95458,
+ "18": 7.67733,
+ "19": 8.07234,
+ "20": 7.78815,
+ "21": 7.48342,
+ "22": 7.48177,
+ "23": 7.34879,
+ "24": 7.34465,
+ "25": 7.61117,
+ "26": 7.01605,
+ "27": 7.54878,
+ "28": 7.26655,
+ "29": 7.43507,
+ "30": 7.56529,
+ "31": 7.32669,
+ "32": 7.50645,
+ "33": 7.5577,
+ "34": 7.60977,
+ "35": 7.14607,
+ "36": 7.00597,
+ "37": 7.34071,
+ "38": 7.11796,
+ "39": 7.46649,
+ "40": 7.47443,
+ "41": 7.41032,
+ "42": 7.17365,
+ "43": 7.16495,
+ "44": 7.34265,
+ "45": 7.10918,
+ "46": 6.83934,
+ "47": 7.22335,
+ "48": 7.05732,
+ "49": 7.53394,
+ "50": 6.95951
}
},
"num-zeros": {
"start_step": 1,
"end_step": 50,
- "step_interval": 5,
+ "step_interval": 1,
"values": {
- "1": 38802620.0,
- "5": 243556240.0,
- "10": 716187584.0,
- "15": 614358336.0,
- "20": 677963584.0,
- "25": 736321856.0,
- "30": 505223648.0,
- "35": 548946176.0,
- "40": 412329664.0,
- "45": 376634624.0,
- "50": 205546672.0
+ "1": 38802536.0,
+ "2": 38543540.0,
+ "3": 38739408.0,
+ "4": 273756736.0,
+ "5": 205853584.0,
+ "6": 284244640.0,
+ "7": 652227968.0,
+ "8": 790994816.0,
+ "9": 762295424.0,
+ "10": 665870592.0,
+ "11": 618336384.0,
+ "12": 639816192.0,
+ "13": 699169600.0,
+ "14": 620502464.0,
+ "15": 623699456.0,
+ "16": 847396864.0,
+ "17": 601834432.0,
+ "18": 642855744.0,
+ "19": 668078912.0,
+ "20": 574651008.0,
+ "21": 608590080.0,
+ "22": 599821504.0,
+ "23": 558380672.0,
+ "24": 688014720.0,
+ "25": 500623296.0,
+ "26": 532887808.0,
+ "27": 506526976.0,
+ "28": 450900800.0,
+ "29": 528748480.0,
+ "30": 445603872.0,
+ "31": 457250368.0,
+ "32": 400653888.0,
+ "33": 347460640.0,
+ "34": 268919904.0,
+ "35": 495515584.0,
+ "36": 332139008.0,
+ "37": 446760768.0,
+ "38": 391328576.0,
+ "39": 378290400.0,
+ "40": 261331328.0,
+ "41": 368680832.0,
+ "42": 337485280.0,
+ "43": 337755968.0,
+ "44": 324657920.0,
+ "45": 216104608.0,
+ "46": 218159872.0,
+ "47": 302569184.0,
+ "48": 296505312.0,
+ "49": 280170176.0,
+ "50": 268486912.0
}
},
"mem-allocated-bytes": {
"start_step": 1,
"end_step": 50,
- "step_interval": 5,
+ "step_interval": 1,
"values": {
- "1": 7321331200.0,
- "5": 7321333248.0,
- "10": 7321333248.0,
- "15": 7321333248.0,
- "20": 7321333248.0,
- "25": 7321333248.0,
- "30": 7321333248.0,
- "35": 7321333248.0,
- "40": 7321333248.0,
- "45": 7321333248.0,
- "50": 7321333248.0
+ "1": 7316093440.0,
+ "2": 7316095488.0,
+ "3": 7316095488.0,
+ "4": 7316095488.0,
+ "5": 7316095488.0,
+ "6": 7316095488.0,
+ "7": 7316095488.0,
+ "8": 7316095488.0,
+ "9": 7316095488.0,
+ "10": 7316095488.0,
+ "11": 7316095488.0,
+ "12": 7316095488.0,
+ "13": 7316095488.0,
+ "14": 7316095488.0,
+ "15": 7316095488.0,
+ "16": 7316095488.0,
+ "17": 7316095488.0,
+ "18": 7316095488.0,
+ "19": 7316095488.0,
+ "20": 7316095488.0,
+ "21": 7316095488.0,
+ "22": 7316095488.0,
+ "23": 7316095488.0,
+ "24": 7316095488.0,
+ "25": 7316095488.0,
+ "26": 7316095488.0,
+ "27": 7316095488.0,
+ "28": 7316095488.0,
+ "29": 7316095488.0,
+ "30": 7316095488.0,
+ "31": 7316095488.0,
+ "32": 7316095488.0,
+ "33": 7316095488.0,
+ "34": 7316095488.0,
+ "35": 7316095488.0,
+ "36": 7316095488.0,
+ "37": 7316095488.0,
+ "38": 7316095488.0,
+ "39": 7316095488.0,
+ "40": 7316095488.0,
+ "41": 7316095488.0,
+ "42": 7316095488.0,
+ "43": 7316095488.0,
+ "44": 7316095488.0,
+ "45": 7316095488.0,
+ "46": 7316095488.0,
+ "47": 7316095488.0,
+ "48": 7316095488.0,
+ "49": 7316095488.0,
+ "50": 7316095488.0
}
},
"mem-max-allocated-bytes": {
"start_step": 1,
"end_step": 50,
- "step_interval": 5,
+ "step_interval": 1,
"values": {
- "1": 53176152064.0,
- "5": 55926337536.0,
- "10": 55926337536.0,
- "15": 55926337536.0,
- "20": 55926337536.0,
- "25": 56534257664.0,
- "30": 57393635328.0,
- "35": 57393635328.0,
- "40": 57578217472.0,
- "45": 57578217472.0,
- "50": 57578217472.0
+ "1": 53549867008.0,
+ "2": 56295710720.0,
+ "3": 56295710720.0,
+ "4": 56295710720.0,
+ "5": 56295710720.0,
+ "6": 56295710720.0,
+ "7": 56295710720.0,
+ "8": 56295710720.0,
+ "9": 56295710720.0,
+ "10": 56295710720.0,
+ "11": 56295710720.0,
+ "12": 56295710720.0,
+ "13": 56295710720.0,
+ "14": 56295710720.0,
+ "15": 56295710720.0,
+ "16": 56295710720.0,
+ "17": 56295710720.0,
+ "18": 56295710720.0,
+ "19": 56295710720.0,
+ "20": 56295710720.0,
+ "21": 56295710720.0,
+ "22": 56295710720.0,
+ "23": 56295710720.0,
+ "24": 56738553856.0,
+ "25": 56738553856.0,
+ "26": 56777162752.0,
+ "27": 56777162752.0,
+ "28": 56777162752.0,
+ "29": 56777162752.0,
+ "30": 56777162752.0,
+ "31": 56777162752.0,
+ "32": 56777162752.0,
+ "33": 56777162752.0,
+ "34": 56824344576.0,
+ "35": 57080135680.0,
+ "36": 57331695616.0,
+ "37": 57331695616.0,
+ "38": 57577013248.0,
+ "39": 57577013248.0,
+ "40": 57577013248.0,
+ "41": 57577013248.0,
+ "42": 57577013248.0,
+ "43": 57587191808.0,
+ "44": 57596944384.0,
+ "45": 57705652224.0,
+ "46": 57790390272.0,
+ "47": 57790390272.0,
+ "48": 57790390272.0,
+ "49": 57790390272.0,
+ "50": 57790390272.0
}
},
"mtp_1 loss": {
"start_step": 1,
"end_step": 50,
- "step_interval": 5,
+ "step_interval": 1,
"values": {
- "1": 11.0776,
- "5": 9.87653,
- "10": 9.02332,
- "15": 7.91471,
- "20": 7.75886,
- "25": 7.56825,
- "30": 7.53841,
- "35": 7.12192,
- "40": 7.44579,
- "45": 7.09307,
- "50": 6.94739
+ "1": 11.07756,
+ "2": 11.07651,
+ "3": 10.53063,
+ "4": 10.08611,
+ "5": 9.87524,
+ "6": 9.55366,
+ "7": 9.62345,
+ "8": 8.91012,
+ "9": 8.72228,
+ "10": 9.02504,
+ "11": 8.39501,
+ "12": 8.42504,
+ "13": 8.32334,
+ "14": 7.76976,
+ "15": 7.91789,
+ "16": 7.97018,
+ "17": 7.92051,
+ "18": 7.65266,
+ "19": 8.0377,
+ "20": 7.76074,
+ "21": 7.44752,
+ "22": 7.43657,
+ "23": 7.30984,
+ "24": 7.31186,
+ "25": 7.56562,
+ "26": 6.97201,
+ "27": 7.50933,
+ "28": 7.2266,
+ "29": 7.40633,
+ "30": 7.53569,
+ "31": 7.28904,
+ "32": 7.47424,
+ "33": 7.53526,
+ "34": 7.59404,
+ "35": 7.11968,
+ "36": 6.9867,
+ "37": 7.32338,
+ "38": 7.09605,
+ "39": 7.45524,
+ "40": 7.44706,
+ "41": 7.39271,
+ "42": 7.14573,
+ "43": 7.13128,
+ "44": 7.31399,
+ "45": 7.08836,
+ "46": 6.80158,
+ "47": 7.2062,
+ "48": 7.0468,
+ "49": 7.47982,
+ "50": 6.94494
}
},
"iteration-time": {
"start_step": 1,
"end_step": 50,
- "step_interval": 5,
+ "step_interval": 1,
"values": {
- "1": 51.33936,
- "5": 1.24167,
- "10": 1.14623,
- "15": 1.16973,
- "20": 1.23165,
- "25": 1.13719,
- "30": 1.15864,
- "35": 1.13509,
- "40": 1.14729,
- "45": 1.14136,
- "50": 1.13625
+ "1": 102.52307,
+ "2": 1.75305,
+ "3": 1.36681,
+ "4": 1.62808,
+ "5": 1.13714,
+ "6": 1.45805,
+ "7": 1.6121,
+ "8": 1.20031,
+ "9": 1.09784,
+ "10": 1.10383,
+ "11": 1.10878,
+ "12": 1.18093,
+ "13": 1.43808,
+ "14": 1.17223,
+ "15": 1.11575,
+ "16": 1.1159,
+ "17": 1.11727,
+ "18": 1.10751,
+ "19": 1.11189,
+ "20": 1.1082,
+ "21": 1.10459,
+ "22": 1.11252,
+ "23": 1.10744,
+ "24": 1.12218,
+ "25": 1.09823,
+ "26": 1.11657,
+ "27": 1.08949,
+ "28": 1.10254,
+ "29": 1.10189,
+ "30": 1.08963,
+ "31": 1.10454,
+ "32": 1.09654,
+ "33": 1.08747,
+ "34": 1.09674,
+ "35": 1.09106,
+ "36": 1.08904,
+ "37": 1.1178,
+ "38": 1.09379,
+ "39": 1.10306,
+ "40": 1.09998,
+ "41": 1.08808,
+ "42": 1.0941,
+ "43": 1.0919,
+ "44": 1.0813,
+ "45": 1.08715,
+ "46": 1.07061,
+ "47": 1.07098,
+ "48": 1.07438,
+ "49": 1.07469,
+ "50": 1.0719
}
}
}
\ No newline at end of file
diff --git a/tests/functional_tests/test_cases/moe/gpt3_mcore_te_tp2_pp2_ep4_etp1_resume_torch_dist_attn_cudagraph/golden_values_dev_dgx_h100.json b/tests/functional_tests/test_cases/moe/gpt3_mcore_te_tp2_pp2_ep4_etp1_resume_torch_dist_attn_cudagraph/golden_values_dev_dgx_h100.json
index f50f32bf276..c55faf839a8 100644
--- a/tests/functional_tests/test_cases/moe/gpt3_mcore_te_tp2_pp2_ep4_etp1_resume_torch_dist_attn_cudagraph/golden_values_dev_dgx_h100.json
+++ b/tests/functional_tests/test_cases/moe/gpt3_mcore_te_tp2_pp2_ep4_etp1_resume_torch_dist_attn_cudagraph/golden_values_dev_dgx_h100.json
@@ -6,54 +6,54 @@
"values": {
"1": 10.94944,
"2": 10.95158,
- "3": 10.50291,
- "4": 9.96373,
- "5": 9.94051,
- "6": 9.67323,
- "7": 10.22821,
- "8": 9.49736,
- "9": 9.54323,
- "10": 9.79347,
+ "3": 10.50318,
+ "4": 9.964,
+ "5": 9.94016,
+ "6": 9.67332,
+ "7": 10.23184,
+ "8": 9.4965,
+ "9": 9.54631,
+ "10": 9.79388,
"11": 9.3003,
- "12": 9.40372,
- "13": 9.39468,
- "14": 8.84935,
- "15": 9.02277,
- "16": 9.06983,
- "17": 9.04403,
- "18": 8.75568,
- "19": 9.17822,
- "20": 8.86078,
- "21": 8.53542,
- "22": 8.54991,
- "23": 8.42524,
- "24": 8.37607,
- "25": 8.63809,
- "26": 7.96681,
- "27": 8.57149,
- "28": 8.19023,
- "29": 8.39544,
- "30": 8.67048,
- "31": 8.28487,
- "32": 8.43358,
- "33": 8.55518,
- "34": 8.65834,
- "35": 8.07752,
- "36": 7.94541,
- "37": 8.29246,
- "38": 7.97753,
- "39": 8.38915,
- "40": 8.35513,
- "41": 8.31736,
- "42": 8.05606,
- "43": 8.03035,
- "44": 8.23838,
- "45": 8.09696,
- "46": 7.61491,
- "47": 8.15046,
- "48": 8.0039,
- "49": 8.38371,
- "50": 7.81253
+ "12": 9.40451,
+ "13": 9.39562,
+ "14": 8.8513,
+ "15": 9.02474,
+ "16": 9.07111,
+ "17": 9.04534,
+ "18": 8.75805,
+ "19": 9.1794,
+ "20": 8.86325,
+ "21": 8.5391,
+ "22": 8.55134,
+ "23": 8.42688,
+ "24": 8.38109,
+ "25": 8.63783,
+ "26": 7.96861,
+ "27": 8.57603,
+ "28": 8.1922,
+ "29": 8.3971,
+ "30": 8.67285,
+ "31": 8.28458,
+ "32": 8.43378,
+ "33": 8.55597,
+ "34": 8.65985,
+ "35": 8.07899,
+ "36": 7.94715,
+ "37": 8.29413,
+ "38": 7.97958,
+ "39": 8.39117,
+ "40": 8.35496,
+ "41": 8.31782,
+ "42": 8.05717,
+ "43": 8.03152,
+ "44": 8.24042,
+ "45": 8.0999,
+ "46": 7.61677,
+ "47": 8.15178,
+ "48": 8.00508,
+ "49": 8.38458,
+ "50": 7.81369
}
},
"num-zeros": {
@@ -61,56 +61,56 @@
"end_step": 50,
"step_interval": 1,
"values": {
- "1": 19403592.0,
- "2": 19274176.0,
- "3": 20945222.0,
- "4": 89687760.0,
- "5": 151693248.0,
- "6": 138938096.0,
- "7": 164021920.0,
- "8": 198936768.0,
- "9": 160969488.0,
- "10": 159820768.0,
- "11": 216424656.0,
- "12": 209851488.0,
- "13": 225333088.0,
- "14": 222140112.0,
- "15": 231619680.0,
- "16": 216080960.0,
- "17": 288314816.0,
- "18": 170463296.0,
- "19": 167479232.0,
- "20": 178590448.0,
- "21": 241500624.0,
- "22": 220658528.0,
- "23": 197474784.0,
- "24": 226071040.0,
- "25": 237749008.0,
- "26": 288417664.0,
- "27": 232076720.0,
- "28": 286654304.0,
- "29": 258070544.0,
- "30": 214923920.0,
- "31": 241275712.0,
- "32": 214510896.0,
- "33": 203527888.0,
- "34": 228752368.0,
- "35": 194293392.0,
- "36": 236711744.0,
- "37": 162157968.0,
- "38": 225545168.0,
- "39": 214299328.0,
- "40": 218746384.0,
- "41": 163931104.0,
- "42": 162458624.0,
- "43": 192453632.0,
- "44": 149739552.0,
- "45": 175646608.0,
- "46": 129510480.0,
- "47": 170153408.0,
- "48": 157697168.0,
- "49": 92955200.0,
- "50": 157824256.0
+ "1": 19403652.0,
+ "2": 19274102.0,
+ "3": 19373168.0,
+ "4": 86562120.0,
+ "5": 151677296.0,
+ "6": 142091232.0,
+ "7": 167132032.0,
+ "8": 197337088.0,
+ "9": 168836496.0,
+ "10": 162963792.0,
+ "11": 211653824.0,
+ "12": 214575616.0,
+ "13": 231549168.0,
+ "14": 220571728.0,
+ "15": 250508240.0,
+ "16": 168968368.0,
+ "17": 294610112.0,
+ "18": 167327952.0,
+ "19": 156385504.0,
+ "20": 177007072.0,
+ "21": 219468816.0,
+ "22": 217511168.0,
+ "23": 194318208.0,
+ "24": 208788192.0,
+ "25": 240820928.0,
+ "26": 250667072.0,
+ "27": 235205856.0,
+ "28": 285071552.0,
+ "29": 270668736.0,
+ "30": 241596448.0,
+ "31": 256938208.0,
+ "32": 252232640.0,
+ "33": 213058752.0,
+ "34": 217720576.0,
+ "35": 172316416.0,
+ "36": 246137120.0,
+ "37": 228162320.0,
+ "38": 238162048.0,
+ "39": 211207168.0,
+ "40": 206162560.0,
+ "41": 151397232.0,
+ "42": 206473424.0,
+ "43": 175165248.0,
+ "44": 182768560.0,
+ "45": 158317856.0,
+ "46": 159388704.0,
+ "47": 152897904.0,
+ "48": 143548896.0,
+ "49": 124357696.0,
+ "50": 151519648.0
}
},
"mem-allocated-bytes": {
@@ -118,56 +118,56 @@
"end_step": 50,
"step_interval": 1,
"values": {
- "1": 4876471296.0,
- "2": 4876535296.0,
- "3": 4875369984.0,
- "4": 4874512896.0,
- "5": 4874505728.0,
- "6": 4876898816.0,
- "7": 4875386368.0,
- "8": 4876464640.0,
- "9": 4876400128.0,
- "10": 4877448704.0,
- "11": 4876193280.0,
- "12": 4874407424.0,
- "13": 4875226624.0,
- "14": 4875415040.0,
- "15": 4876397056.0,
- "16": 4877806080.0,
- "17": 4876205568.0,
- "18": 4876743168.0,
- "19": 4875044352.0,
- "20": 4877310464.0,
- "21": 4875642368.0,
- "22": 4874806784.0,
- "23": 4875531776.0,
- "24": 4878220800.0,
- "25": 4875477504.0,
- "26": 4877613568.0,
- "27": 4875030016.0,
- "28": 4875365888.0,
- "29": 4876291584.0,
- "30": 4876465664.0,
- "31": 4874710528.0,
- "32": 4875980288.0,
- "33": 4874096128.0,
- "34": 4875379200.0,
- "35": 4875995648.0,
- "36": 4876016128.0,
- "37": 4874497536.0,
- "38": 4875453952.0,
- "39": 4875932160.0,
- "40": 4876112384.0,
- "41": 4875683328.0,
- "42": 4877188608.0,
- "43": 4875977216.0,
- "44": 4878347776.0,
- "45": 4876845568.0,
- "46": 4875212288.0,
- "47": 4876330496.0,
- "48": 4875971072.0,
- "49": 4875368960.0,
- "50": 4875349504.0
+ "1": 4875597824.0,
+ "2": 4875363840.0,
+ "3": 4874979840.0,
+ "4": 4874899968.0,
+ "5": 4875749888.0,
+ "6": 4876656128.0,
+ "7": 4875178496.0,
+ "8": 4874036736.0,
+ "9": 4876568064.0,
+ "10": 4876058112.0,
+ "11": 4876045824.0,
+ "12": 4874515968.0,
+ "13": 4875086336.0,
+ "14": 4874568192.0,
+ "15": 4875987456.0,
+ "16": 4874790400.0,
+ "17": 4875477504.0,
+ "18": 4875512320.0,
+ "19": 4876186112.0,
+ "20": 4875747840.0,
+ "21": 4874790400.0,
+ "22": 4876221952.0,
+ "23": 4874534400.0,
+ "24": 4875733504.0,
+ "25": 4875019776.0,
+ "26": 4875168256.0,
+ "27": 4874978816.0,
+ "28": 4875781632.0,
+ "29": 4876329472.0,
+ "30": 4875107840.0,
+ "31": 4874253824.0,
+ "32": 4874167808.0,
+ "33": 4876044800.0,
+ "34": 4875914752.0,
+ "35": 4874962432.0,
+ "36": 4875862528.0,
+ "37": 4877336064.0,
+ "38": 4875002368.0,
+ "39": 4874599936.0,
+ "40": 4874880512.0,
+ "41": 4875294208.0,
+ "42": 4875419136.0,
+ "43": 4875780608.0,
+ "44": 4874780160.0,
+ "45": 4875191808.0,
+ "46": 4875717120.0,
+ "47": 4874050048.0,
+ "48": 4875580928.0,
+ "49": 4875412992.0,
+ "50": 4875462144.0
}
},
"mem-max-allocated-bytes": {
@@ -175,56 +175,56 @@
"end_step": 50,
"step_interval": 1,
"values": {
- "1": 41199984640.0,
- "2": 41199984640.0,
- "3": 41199984640.0,
- "4": 41199984640.0,
- "5": 41199984640.0,
- "6": 41199984640.0,
- "7": 41199984640.0,
- "8": 41199984640.0,
- "9": 41199984640.0,
- "10": 41199984640.0,
- "11": 41199984640.0,
- "12": 41199984640.0,
- "13": 41199984640.0,
- "14": 41199984640.0,
- "15": 41199984640.0,
- "16": 41199984640.0,
- "17": 41199984640.0,
- "18": 41199984640.0,
- "19": 41199984640.0,
- "20": 41199984640.0,
- "21": 41199984640.0,
- "22": 41199984640.0,
- "23": 41199984640.0,
- "24": 41199984640.0,
- "25": 41199984640.0,
- "26": 41199984640.0,
- "27": 41199984640.0,
- "28": 41199984640.0,
- "29": 41199984640.0,
- "30": 41199984640.0,
- "31": 41199984640.0,
- "32": 41199984640.0,
- "33": 41199984640.0,
- "34": 41199984640.0,
- "35": 41199984640.0,
- "36": 41199984640.0,
- "37": 41199984640.0,
- "38": 41199984640.0,
- "39": 41199984640.0,
- "40": 41199984640.0,
- "41": 41199984640.0,
- "42": 41199984640.0,
- "43": 41199984640.0,
- "44": 41199984640.0,
- "45": 41199984640.0,
- "46": 41199984640.0,
- "47": 41199984640.0,
- "48": 41199984640.0,
- "49": 41199984640.0,
- "50": 41199984640.0
+ "1": 41201033216.0,
+ "2": 41201033216.0,
+ "3": 41201033216.0,
+ "4": 41201033216.0,
+ "5": 41201033216.0,
+ "6": 41201033216.0,
+ "7": 41201033216.0,
+ "8": 41201033216.0,
+ "9": 41201033216.0,
+ "10": 41201033216.0,
+ "11": 41201033216.0,
+ "12": 41201033216.0,
+ "13": 41201033216.0,
+ "14": 41201033216.0,
+ "15": 41201033216.0,
+ "16": 41201033216.0,
+ "17": 41201033216.0,
+ "18": 41201033216.0,
+ "19": 41201033216.0,
+ "20": 41201033216.0,
+ "21": 41201033216.0,
+ "22": 41201033216.0,
+ "23": 41201033216.0,
+ "24": 41201033216.0,
+ "25": 41201033216.0,
+ "26": 41201033216.0,
+ "27": 41201033216.0,
+ "28": 41201033216.0,
+ "29": 41201033216.0,
+ "30": 41201033216.0,
+ "31": 41201033216.0,
+ "32": 41201033216.0,
+ "33": 41201033216.0,
+ "34": 41201033216.0,
+ "35": 41201033216.0,
+ "36": 41201033216.0,
+ "37": 41201033216.0,
+ "38": 41201033216.0,
+ "39": 41201033216.0,
+ "40": 41201033216.0,
+ "41": 41201033216.0,
+ "42": 41201033216.0,
+ "43": 41201033216.0,
+ "44": 41201033216.0,
+ "45": 41201033216.0,
+ "46": 41201033216.0,
+ "47": 41201033216.0,
+ "48": 41201033216.0,
+ "49": 41201033216.0,
+ "50": 41201033216.0
}
},
"iteration-time": {
@@ -232,56 +232,56 @@
"end_step": 50,
"step_interval": 1,
"values": {
- "1": 86.59245,
- "2": 1.11188,
- "3": 0.94659,
- "4": 0.89686,
- "5": 1.40432,
- "6": 1.06239,
- "7": 1.03181,
- "8": 1.07838,
- "9": 0.88529,
- "10": 0.87346,
- "11": 0.9764,
- "12": 0.87397,
- "13": 0.87922,
- "14": 0.87464,
- "15": 0.86356,
- "16": 0.88539,
- "17": 0.86198,
- "18": 0.86676,
- "19": 0.85335,
- "20": 0.85904,
- "21": 0.84697,
- "22": 0.84984,
- "23": 0.84683,
- "24": 0.85172,
- "25": 0.84975,
- "26": 0.86347,
- "27": 0.86726,
- "28": 0.84853,
- "29": 0.84946,
- "30": 0.85197,
- "31": 0.85026,
- "32": 0.84681,
- "33": 0.84571,
- "34": 0.85295,
- "35": 0.8568,
- "36": 0.84946,
- "37": 0.8495,
- "38": 0.84754,
- "39": 0.85264,
- "40": 0.8452,
- "41": 0.84517,
- "42": 0.84876,
- "43": 0.84152,
- "44": 0.84772,
- "45": 0.84803,
- "46": 0.84148,
- "47": 0.84697,
- "48": 0.84232,
- "49": 0.84236,
- "50": 0.84249
+ "1": 84.85893,
+ "2": 1.16099,
+ "3": 0.98814,
+ "4": 0.90006,
+ "5": 1.44704,
+ "6": 1.12424,
+ "7": 1.08423,
+ "8": 1.07558,
+ "9": 1.1513,
+ "10": 0.88417,
+ "11": 1.07532,
+ "12": 0.88519,
+ "13": 0.87318,
+ "14": 0.87758,
+ "15": 0.87276,
+ "16": 0.8776,
+ "17": 0.86863,
+ "18": 0.87011,
+ "19": 0.86845,
+ "20": 0.86617,
+ "21": 0.85521,
+ "22": 0.86783,
+ "23": 0.86126,
+ "24": 0.85746,
+ "25": 0.85758,
+ "26": 0.86093,
+ "27": 0.85634,
+ "28": 0.85365,
+ "29": 0.86147,
+ "30": 0.86891,
+ "31": 0.85512,
+ "32": 0.85344,
+ "33": 0.85409,
+ "34": 0.85597,
+ "35": 0.85605,
+ "36": 0.84565,
+ "37": 0.84908,
+ "38": 0.85623,
+ "39": 0.8586,
+ "40": 0.87856,
+ "41": 0.85187,
+ "42": 0.86298,
+ "43": 0.85814,
+ "44": 0.85706,
+ "45": 0.85473,
+ "46": 0.85417,
+ "47": 0.85861,
+ "48": 0.85261,
+ "49": 0.85118,
+ "50": 0.84383
}
}
}
\ No newline at end of file
diff --git a/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_ep8_resume_torch_dist_dist_muon/golden_values_dev_dgx_h100.json b/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_ep8_resume_torch_dist_dist_muon/golden_values_dev_dgx_h100.json
new file mode 100644
index 00000000000..13bfff6c765
--- /dev/null
+++ b/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_ep8_resume_torch_dist_dist_muon/golden_values_dev_dgx_h100.json
@@ -0,0 +1,537 @@
+{
+ "lm loss": {
+ "start_step": 1,
+ "end_step": 100,
+ "step_interval": 1,
+ "values": {
+ "1": 10.81131,
+ "2": 10.83052,
+ "3": 10.82065,
+ "4": 10.81318,
+ "5": 10.84363,
+ "6": 10.84747,
+ "7": 10.85338,
+ "8": 10.83667,
+ "9": 10.8468,
+ "10": 10.7825,
+ "11": 10.85216,
+ "12": 10.86296,
+ "13": 10.85469,
+ "14": 10.88433,
+ "15": 10.87748,
+ "16": 10.84698,
+ "17": 10.83109,
+ "18": 10.86619,
+ "19": 10.84965,
+ "20": 10.84503,
+ "21": 10.84788,
+ "22": 10.79628,
+ "23": 10.88209,
+ "24": 10.83272,
+ "25": 10.82407,
+ "26": 10.84275,
+ "27": 10.85284,
+ "28": 10.87701,
+ "29": 10.8644,
+ "30": 10.81288,
+ "31": 10.78708,
+ "32": 10.85504,
+ "33": 10.85616,
+ "34": 10.84955,
+ "35": 10.83713,
+ "36": 10.80378,
+ "37": 10.83848,
+ "38": 10.80562,
+ "39": 10.8422,
+ "40": 10.80302,
+ "41": 10.84057,
+ "42": 10.84402,
+ "43": 10.81002,
+ "44": 10.80246,
+ "45": 10.78649,
+ "46": 10.80799,
+ "47": 10.817,
+ "48": 10.80324,
+ "49": 10.78157,
+ "50": 10.80218,
+ "51": 10.82262,
+ "52": 10.80415,
+ "53": 10.83258,
+ "54": 10.81542,
+ "55": 10.82524,
+ "56": 10.77667,
+ "57": 10.75278,
+ "58": 10.8075,
+ "59": 10.79063,
+ "60": 10.73975,
+ "61": 10.79974,
+ "62": 10.81288,
+ "63": 10.72014,
+ "64": 10.78563,
+ "65": 10.68987,
+ "66": 10.76119,
+ "67": 10.73431,
+ "68": 10.80192,
+ "69": 10.78336,
+ "70": 10.77619,
+ "71": 10.76644,
+ "72": 10.73613,
+ "73": 10.72971,
+ "74": 10.62238,
+ "75": 10.69054,
+ "76": 10.65471,
+ "77": 10.82153,
+ "78": 10.76381,
+ "79": 10.705,
+ "80": 10.69388,
+ "81": 10.72432,
+ "82": 10.74257,
+ "83": 10.66783,
+ "84": 10.69845,
+ "85": 10.71465,
+ "86": 10.63873,
+ "87": 10.71762,
+ "88": 10.73506,
+ "89": 10.71394,
+ "90": 10.74649,
+ "91": 10.64881,
+ "92": 10.64684,
+ "93": 10.60201,
+ "94": 10.53283,
+ "95": 10.66127,
+ "96": 10.67245,
+ "97": 10.61405,
+ "98": 10.68482,
+ "99": 10.52006,
+ "100": 10.61575
+ }
+ },
+ "num-zeros": {
+ "start_step": 1,
+ "end_step": 100,
+ "step_interval": 1,
+ "values": {
+ "1": 1216.0,
+ "2": 1361.0,
+ "3": 1290.0,
+ "4": 1255.0,
+ "5": 1433.0,
+ "6": 1548.0,
+ "7": 1277.0,
+ "8": 1340.0,
+ "9": 1318.0,
+ "10": 1284.0,
+ "11": 1307.0,
+ "12": 1174.0,
+ "13": 1268.0,
+ "14": 1421.0,
+ "15": 1220.0,
+ "16": 1229.0,
+ "17": 1346.0,
+ "18": 1311.0,
+ "19": 1252.0,
+ "20": 1273.0,
+ "21": 1283.0,
+ "22": 1145.0,
+ "23": 1454.0,
+ "24": 1348.0,
+ "25": 1258.0,
+ "26": 1212.0,
+ "27": 1343.0,
+ "28": 1389.0,
+ "29": 1282.0,
+ "30": 1203.0,
+ "31": 1152.0,
+ "32": 1244.0,
+ "33": 1290.0,
+ "34": 1082.0,
+ "35": 1176.0,
+ "36": 1168.0,
+ "37": 1242.0,
+ "38": 1316.0,
+ "39": 1589.0,
+ "40": 1218.0,
+ "41": 1391.0,
+ "42": 1137.0,
+ "43": 1234.0,
+ "44": 1265.0,
+ "45": 1194.0,
+ "46": 1124.0,
+ "47": 1300.0,
+ "48": 1102.0,
+ "49": 1124.0,
+ "50": 1211.0,
+ "51": 1266.0,
+ "52": 1269.0,
+ "53": 1355.0,
+ "54": 1212.0,
+ "55": 1137.0,
+ "56": 1313.0,
+ "57": 1288.0,
+ "58": 1341.0,
+ "59": 1261.0,
+ "60": 1287.0,
+ "61": 1139.0,
+ "62": 1205.0,
+ "63": 1265.0,
+ "64": 1350.0,
+ "65": 1195.0,
+ "66": 1207.0,
+ "67": 1121.0,
+ "68": 1212.0,
+ "69": 1335.0,
+ "70": 1356.0,
+ "71": 1316.0,
+ "72": 1232.0,
+ "73": 1121.0,
+ "74": 1130.0,
+ "75": 1295.0,
+ "76": 1335.0,
+ "77": 1371.0,
+ "78": 1336.0,
+ "79": 1042.0,
+ "80": 1149.0,
+ "81": 1117.0,
+ "82": 1202.0,
+ "83": 1289.0,
+ "84": 1140.0,
+ "85": 1323.0,
+ "86": 1219.0,
+ "87": 1219.0,
+ "88": 1221.0,
+ "89": 1294.0,
+ "90": 1402.0,
+ "91": 1197.0,
+ "92": 1269.0,
+ "93": 1106.0,
+ "94": 960.0,
+ "95": 1192.0,
+ "96": 1253.0,
+ "97": 1148.0,
+ "98": 1218.0,
+ "99": 1273.0,
+ "100": 1249.0
+ }
+ },
+ "mem-allocated-bytes": {
+ "start_step": 1,
+ "end_step": 100,
+ "step_interval": 1,
+ "values": {
+ "1": 994082816.0,
+ "2": 994053120.0,
+ "3": 994100224.0,
+ "4": 994081280.0,
+ "5": 994103808.0,
+ "6": 994043392.0,
+ "7": 994066944.0,
+ "8": 994074112.0,
+ "9": 994091008.0,
+ "10": 994104320.0,
+ "11": 994077696.0,
+ "12": 994044416.0,
+ "13": 994100736.0,
+ "14": 994012160.0,
+ "15": 994057216.0,
+ "16": 993989120.0,
+ "17": 994107904.0,
+ "18": 994082304.0,
+ "19": 994089472.0,
+ "20": 994008064.0,
+ "21": 994033152.0,
+ "22": 994105344.0,
+ "23": 994081280.0,
+ "24": 994021888.0,
+ "25": 994152960.0,
+ "26": 994058752.0,
+ "27": 994118144.0,
+ "28": 994044416.0,
+ "29": 994075648.0,
+ "30": 994039296.0,
+ "31": 994107392.0,
+ "32": 994037760.0,
+ "33": 994046976.0,
+ "34": 994015232.0,
+ "35": 994064384.0,
+ "36": 994078208.0,
+ "37": 994037248.0,
+ "38": 994120192.0,
+ "39": 994128896.0,
+ "40": 994016768.0,
+ "41": 994044928.0,
+ "42": 994063872.0,
+ "43": 994075648.0,
+ "44": 994180096.0,
+ "45": 994053632.0,
+ "46": 994070016.0,
+ "47": 994091520.0,
+ "48": 994076672.0,
+ "49": 994042368.0,
+ "50": 994061312.0,
+ "51": 994132992.0,
+ "52": 994076160.0,
+ "53": 994139136.0,
+ "54": 994086400.0,
+ "55": 994076160.0,
+ "56": 994066944.0,
+ "57": 994113536.0,
+ "58": 994111488.0,
+ "59": 994096128.0,
+ "60": 994060288.0,
+ "61": 994060800.0,
+ "62": 994054656.0,
+ "63": 994068992.0,
+ "64": 994058752.0,
+ "65": 994064896.0,
+ "66": 994074624.0,
+ "67": 994061824.0,
+ "68": 994071552.0,
+ "69": 994058240.0,
+ "70": 994103808.0,
+ "71": 994077184.0,
+ "72": 994002944.0,
+ "73": 994104320.0,
+ "74": 994116608.0,
+ "75": 994081792.0,
+ "76": 994104320.0,
+ "77": 994054656.0,
+ "78": 994114048.0,
+ "79": 994085376.0,
+ "80": 994039296.0,
+ "81": 994073600.0,
+ "82": 994020864.0,
+ "83": 994123776.0,
+ "84": 994103296.0,
+ "85": 994070528.0,
+ "86": 994070016.0,
+ "87": 994093056.0,
+ "88": 994079232.0,
+ "89": 994066432.0,
+ "90": 994060800.0,
+ "91": 994116096.0,
+ "92": 994098176.0,
+ "93": 994076672.0,
+ "94": 994083840.0,
+ "95": 994082816.0,
+ "96": 994086400.0,
+ "97": 994094080.0,
+ "98": 994070016.0,
+ "99": 994088448.0,
+ "100": 994124800.0
+ }
+ },
+ "mem-max-allocated-bytes": {
+ "start_step": 1,
+ "end_step": 100,
+ "step_interval": 1,
+ "values": {
+ "1": 3209166336.0,
+ "2": 3482067456.0,
+ "3": 3514878464.0,
+ "4": 3514878464.0,
+ "5": 3515977728.0,
+ "6": 3515977728.0,
+ "7": 3515977728.0,
+ "8": 3515977728.0,
+ "9": 3515977728.0,
+ "10": 3519236608.0,
+ "11": 3519236608.0,
+ "12": 3519236608.0,
+ "13": 3519236608.0,
+ "14": 3519236608.0,
+ "15": 3519236608.0,
+ "16": 3519236608.0,
+ "17": 3519236608.0,
+ "18": 3519236608.0,
+ "19": 3519236608.0,
+ "20": 3519236608.0,
+ "21": 3519236608.0,
+ "22": 3519236608.0,
+ "23": 3519236608.0,
+ "24": 3519236608.0,
+ "25": 3549031424.0,
+ "26": 3549031424.0,
+ "27": 3549031424.0,
+ "28": 3549031424.0,
+ "29": 3549031424.0,
+ "30": 3549031424.0,
+ "31": 3549031424.0,
+ "32": 3549031424.0,
+ "33": 3549031424.0,
+ "34": 3549031424.0,
+ "35": 3549031424.0,
+ "36": 3549031424.0,
+ "37": 3549031424.0,
+ "38": 3549031424.0,
+ "39": 3549031424.0,
+ "40": 3549031424.0,
+ "41": 3549031424.0,
+ "42": 3549031424.0,
+ "43": 3549031424.0,
+ "44": 3560927744.0,
+ "45": 3560927744.0,
+ "46": 3560927744.0,
+ "47": 3560927744.0,
+ "48": 3560927744.0,
+ "49": 3560927744.0,
+ "50": 3560927744.0,
+ "51": 3560927744.0,
+ "52": 3560927744.0,
+ "53": 3560927744.0,
+ "54": 3560927744.0,
+ "55": 3560927744.0,
+ "56": 3560927744.0,
+ "57": 3560927744.0,
+ "58": 3560927744.0,
+ "59": 3560927744.0,
+ "60": 3560927744.0,
+ "61": 3560927744.0,
+ "62": 3560927744.0,
+ "63": 3560927744.0,
+ "64": 3560927744.0,
+ "65": 3560927744.0,
+ "66": 3560927744.0,
+ "67": 3560927744.0,
+ "68": 3560927744.0,
+ "69": 3560927744.0,
+ "70": 3560927744.0,
+ "71": 3560927744.0,
+ "72": 3560927744.0,
+ "73": 3560927744.0,
+ "74": 3560927744.0,
+ "75": 3560927744.0,
+ "76": 3560927744.0,
+ "77": 3560927744.0,
+ "78": 3560927744.0,
+ "79": 3560927744.0,
+ "80": 3560927744.0,
+ "81": 3560927744.0,
+ "82": 3560927744.0,
+ "83": 3560927744.0,
+ "84": 3560927744.0,
+ "85": 3560927744.0,
+ "86": 3560927744.0,
+ "87": 3560927744.0,
+ "88": 3560927744.0,
+ "89": 3560927744.0,
+ "90": 3560927744.0,
+ "91": 3560927744.0,
+ "92": 3560927744.0,
+ "93": 3560927744.0,
+ "94": 3560927744.0,
+ "95": 3560927744.0,
+ "96": 3560927744.0,
+ "97": 3560927744.0,
+ "98": 3560927744.0,
+ "99": 3560927744.0,
+ "100": 3560927744.0
+ }
+ },
+ "iteration-time": {
+ "start_step": 1,
+ "end_step": 100,
+ "step_interval": 1,
+ "values": {
+ "1": 12.93942,
+ "2": 0.24599,
+ "3": 0.18905,
+ "4": 0.15958,
+ "5": 0.17376,
+ "6": 0.15827,
+ "7": 0.1625,
+ "8": 0.15602,
+ "9": 0.14535,
+ "10": 0.15058,
+ "11": 0.15764,
+ "12": 0.14977,
+ "13": 0.14045,
+ "14": 0.14809,
+ "15": 0.14641,
+ "16": 0.14226,
+ "17": 0.14811,
+ "18": 0.14049,
+ "19": 0.14226,
+ "20": 0.14343,
+ "21": 0.13924,
+ "22": 0.13727,
+ "23": 0.14079,
+ "24": 0.13602,
+ "25": 0.1322,
+ "26": 0.14315,
+ "27": 0.1347,
+ "28": 0.13221,
+ "29": 0.14595,
+ "30": 0.13083,
+ "31": 0.13326,
+ "32": 0.14065,
+ "33": 0.1383,
+ "34": 0.12953,
+ "35": 0.12541,
+ "36": 0.13129,
+ "37": 0.13317,
+ "38": 0.13535,
+ "39": 0.14664,
+ "40": 0.13368,
+ "41": 0.13115,
+ "42": 0.13308,
+ "43": 0.14022,
+ "44": 0.12946,
+ "45": 0.134,
+ "46": 0.12714,
+ "47": 0.13354,
+ "48": 0.13449,
+ "49": 0.13041,
+ "50": 0.13278,
+ "51": 0.14094,
+ "52": 0.12708,
+ "53": 0.13344,
+ "54": 0.13202,
+ "55": 0.13136,
+ "56": 0.13508,
+ "57": 0.13876,
+ "58": 0.13736,
+ "59": 0.12763,
+ "60": 0.13185,
+ "61": 0.12865,
+ "62": 0.13343,
+ "63": 0.13403,
+ "64": 0.12891,
+ "65": 0.13097,
+ "66": 0.12741,
+ "67": 0.13812,
+ "68": 0.13131,
+ "69": 0.13389,
+ "70": 0.13833,
+ "71": 0.12822,
+ "72": 0.12851,
+ "73": 0.13747,
+ "74": 0.13403,
+ "75": 0.12846,
+ "76": 0.13178,
+ "77": 0.12922,
+ "78": 0.12906,
+ "79": 0.12676,
+ "80": 0.13361,
+ "81": 0.12867,
+ "82": 0.1295,
+ "83": 0.12961,
+ "84": 0.12795,
+ "85": 0.13547,
+ "86": 0.13067,
+ "87": 0.13455,
+ "88": 0.13573,
+ "89": 0.12632,
+ "90": 0.13428,
+ "91": 0.13373,
+ "92": 0.12985,
+ "93": 0.1291,
+ "94": 0.12972,
+ "95": 0.13089,
+ "96": 0.13658,
+ "97": 0.12767,
+ "98": 0.14125,
+ "99": 0.13279,
+ "100": 0.12715
+ }
+ }
+}
\ No newline at end of file
diff --git a/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_ep8_resume_torch_dist_dist_muon/model_config.yaml b/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_ep8_resume_torch_dist_dist_muon/model_config.yaml
new file mode 100644
index 00000000000..1d0ef19232e
--- /dev/null
+++ b/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_ep8_resume_torch_dist_dist_muon/model_config.yaml
@@ -0,0 +1,66 @@
+ENV_VARS:
+ CUDA_DEVICE_MAX_CONNECTIONS: 1
+ NVTE_ALLOW_NONDETERMINISTIC_ALGO: 0
+ NCCL_ALGO: Ring
+ CUBLAS_WORKSPACE_CONFIG: :4096:8
+MODEL_ARGS:
+ --num-layers: 12
+ --hidden-size: 512
+ --num-attention-heads: 8
+ --log-params-norm: true
+ --log-num-zeros-in-grad: true
+ --log-validation-ppl-to-tensorboard: true
+ --log-timers-to-tensorboard: true
+ --tensorboard-dir: ${TENSORBOARD_PATH}
+ --micro-batch-size: 4
+ --global-batch-size: 32
+ --seq-length: 1024
+ --max-position-embeddings: 1024
+ --disable-bias-linear: true
+ --train-iters: 100
+ --timing-log-level: 0
+ --lr-decay-iters: 320000
+ --save: ${CHECKPOINT_SAVE_PATH}
+ --load: ${CHECKPOINT_LOAD_PATH}
+ --data-path: ${DATA_PATH}/text/the_pile/shard00/my-gpt3_00_text_document
+ --vocab-file: ${DATA_PATH}/text/the_pile/shard00/bpe/vocab.json
+ --merge-file: ${DATA_PATH}/text/the_pile/shard00/bpe/merges.txt
+ --split: 949,50,1
+ --distributed-backend: nccl
+ --lr: 0.00015
+ --lr-decay-style: cosine
+ --min-lr: 1.0e-5
+ --weight-decay: 1e-2
+ --clip-grad: 1.0
+ --lr-warmup-fraction: .01
+ --log-interval: 1
+ --save-interval: 50
+ --eval-interval: 1000
+ --eval-iters: 10
+ --transformer-impl: transformer_engine
+ --tensor-model-parallel-size: 1
+ --pipeline-model-parallel-size: 1
+ --expert-model-parallel-size: 8
+ --num-experts: 8
+ --moe-token-dispatcher-type: allgather
+ --moe-router-load-balancing-type: aux_loss
+ --moe-router-topk: 2
+ --moe-router-dtype: fp32
+ --moe-ffn-hidden-size: 1024
+ --moe-grouped-gemm: true
+ --ckpt-fully-parallel-load: true
+ --deterministic-mode: true
+ --no-gradient-accumulation-fusion: true
+ --attention-softmax-in-fp32: true
+ --use-checkpoint-opt_param-scheduler: true
+ --use-mcore-models: true
+ --ckpt-format: torch_dist
+ --data-cache-path: ${DATA_CACHE_PATH}
+ --bf16: true
+ --no-bias-gelu-fusion: true
+ --log-memory-to-tensorboard: true
+ --optimizer: dist_muon
+ --muon-momentum: 0.9
+ --muon-extra-scale-factor: 0.2
+ --muon-scale-mode: spectral
+TEST_TYPE: ckpt-resume
diff --git a/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_ep8_resume_torch_dist_muon/golden_values_dev_dgx_h100.json b/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_ep8_resume_torch_dist_muon/golden_values_dev_dgx_h100.json
new file mode 100644
index 00000000000..dd58e4cb1e6
--- /dev/null
+++ b/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_ep8_resume_torch_dist_muon/golden_values_dev_dgx_h100.json
@@ -0,0 +1,537 @@
+{
+ "lm loss": {
+ "start_step": 1,
+ "end_step": 100,
+ "step_interval": 1,
+ "values": {
+ "1": 10.81103,
+ "2": 10.83065,
+ "3": 10.82041,
+ "4": 10.81308,
+ "5": 10.84384,
+ "6": 10.84719,
+ "7": 10.85346,
+ "8": 10.83656,
+ "9": 10.84673,
+ "10": 10.78252,
+ "11": 10.85208,
+ "12": 10.86326,
+ "13": 10.85438,
+ "14": 10.88369,
+ "15": 10.87797,
+ "16": 10.84675,
+ "17": 10.83091,
+ "18": 10.86618,
+ "19": 10.84893,
+ "20": 10.84566,
+ "21": 10.8476,
+ "22": 10.79629,
+ "23": 10.88263,
+ "24": 10.83271,
+ "25": 10.82477,
+ "26": 10.84285,
+ "27": 10.85338,
+ "28": 10.87687,
+ "29": 10.86419,
+ "30": 10.81306,
+ "31": 10.78638,
+ "32": 10.85513,
+ "33": 10.85601,
+ "34": 10.8492,
+ "35": 10.83702,
+ "36": 10.80421,
+ "37": 10.83785,
+ "38": 10.80484,
+ "39": 10.84147,
+ "40": 10.80332,
+ "41": 10.83988,
+ "42": 10.84406,
+ "43": 10.81051,
+ "44": 10.8022,
+ "45": 10.78682,
+ "46": 10.80806,
+ "47": 10.81757,
+ "48": 10.80298,
+ "49": 10.78142,
+ "50": 10.8023,
+ "51": 10.82205,
+ "52": 10.80341,
+ "53": 10.83273,
+ "54": 10.81558,
+ "55": 10.82551,
+ "56": 10.77761,
+ "57": 10.7527,
+ "58": 10.80801,
+ "59": 10.79071,
+ "60": 10.73971,
+ "61": 10.80018,
+ "62": 10.81299,
+ "63": 10.72069,
+ "64": 10.78573,
+ "65": 10.69001,
+ "66": 10.76067,
+ "67": 10.73433,
+ "68": 10.80225,
+ "69": 10.7835,
+ "70": 10.77632,
+ "71": 10.76604,
+ "72": 10.736,
+ "73": 10.72965,
+ "74": 10.62244,
+ "75": 10.69059,
+ "76": 10.65429,
+ "77": 10.82179,
+ "78": 10.76341,
+ "79": 10.70461,
+ "80": 10.69433,
+ "81": 10.72473,
+ "82": 10.74232,
+ "83": 10.66784,
+ "84": 10.69896,
+ "85": 10.7144,
+ "86": 10.63886,
+ "87": 10.71783,
+ "88": 10.73541,
+ "89": 10.7139,
+ "90": 10.74667,
+ "91": 10.64906,
+ "92": 10.64667,
+ "93": 10.60204,
+ "94": 10.53296,
+ "95": 10.66128,
+ "96": 10.67208,
+ "97": 10.61439,
+ "98": 10.68466,
+ "99": 10.52017,
+ "100": 10.61535
+ }
+ },
+ "num-zeros": {
+ "start_step": 1,
+ "end_step": 100,
+ "step_interval": 1,
+ "values": {
+ "1": 1248.0,
+ "2": 1334.0,
+ "3": 1279.0,
+ "4": 1251.0,
+ "5": 1382.0,
+ "6": 1478.0,
+ "7": 1205.0,
+ "8": 1414.0,
+ "9": 1410.0,
+ "10": 1302.0,
+ "11": 1305.0,
+ "12": 1282.0,
+ "13": 1254.0,
+ "14": 1432.0,
+ "15": 1176.0,
+ "16": 1222.0,
+ "17": 1256.0,
+ "18": 1355.0,
+ "19": 1265.0,
+ "20": 1285.0,
+ "21": 1258.0,
+ "22": 1186.0,
+ "23": 1312.0,
+ "24": 1329.0,
+ "25": 1270.0,
+ "26": 1206.0,
+ "27": 1432.0,
+ "28": 1433.0,
+ "29": 1297.0,
+ "30": 1191.0,
+ "31": 1166.0,
+ "32": 1273.0,
+ "33": 1273.0,
+ "34": 1165.0,
+ "35": 1200.0,
+ "36": 1216.0,
+ "37": 1241.0,
+ "38": 1343.0,
+ "39": 1544.0,
+ "40": 1200.0,
+ "41": 1350.0,
+ "42": 1218.0,
+ "43": 1213.0,
+ "44": 1223.0,
+ "45": 1179.0,
+ "46": 1211.0,
+ "47": 1353.0,
+ "48": 1180.0,
+ "49": 1180.0,
+ "50": 1182.0,
+ "51": 1221.0,
+ "52": 1192.0,
+ "53": 1460.0,
+ "54": 1267.0,
+ "55": 1209.0,
+ "56": 1312.0,
+ "57": 1287.0,
+ "58": 1291.0,
+ "59": 1292.0,
+ "60": 1229.0,
+ "61": 1153.0,
+ "62": 1228.0,
+ "63": 1200.0,
+ "64": 1307.0,
+ "65": 1183.0,
+ "66": 1202.0,
+ "67": 1163.0,
+ "68": 1246.0,
+ "69": 1316.0,
+ "70": 1336.0,
+ "71": 1209.0,
+ "72": 1196.0,
+ "73": 1115.0,
+ "74": 1121.0,
+ "75": 1276.0,
+ "76": 1299.0,
+ "77": 1349.0,
+ "78": 1322.0,
+ "79": 1092.0,
+ "80": 1223.0,
+ "81": 1098.0,
+ "82": 1237.0,
+ "83": 1317.0,
+ "84": 1179.0,
+ "85": 1286.0,
+ "86": 1152.0,
+ "87": 1188.0,
+ "88": 1294.0,
+ "89": 1227.0,
+ "90": 1392.0,
+ "91": 1150.0,
+ "92": 1268.0,
+ "93": 1105.0,
+ "94": 1010.0,
+ "95": 1265.0,
+ "96": 1276.0,
+ "97": 1181.0,
+ "98": 1194.0,
+ "99": 1221.0,
+ "100": 1285.0
+ }
+ },
+ "mem-allocated-bytes": {
+ "start_step": 1,
+ "end_step": 100,
+ "step_interval": 1,
+ "values": {
+ "1": 1095884800.0,
+ "2": 1095854592.0,
+ "3": 1095902720.0,
+ "4": 1095882752.0,
+ "5": 1095905792.0,
+ "6": 1095847424.0,
+ "7": 1095868928.0,
+ "8": 1095876608.0,
+ "9": 1095891968.0,
+ "10": 1095904768.0,
+ "11": 1095880704.0,
+ "12": 1095849472.0,
+ "13": 1095902720.0,
+ "14": 1095814144.0,
+ "15": 1095857152.0,
+ "16": 1095790592.0,
+ "17": 1095910400.0,
+ "18": 1095884288.0,
+ "19": 1095892480.0,
+ "20": 1095812096.0,
+ "21": 1095834112.0,
+ "22": 1095906816.0,
+ "23": 1095881728.0,
+ "24": 1095824896.0,
+ "25": 1095955968.0,
+ "26": 1095862272.0,
+ "27": 1095919616.0,
+ "28": 1095846400.0,
+ "29": 1095878656.0,
+ "30": 1095843840.0,
+ "31": 1095906816.0,
+ "32": 1095840768.0,
+ "33": 1095851008.0,
+ "34": 1095815680.0,
+ "35": 1095865856.0,
+ "36": 1095880192.0,
+ "37": 1095838720.0,
+ "38": 1095921664.0,
+ "39": 1095930368.0,
+ "40": 1095818240.0,
+ "41": 1095847424.0,
+ "42": 1095864320.0,
+ "43": 1095878144.0,
+ "44": 1095982080.0,
+ "45": 1095855104.0,
+ "46": 1095869952.0,
+ "47": 1095891968.0,
+ "48": 1095878144.0,
+ "49": 1095843840.0,
+ "50": 1095862272.0,
+ "51": 1095934464.0,
+ "52": 1095880192.0,
+ "53": 1095940608.0,
+ "54": 1095887872.0,
+ "55": 1095877632.0,
+ "56": 1095868416.0,
+ "57": 1095913472.0,
+ "58": 1095910912.0,
+ "59": 1095898112.0,
+ "60": 1095865344.0,
+ "61": 1095864320.0,
+ "62": 1095858176.0,
+ "63": 1095872000.0,
+ "64": 1095862272.0,
+ "65": 1095868928.0,
+ "66": 1095877120.0,
+ "67": 1095863808.0,
+ "68": 1095873024.0,
+ "69": 1095859712.0,
+ "70": 1095904768.0,
+ "71": 1095876608.0,
+ "72": 1095805952.0,
+ "73": 1095908352.0,
+ "74": 1095918592.0,
+ "75": 1095884288.0,
+ "76": 1095903744.0,
+ "77": 1095857664.0,
+ "78": 1095914496.0,
+ "79": 1095888896.0,
+ "80": 1095839232.0,
+ "81": 1095875584.0,
+ "82": 1095825408.0,
+ "83": 1095925248.0,
+ "84": 1095904256.0,
+ "85": 1095870976.0,
+ "86": 1095870976.0,
+ "87": 1095893504.0,
+ "88": 1095882240.0,
+ "89": 1095869952.0,
+ "90": 1095860224.0,
+ "91": 1095916032.0,
+ "92": 1095900672.0,
+ "93": 1095878144.0,
+ "94": 1095884800.0,
+ "95": 1095889920.0,
+ "96": 1095886848.0,
+ "97": 1095896576.0,
+ "98": 1095873024.0,
+ "99": 1095887872.0,
+ "100": 1095927808.0
+ }
+ },
+ "mem-max-allocated-bytes": {
+ "start_step": 1,
+ "end_step": 100,
+ "step_interval": 1,
+ "values": {
+ "1": 3260419584.0,
+ "2": 3582873600.0,
+ "3": 3615975424.0,
+ "4": 3615975424.0,
+ "5": 3616532480.0,
+ "6": 3616532480.0,
+ "7": 3616532480.0,
+ "8": 3616532480.0,
+ "9": 3616532480.0,
+ "10": 3618800640.0,
+ "11": 3618800640.0,
+ "12": 3618800640.0,
+ "13": 3618800640.0,
+ "14": 3618800640.0,
+ "15": 3618800640.0,
+ "16": 3618800640.0,
+ "17": 3618800640.0,
+ "18": 3618800640.0,
+ "19": 3618800640.0,
+ "20": 3618800640.0,
+ "21": 3618800640.0,
+ "22": 3618800640.0,
+ "23": 3618800640.0,
+ "24": 3618800640.0,
+ "25": 3648277504.0,
+ "26": 3648277504.0,
+ "27": 3648277504.0,
+ "28": 3648277504.0,
+ "29": 3648277504.0,
+ "30": 3648277504.0,
+ "31": 3648277504.0,
+ "32": 3648277504.0,
+ "33": 3648277504.0,
+ "34": 3648277504.0,
+ "35": 3648277504.0,
+ "36": 3648277504.0,
+ "37": 3648277504.0,
+ "38": 3648277504.0,
+ "39": 3648277504.0,
+ "40": 3648277504.0,
+ "41": 3648277504.0,
+ "42": 3648277504.0,
+ "43": 3648277504.0,
+ "44": 3667798528.0,
+ "45": 3667798528.0,
+ "46": 3667798528.0,
+ "47": 3667798528.0,
+ "48": 3667798528.0,
+ "49": 3667798528.0,
+ "50": 3667798528.0,
+ "51": 3667798528.0,
+ "52": 3667798528.0,
+ "53": 3667798528.0,
+ "54": 3667798528.0,
+ "55": 3667798528.0,
+ "56": 3667798528.0,
+ "57": 3667798528.0,
+ "58": 3667798528.0,
+ "59": 3667798528.0,
+ "60": 3667798528.0,
+ "61": 3667798528.0,
+ "62": 3667798528.0,
+ "63": 3667798528.0,
+ "64": 3667798528.0,
+ "65": 3667798528.0,
+ "66": 3667798528.0,
+ "67": 3667798528.0,
+ "68": 3667798528.0,
+ "69": 3667798528.0,
+ "70": 3667798528.0,
+ "71": 3667798528.0,
+ "72": 3667798528.0,
+ "73": 3667798528.0,
+ "74": 3667798528.0,
+ "75": 3667798528.0,
+ "76": 3667798528.0,
+ "77": 3667798528.0,
+ "78": 3667798528.0,
+ "79": 3667798528.0,
+ "80": 3667798528.0,
+ "81": 3667798528.0,
+ "82": 3667798528.0,
+ "83": 3667798528.0,
+ "84": 3667798528.0,
+ "85": 3667798528.0,
+ "86": 3667798528.0,
+ "87": 3667798528.0,
+ "88": 3667798528.0,
+ "89": 3667798528.0,
+ "90": 3667798528.0,
+ "91": 3667798528.0,
+ "92": 3667798528.0,
+ "93": 3667798528.0,
+ "94": 3667798528.0,
+ "95": 3667798528.0,
+ "96": 3667798528.0,
+ "97": 3667798528.0,
+ "98": 3667798528.0,
+ "99": 3667798528.0,
+ "100": 3667798528.0
+ }
+ },
+ "iteration-time": {
+ "start_step": 1,
+ "end_step": 100,
+ "step_interval": 1,
+ "values": {
+ "1": 14.85862,
+ "2": 0.28762,
+ "3": 0.23592,
+ "4": 0.20463,
+ "5": 0.21635,
+ "6": 0.20801,
+ "7": 0.20692,
+ "8": 0.20277,
+ "9": 0.20138,
+ "10": 0.19098,
+ "11": 0.19711,
+ "12": 0.19844,
+ "13": 0.18786,
+ "14": 0.19577,
+ "15": 0.18886,
+ "16": 0.18411,
+ "17": 0.18416,
+ "18": 0.18182,
+ "19": 0.17759,
+ "20": 0.18827,
+ "21": 0.18366,
+ "22": 0.18163,
+ "23": 0.18941,
+ "24": 0.18055,
+ "25": 0.18951,
+ "26": 0.18201,
+ "27": 0.17466,
+ "28": 0.18234,
+ "29": 0.1853,
+ "30": 0.17307,
+ "31": 0.18014,
+ "32": 0.17813,
+ "33": 0.18392,
+ "34": 0.1759,
+ "35": 0.18165,
+ "36": 0.17738,
+ "37": 0.18009,
+ "38": 0.17899,
+ "39": 0.18864,
+ "40": 0.17767,
+ "41": 0.17797,
+ "42": 0.18018,
+ "43": 0.18155,
+ "44": 0.17807,
+ "45": 0.17732,
+ "46": 0.17196,
+ "47": 0.1803,
+ "48": 0.17785,
+ "49": 0.17302,
+ "50": 0.1824,
+ "51": 0.19257,
+ "52": 0.17832,
+ "53": 0.18137,
+ "54": 0.17448,
+ "55": 0.178,
+ "56": 0.17346,
+ "57": 0.17236,
+ "58": 0.17018,
+ "59": 0.16863,
+ "60": 0.17468,
+ "61": 0.1713,
+ "62": 0.1744,
+ "63": 0.17553,
+ "64": 0.57804,
+ "65": 0.17627,
+ "66": 0.17362,
+ "67": 0.17436,
+ "68": 0.17766,
+ "69": 0.18446,
+ "70": 0.18419,
+ "71": 0.17131,
+ "72": 0.16832,
+ "73": 0.17321,
+ "74": 0.17561,
+ "75": 0.17215,
+ "76": 0.17083,
+ "77": 0.1791,
+ "78": 0.16898,
+ "79": 0.17382,
+ "80": 0.17586,
+ "81": 0.18035,
+ "82": 0.17931,
+ "83": 0.17665,
+ "84": 0.17692,
+ "85": 0.1765,
+ "86": 0.17412,
+ "87": 0.59045,
+ "88": 0.17964,
+ "89": 0.17565,
+ "90": 0.18664,
+ "91": 0.1784,
+ "92": 0.17175,
+ "93": 0.17523,
+ "94": 0.17223,
+ "95": 0.17436,
+ "96": 0.18556,
+ "97": 0.17929,
+ "98": 0.1847,
+ "99": 0.17691,
+ "100": 0.57857
+ }
+ }
+}
\ No newline at end of file
diff --git a/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_ep8_resume_torch_dist_muon/model_config.yaml b/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_ep8_resume_torch_dist_muon/model_config.yaml
new file mode 100644
index 00000000000..d3e3baa9f14
--- /dev/null
+++ b/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_ep8_resume_torch_dist_muon/model_config.yaml
@@ -0,0 +1,67 @@
+ENV_VARS:
+ CUDA_DEVICE_MAX_CONNECTIONS: 1
+ NVTE_ALLOW_NONDETERMINISTIC_ALGO: 0
+ NCCL_ALGO: Ring
+ CUBLAS_WORKSPACE_CONFIG: :4096:8
+MODEL_ARGS:
+ --num-layers: 12
+ --hidden-size: 512
+ --num-attention-heads: 8
+ --log-params-norm: true
+ --log-num-zeros-in-grad: true
+ --log-validation-ppl-to-tensorboard: true
+ --log-timers-to-tensorboard: true
+ --tensorboard-dir: ${TENSORBOARD_PATH}
+ --micro-batch-size: 4
+ --global-batch-size: 32
+ --seq-length: 1024
+ --max-position-embeddings: 1024
+ --disable-bias-linear: true
+ --train-iters: 100
+ --timing-log-level: 0
+ --lr-decay-iters: 320000
+ --save: ${CHECKPOINT_SAVE_PATH}
+ --load: ${CHECKPOINT_LOAD_PATH}
+ --data-path: ${DATA_PATH}/text/the_pile/shard00/my-gpt3_00_text_document
+ --vocab-file: ${DATA_PATH}/text/the_pile/shard00/bpe/vocab.json
+ --merge-file: ${DATA_PATH}/text/the_pile/shard00/bpe/merges.txt
+ --split: 949,50,1
+ --distributed-backend: nccl
+ --lr: 0.00015
+ --lr-decay-style: cosine
+ --min-lr: 1.0e-5
+ --weight-decay: 1e-2
+ --clip-grad: 1.0
+ --lr-warmup-fraction: .01
+ --log-interval: 1
+ --save-interval: 50
+ --eval-interval: 1000
+ --eval-iters: 10
+ --transformer-impl: transformer_engine
+ --tensor-model-parallel-size: 1
+ --pipeline-model-parallel-size: 1
+ --expert-model-parallel-size: 8
+ --num-experts: 8
+ --moe-token-dispatcher-type: allgather
+ --moe-router-load-balancing-type: aux_loss
+ --moe-router-topk: 2
+ --moe-router-dtype: fp32
+ --moe-ffn-hidden-size: 1024
+ --moe-grouped-gemm: true
+ --ckpt-fully-parallel-load: true
+ --deterministic-mode: true
+ --no-gradient-accumulation-fusion: true
+ --attention-softmax-in-fp32: true
+ --use-checkpoint-opt_param-scheduler: true
+ --use-mcore-models: true
+ --ckpt-format: torch_dist
+ --ckpt-assume-constant-structure: true
+ --data-cache-path: ${DATA_CACHE_PATH}
+ --bf16: true
+ --no-bias-gelu-fusion: true
+ --log-memory-to-tensorboard: true
+ --optimizer: muon
+ --muon-momentum: 0.9
+ --muon-extra-scale-factor: 0.2
+ --muon-scale-mode: spectral
+TEST_TYPE: regular
diff --git a/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_tp2_pp2_ep4_etp1_fine_grained_offloading/golden_values_dev_dgx_h100.json b/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_tp2_pp2_ep4_etp1_fine_grained_offloading/golden_values_dev_dgx_h100.json
new file mode 100644
index 00000000000..038ed2be724
--- /dev/null
+++ b/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_tp2_pp2_ep4_etp1_fine_grained_offloading/golden_values_dev_dgx_h100.json
@@ -0,0 +1,344 @@
+{
+ "lm loss": {
+ "start_step": 1,
+ "end_step": 50,
+ "step_interval": 1,
+ "values": {
+ "1": 11.06693,
+ "2": 11.0602,
+ "3": 10.21173,
+ "4": 9.95255,
+ "5": 10.12502,
+ "6": 8.8231,
+ "7": 9.52825,
+ "8": 8.44297,
+ "9": 7.84977,
+ "10": 7.0728,
+ "11": 9.30154,
+ "12": 9.14531,
+ "13": 7.86583,
+ "14": 8.21069,
+ "15": 8.2169,
+ "16": 8.17413,
+ "17": 8.21514,
+ "18": 7.49348,
+ "19": 8.08414,
+ "20": 7.63479,
+ "21": 7.95116,
+ "22": 7.29475,
+ "23": 7.9358,
+ "24": 7.43073,
+ "25": 8.23819,
+ "26": 7.75508,
+ "27": 7.6991,
+ "28": 7.65492,
+ "29": 7.75272,
+ "30": 7.56401,
+ "31": 7.81794,
+ "32": 6.46781,
+ "33": 7.20433,
+ "34": 7.77611,
+ "35": 7.72648,
+ "36": 6.71848,
+ "37": 8.09106,
+ "38": 7.61823,
+ "39": 7.96665,
+ "40": 7.49555,
+ "41": 7.49366,
+ "42": 6.10456,
+ "43": 7.59158,
+ "44": 7.91315,
+ "45": 6.83253,
+ "46": 7.4064,
+ "47": 7.78787,
+ "48": 7.87227,
+ "49": 7.58424,
+ "50": 6.83739
+ }
+ },
+ "num-zeros": {
+ "start_step": 1,
+ "end_step": 50,
+ "step_interval": 1,
+ "values": {
+ "1": 47165248.0,
+ "2": 46897896.0,
+ "3": 52684328.0,
+ "4": 297102368.0,
+ "5": 569266880.0,
+ "6": 661848704.0,
+ "7": 1027448384.0,
+ "8": 752263424.0,
+ "9": 852974912.0,
+ "10": 683720576.0,
+ "11": 833170624.0,
+ "12": 814312640.0,
+ "13": 639456320.0,
+ "14": 628553664.0,
+ "15": 706814592.0,
+ "16": 848848256.0,
+ "17": 676948992.0,
+ "18": 676681088.0,
+ "19": 892688576.0,
+ "20": 890700864.0,
+ "21": 676293696.0,
+ "22": 701562304.0,
+ "23": 796268224.0,
+ "24": 786414720.0,
+ "25": 667072192.0,
+ "26": 767487552.0,
+ "27": 773408512.0,
+ "28": 758333696.0,
+ "29": 770627840.0,
+ "30": 758410304.0,
+ "31": 644127616.0,
+ "32": 806561088.0,
+ "33": 811820352.0,
+ "34": 780254848.0,
+ "35": 757223808.0,
+ "36": 758778496.0,
+ "37": 753072832.0,
+ "38": 752875328.0,
+ "39": 767575744.0,
+ "40": 760803392.0,
+ "41": 742253440.0,
+ "42": 718278848.0,
+ "43": 676047424.0,
+ "44": 673998592.0,
+ "45": 635196864.0,
+ "46": 629090048.0,
+ "47": 623565376.0,
+ "48": 600849984.0,
+ "49": 578357504.0,
+ "50": 585291904.0
+ }
+ },
+ "mem-allocated-bytes": {
+ "start_step": 1,
+ "end_step": 50,
+ "step_interval": 1,
+ "values": {
+ "1": 5283616256.0,
+ "2": 5288015360.0,
+ "3": 5288218112.0,
+ "4": 5288420864.0,
+ "5": 5288623616.0,
+ "6": 5287812608.0,
+ "7": 5288015360.0,
+ "8": 5288218112.0,
+ "9": 5287711232.0,
+ "10": 5287913984.0,
+ "11": 5288116736.0,
+ "12": 5288319488.0,
+ "13": 5288522240.0,
+ "14": 5288724992.0,
+ "15": 5288927744.0,
+ "16": 5289130496.0,
+ "17": 5289333248.0,
+ "18": 5289536000.0,
+ "19": 5289738752.0,
+ "20": 5289941504.0,
+ "21": 5290144256.0,
+ "22": 5290347008.0,
+ "23": 5290549760.0,
+ "24": 5290752512.0,
+ "25": 5290955264.0,
+ "26": 5291158016.0,
+ "27": 5291360768.0,
+ "28": 5291563520.0,
+ "29": 5291766272.0,
+ "30": 5291969024.0,
+ "31": 5292171776.0,
+ "32": 5292374528.0,
+ "33": 5292577280.0,
+ "34": 5292780032.0,
+ "35": 5292982784.0,
+ "36": 5293185536.0,
+ "37": 5293388288.0,
+ "38": 5293591040.0,
+ "39": 5293793792.0,
+ "40": 5293996544.0,
+ "41": 5294199296.0,
+ "42": 5294402048.0,
+ "43": 5294604800.0,
+ "44": 5294807552.0,
+ "45": 5295010304.0,
+ "46": 5295213056.0,
+ "47": 5295415808.0,
+ "48": 5295618560.0,
+ "49": 5295821312.0,
+ "50": 5296024064.0
+ }
+ },
+ "mem-max-allocated-bytes": {
+ "start_step": 1,
+ "end_step": 50,
+ "step_interval": 1,
+ "values": {
+ "1": 6208857600.0,
+ "2": 8233667072.0,
+ "3": 8233667072.0,
+ "4": 8233667072.0,
+ "5": 8233667072.0,
+ "6": 8233667072.0,
+ "7": 8233667072.0,
+ "8": 8233667072.0,
+ "9": 8233667072.0,
+ "10": 8233667072.0,
+ "11": 8262763008.0,
+ "12": 8262763008.0,
+ "13": 8262763008.0,
+ "14": 8262763008.0,
+ "15": 8262763008.0,
+ "16": 8273029632.0,
+ "17": 8282915328.0,
+ "18": 8282915328.0,
+ "19": 8284467712.0,
+ "20": 8294910464.0,
+ "21": 8294910464.0,
+ "22": 8303365632.0,
+ "23": 8303365632.0,
+ "24": 8303365632.0,
+ "25": 8303365632.0,
+ "26": 8303365632.0,
+ "27": 8303365632.0,
+ "28": 8303365632.0,
+ "29": 8303365632.0,
+ "30": 8328921600.0,
+ "31": 8328921600.0,
+ "32": 8328921600.0,
+ "33": 8328921600.0,
+ "34": 8342317568.0,
+ "35": 8352083456.0,
+ "36": 8352083456.0,
+ "37": 8352083456.0,
+ "38": 8352083456.0,
+ "39": 8352083456.0,
+ "40": 8352083456.0,
+ "41": 8352083456.0,
+ "42": 8352083456.0,
+ "43": 8352083456.0,
+ "44": 8352083456.0,
+ "45": 8352083456.0,
+ "46": 8352083456.0,
+ "47": 8352083456.0,
+ "48": 8352083456.0,
+ "49": 8352083456.0,
+ "50": 8352083456.0
+ }
+ },
+ "mtp_1 loss": {
+ "start_step": 1,
+ "end_step": 50,
+ "step_interval": 1,
+ "values": {
+ "1": 11.07401,
+ "2": 11.0927,
+ "3": 10.82644,
+ "4": 10.27575,
+ "5": 10.45332,
+ "6": 8.3277,
+ "7": 9.8265,
+ "8": 8.01558,
+ "9": 7.47586,
+ "10": 6.7581,
+ "11": 8.9297,
+ "12": 8.98829,
+ "13": 7.80214,
+ "14": 8.02436,
+ "15": 8.11251,
+ "16": 8.14258,
+ "17": 8.13031,
+ "18": 7.44579,
+ "19": 8.03606,
+ "20": 7.54064,
+ "21": 7.90046,
+ "22": 7.27709,
+ "23": 7.88548,
+ "24": 7.37576,
+ "25": 8.17071,
+ "26": 7.69849,
+ "27": 7.62829,
+ "28": 7.61349,
+ "29": 7.69754,
+ "30": 7.47936,
+ "31": 7.73926,
+ "32": 6.37137,
+ "33": 7.1379,
+ "34": 7.71901,
+ "35": 7.63544,
+ "36": 6.61321,
+ "37": 8.03174,
+ "38": 7.58067,
+ "39": 7.89473,
+ "40": 7.41418,
+ "41": 7.42196,
+ "42": 6.01401,
+ "43": 7.49099,
+ "44": 7.86625,
+ "45": 6.74951,
+ "46": 7.30637,
+ "47": 7.72653,
+ "48": 7.78872,
+ "49": 7.48917,
+ "50": 6.75533
+ }
+ },
+ "iteration-time": {
+ "start_step": 1,
+ "end_step": 50,
+ "step_interval": 1,
+ "values": {
+ "1": 88.9425,
+ "2": 2.91855,
+ "3": 2.58352,
+ "4": 3.73409,
+ "5": 2.63585,
+ "6": 2.48926,
+ "7": 2.27523,
+ "8": 2.50563,
+ "9": 2.45577,
+ "10": 1.90482,
+ "11": 1.96806,
+ "12": 2.42331,
+ "13": 1.88872,
+ "14": 1.89773,
+ "15": 1.90418,
+ "16": 1.885,
+ "17": 1.91181,
+ "18": 1.89194,
+ "19": 1.97889,
+ "20": 1.88063,
+ "21": 1.88612,
+ "22": 1.90981,
+ "23": 1.87053,
+ "24": 1.87293,
+ "25": 1.89611,
+ "26": 1.96035,
+ "27": 1.9067,
+ "28": 1.91982,
+ "29": 1.94441,
+ "30": 1.88208,
+ "31": 1.9521,
+ "32": 1.89063,
+ "33": 1.9571,
+ "34": 1.93481,
+ "35": 1.87558,
+ "36": 1.88538,
+ "37": 1.89041,
+ "38": 1.97023,
+ "39": 1.89001,
+ "40": 1.87859,
+ "41": 1.89949,
+ "42": 1.88775,
+ "43": 1.94805,
+ "44": 1.90575,
+ "45": 1.89185,
+ "46": 1.87259,
+ "47": 1.89396,
+ "48": 1.8747,
+ "49": 1.88874,
+ "50": 1.91915
+ }
+ }
+}
diff --git a/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_tp2_pp2_ep4_etp1_fine_grained_offloading/model_config.yaml b/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_tp2_pp2_ep4_etp1_fine_grained_offloading/model_config.yaml
new file mode 100644
index 00000000000..38528836659
--- /dev/null
+++ b/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_tp2_pp2_ep4_etp1_fine_grained_offloading/model_config.yaml
@@ -0,0 +1,140 @@
+ENV_VARS:
+ CUDA_DEVICE_MAX_CONNECTIONS: 32
+ NVTE_ALLOW_NONDETERMINISTIC_ALGO: 0
+ PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True
+ NCCL_NVLS_ENABLE: 0
+ PYTHONWARNINGS: ignore
+ NCCL_DEBUG: VERSION
+ NVTE_CPU_OFFLOAD_V1: 1
+ NVTE_FUSED_ATTN: 0
+ NCCL_ALGO: ^NVLS
+ CUBLAS_WORKSPACE_CONFIG: ':4096:8'
+MODEL_ARGS:
+ # Distributed args
+ --distributed-timeout-minutes: 60
+ --tensor-model-parallel-size: 2
+ --pipeline-model-parallel-size: 2
+ --expert-model-parallel-size: 4
+ --context-parallel-size: 1
+ --expert-tensor-parallel-size: 1
+ --use-distributed-optimizer: true
+ # NOTE: uncomment if TE >= 2.9.0
+ # --overlap-grad-reduce: true
+ # --overlap-param-gather: true
+ # Use unfused attention since MLA with fused attention and deterministic mode leads to NaN
+ --attention-backend: unfused # TODO: switch back to fused attention after fix
+ # Training args
+ --use-mcore-models: true
+ --sequence-parallel: true
+ --disable-bias-linear: true
+ --micro-batch-size: 1
+ --global-batch-size: 8
+ --train-iters: 50
+ --exit-duration-in-mins: 230
+ --no-check-for-nan-in-loss-and-grad: true
+ --no-rope-fusion: true
+ --manual-gc: true
+ --manual-gc-interval: 100
+ --recompute-granularity: selective
+ --recompute-modules: "[layernorm mla_up_proj mlp moe_act]"
+ --fine-grained-activation-offloading: true
+ --offload-modules: "[expert_fc1 moe_act attn_norm mlp_norm qkv_linear core_attn attn_proj]"
+ # Transformer Engine args
+ --transformer-impl: transformer_engine
+ # Data args
+ --seq-length: 4096
+ --data-cache-path: ${DATA_CACHE_PATH}
+ --data-path: ${DATA_PATH}/text/the_pile/shard00/my-gpt3_00_text_document
+ --vocab-file: ${DATA_PATH}/text/the_pile/shard00/bpe/vocab.json
+ --merge-file: ${DATA_PATH}/text/the_pile/shard00/bpe/merges.txt
+ --split: 949,50,1
+ # Add network size args
+ --num-layers: 15
+ --moe-layer-freq: ([0]*3+[1]*12)
+ --pipeline-model-parallel-layout: Et*3\\|\\(tt\\|\\)*6mL # Et*3|(tt|)*6mL
+ --hidden-size: 1024
+ --ffn-hidden-size: 4096
+ --num-attention-heads: 32
+ --kv-channels: 128
+ --max-position-embeddings: 4096
+ --position-embedding-type: rope
+ --rotary-base: 10000
+ --make-vocab-size-divisible-by: 3232
+ --normalization: RMSNorm
+ --norm-epsilon: 1e-6
+ --swiglu: true
+ --untie-embeddings-and-output-weights: true
+ --multi-latent-attention: true
+ # Comment out the following MTP args to disable MTP
+ --mtp-num-layers: 1
+ --mtp-loss-scaling-factor: 0.1
+ # Add regularization args
+ --attention-dropout: 0.0
+ --hidden-dropout: 0.0
+ --clip-grad: 1.0
+ --weight-decay: 0.1
+ --qk-layernorm: true
+ # Add learning rate args
+ --lr-warmup-fraction: .01
+ --lr: 0.00015
+ --min-lr: 1.0e-5
+ --lr-decay-style: cosine
+ --adam-beta1: 0.9
+ --adam-beta2: 0.95
+ # Add MoE args
+ --num-experts: 32
+ --moe-ffn-hidden-size: 1024
+ --moe-shared-expert-intermediate-size: 1024
+ --moe-router-load-balancing-type: seq_aux_loss
+ --moe-router-topk: 4
+ --moe-token-dispatcher-type: alltoall
+ --moe-router-pre-softmax: true
+ --moe-grouped-gemm: true
+ --moe-aux-loss-coeff: 1e-4
+ --moe-router-group-topk: 2
+ --moe-router-num-groups: 4
+ --moe-router-topk-scaling-factor: 2.0
+ --moe-router-score-function: sigmoid
+ --moe-router-enable-expert-bias: true
+ --moe-router-bias-update-rate: 1e-3
+ --moe-router-dtype: fp32
+ --moe-permute-fusion: true
+ # Add MLA args
+ --q-lora-rank: 1536
+ --kv-lora-rank: 512
+ --qk-head-dim: 128
+ --qk-pos-emb-head-dim: 64
+ --v-head-dim: 128
+ --rotary-scaling-factor: 40
+ --mscale: 1.0
+ --mscale-all-dim: 1.0
+ # Add validation args
+ --eval-iters: 32
+ --eval-interval: 200
+ # Add checkpointing args
+ --save: ${CHECKPOINT_SAVE_PATH}
+ --load: ${CHECKPOINT_LOAD_PATH}
+ --save-interval: 25
+ # Add initialization args
+ --init-method-std: 0.02
+ # Add logging args
+ --log-timers-to-tensorboard: true
+ --log-memory-to-tensorboard: true
+ --log-num-zeros-in-grad: true
+ --log-params-norm: true
+ --log-validation-ppl-to-tensorboard: true
+ --log-throughput: true
+ --log-interval: 1
+ --logging-level: 40
+ --tensorboard-dir: ${TENSORBOARD_PATH}
+ # Add mixed precision args
+ --bf16: true
+ --exit-interval: 50
+ --overlap-moe-expert-parallel-comm: true
+TEST_TYPE: regular # Usually ckpt-resume, but as a WAR to #513 set to regular
+METRICS:
+ # - "iteration-time"
+ - "lm loss"
+ - "mem-allocated-bytes"
+ - "mem-max-allocated-bytes"
+ - "mtp_1 loss"
diff --git a/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_tp2_pp2_ep4_etp1_no_mtp_no_a2a_ovlp_fine_grained_offloading/golden_values_dev_dgx_h100.json b/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_tp2_pp2_ep4_etp1_no_mtp_no_a2a_ovlp_fine_grained_offloading/golden_values_dev_dgx_h100.json
new file mode 100644
index 00000000000..9cc2fa69da7
--- /dev/null
+++ b/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_tp2_pp2_ep4_etp1_no_mtp_no_a2a_ovlp_fine_grained_offloading/golden_values_dev_dgx_h100.json
@@ -0,0 +1,287 @@
+{
+ "lm loss": {
+ "start_step": 1,
+ "end_step": 50,
+ "step_interval": 1,
+ "values": {
+ "1": 11.01693,
+ "2": 11.06263,
+ "3": 10.17828,
+ "4": 10.86162,
+ "5": 9.8171,
+ "6": 9.10066,
+ "7": 9.61216,
+ "8": 8.39629,
+ "9": 7.79624,
+ "10": 7.15182,
+ "11": 9.06686,
+ "12": 12.41529,
+ "13": 8.05859,
+ "14": 8.25078,
+ "15": 8.25932,
+ "16": 8.33199,
+ "17": 8.33144,
+ "18": 7.58852,
+ "19": 8.19681,
+ "20": 7.68193,
+ "21": 8.00256,
+ "22": 7.37928,
+ "23": 7.95036,
+ "24": 7.52138,
+ "25": 8.32313,
+ "26": 7.80137,
+ "27": 7.73067,
+ "28": 7.70985,
+ "29": 7.77487,
+ "30": 7.57653,
+ "31": 7.85303,
+ "32": 6.5208,
+ "33": 7.2477,
+ "34": 7.80024,
+ "35": 7.74614,
+ "36": 6.73365,
+ "37": 8.154,
+ "38": 7.62714,
+ "39": 7.97924,
+ "40": 7.524,
+ "41": 7.52079,
+ "42": 6.11188,
+ "43": 7.6025,
+ "44": 7.97264,
+ "45": 6.84479,
+ "46": 7.4241,
+ "47": 7.82528,
+ "48": 7.87668,
+ "49": 7.5987,
+ "50": 6.8481
+ }
+ },
+ "num-zeros": {
+ "start_step": 1,
+ "end_step": 50,
+ "step_interval": 1,
+ "values": {
+ "1": 47167760.0,
+ "2": 46900544.0,
+ "3": 84151152.0,
+ "4": 237329488.0,
+ "5": 471710816.0,
+ "6": 558040704.0,
+ "7": 958277696.0,
+ "8": 723945792.0,
+ "9": 812038208.0,
+ "10": 721441280.0,
+ "11": 622437632.0,
+ "12": 556346176.0,
+ "13": 633166464.0,
+ "14": 700920576.0,
+ "15": 766532480.0,
+ "16": 719878656.0,
+ "17": 673785280.0,
+ "18": 733291456.0,
+ "19": 713440768.0,
+ "20": 859244608.0,
+ "21": 836730112.0,
+ "22": 789566720.0,
+ "23": 808848960.0,
+ "24": 644896128.0,
+ "25": 852631104.0,
+ "26": 836696384.0,
+ "27": 550069504.0,
+ "28": 604192832.0,
+ "29": 761193792.0,
+ "30": 758412160.0,
+ "31": 782509568.0,
+ "32": 765664256.0,
+ "33": 745758912.0,
+ "34": 569510656.0,
+ "35": 728914304.0,
+ "36": 699003840.0,
+ "37": 705883072.0,
+ "38": 705682240.0,
+ "39": 685787136.0,
+ "40": 656996352.0,
+ "41": 484325760.0,
+ "42": 633345536.0,
+ "43": 641441984.0,
+ "44": 466413888.0,
+ "45": 427604864.0,
+ "46": 566181184.0,
+ "47": 563795904.0,
+ "48": 421565312.0,
+ "49": 537463040.0,
+ "50": 494058176.0
+ }
+ },
+ "mem-allocated-bytes": {
+ "start_step": 1,
+ "end_step": 50,
+ "step_interval": 1,
+ "values": {
+ "1": 4313446912.0,
+ "2": 4313448448.0,
+ "3": 4313448448.0,
+ "4": 4313448448.0,
+ "5": 4313448448.0,
+ "6": 4313448448.0,
+ "7": 4313448448.0,
+ "8": 4313448448.0,
+ "9": 4313448448.0,
+ "10": 4313448448.0,
+ "11": 4313448448.0,
+ "12": 4313448448.0,
+ "13": 4313448448.0,
+ "14": 4313448448.0,
+ "15": 4313448448.0,
+ "16": 4313448448.0,
+ "17": 4313448448.0,
+ "18": 4313448448.0,
+ "19": 4313448448.0,
+ "20": 4313448448.0,
+ "21": 4313448448.0,
+ "22": 4313448448.0,
+ "23": 4313448448.0,
+ "24": 4313448448.0,
+ "25": 4313448448.0,
+ "26": 4313448448.0,
+ "27": 4313448448.0,
+ "28": 4313448448.0,
+ "29": 4313448448.0,
+ "30": 4313448448.0,
+ "31": 4313448448.0,
+ "32": 4313448448.0,
+ "33": 4313448448.0,
+ "34": 4313448448.0,
+ "35": 4313448448.0,
+ "36": 4313448448.0,
+ "37": 4313448448.0,
+ "38": 4313448448.0,
+ "39": 4313448448.0,
+ "40": 4313448448.0,
+ "41": 4313448448.0,
+ "42": 4313448448.0,
+ "43": 4313448448.0,
+ "44": 4313448448.0,
+ "45": 4313448448.0,
+ "46": 4313448448.0,
+ "47": 4313448448.0,
+ "48": 4313448448.0,
+ "49": 4313448448.0,
+ "50": 4313448448.0
+ }
+ },
+ "mem-max-allocated-bytes": {
+ "start_step": 1,
+ "end_step": 50,
+ "step_interval": 1,
+ "values": {
+ "1": 4305060864.0,
+ "2": 5850929152.0,
+ "3": 5850929152.0,
+ "4": 5857061888.0,
+ "5": 5857061888.0,
+ "6": 5857061888.0,
+ "7": 5857061888.0,
+ "8": 5857061888.0,
+ "9": 5857061888.0,
+ "10": 5857061888.0,
+ "11": 5857061888.0,
+ "12": 5857061888.0,
+ "13": 5857061888.0,
+ "14": 5857061888.0,
+ "15": 5857061888.0,
+ "16": 5857061888.0,
+ "17": 5857061888.0,
+ "18": 5857061888.0,
+ "19": 5857061888.0,
+ "20": 5857061888.0,
+ "21": 5857061888.0,
+ "22": 5857061888.0,
+ "23": 5857061888.0,
+ "24": 5857061888.0,
+ "25": 5857061888.0,
+ "26": 5857061888.0,
+ "27": 5857061888.0,
+ "28": 5857061888.0,
+ "29": 5857061888.0,
+ "30": 5857061888.0,
+ "31": 5857061888.0,
+ "32": 5857061888.0,
+ "33": 5857061888.0,
+ "34": 5857061888.0,
+ "35": 5857061888.0,
+ "36": 5857061888.0,
+ "37": 5857061888.0,
+ "38": 5857061888.0,
+ "39": 5860414976.0,
+ "40": 5860414976.0,
+ "41": 5860414976.0,
+ "42": 5860414976.0,
+ "43": 5860414976.0,
+ "44": 5860414976.0,
+ "45": 5860414976.0,
+ "46": 5860414976.0,
+ "47": 5860414976.0,
+ "48": 5860414976.0,
+ "49": 5860414976.0,
+ "50": 5860414976.0
+ }
+ },
+ "iteration-time": {
+ "start_step": 1,
+ "end_step": 50,
+ "step_interval": 1,
+ "values": {
+ "1": 92.74621,
+ "2": 3.05215,
+ "3": 3.87635,
+ "4": 2.96691,
+ "5": 3.09601,
+ "6": 1.94793,
+ "7": 2.58283,
+ "8": 2.00403,
+ "9": 1.96081,
+ "10": 1.955,
+ "11": 1.95251,
+ "12": 2.07845,
+ "13": 2.01952,
+ "14": 1.96206,
+ "15": 1.96234,
+ "16": 1.97406,
+ "17": 2.0423,
+ "18": 1.96841,
+ "19": 1.95796,
+ "20": 2.48713,
+ "21": 2.55338,
+ "22": 1.97633,
+ "23": 1.95723,
+ "24": 1.98425,
+ "25": 1.95827,
+ "26": 1.95919,
+ "27": 1.95629,
+ "28": 1.96685,
+ "29": 1.95089,
+ "30": 2.55672,
+ "31": 1.93918,
+ "32": 1.95892,
+ "33": 1.95987,
+ "34": 1.95394,
+ "35": 1.96053,
+ "36": 1.96074,
+ "37": 1.96542,
+ "38": 1.97304,
+ "39": 2.00073,
+ "40": 1.98223,
+ "41": 1.95986,
+ "42": 1.96976,
+ "43": 1.94793,
+ "44": 1.95897,
+ "45": 1.96904,
+ "46": 1.96519,
+ "47": 1.95996,
+ "48": 1.96564,
+ "49": 1.96485,
+ "50": 1.97038
+ }
+ }
+}
diff --git a/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_tp2_pp2_ep4_etp1_no_mtp_no_a2a_ovlp_fine_grained_offloading/model_config.yaml b/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_tp2_pp2_ep4_etp1_no_mtp_no_a2a_ovlp_fine_grained_offloading/model_config.yaml
new file mode 100644
index 00000000000..d1fcd8fd4b7
--- /dev/null
+++ b/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_tp2_pp2_ep4_etp1_no_mtp_no_a2a_ovlp_fine_grained_offloading/model_config.yaml
@@ -0,0 +1,135 @@
+ENV_VARS:
+ CUDA_DEVICE_MAX_CONNECTIONS: 1
+ NVTE_ALLOW_NONDETERMINISTIC_ALGO: 0
+ PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True
+ NCCL_NVLS_ENABLE: 0
+ PYTHONWARNINGS: ignore
+ NCCL_DEBUG: VERSION
+ NVTE_CPU_OFFLOAD_V1: 1
+ NVTE_FUSED_ATTN: 0
+ NCCL_ALGO: ^NVLS
+ CUBLAS_WORKSPACE_CONFIG: ':4096:8'
+MODEL_ARGS:
+ # Distributed args
+ --distributed-timeout-minutes: 60
+ --tensor-model-parallel-size: 2
+ --pipeline-model-parallel-size: 2
+ --expert-model-parallel-size: 4
+ --context-parallel-size: 1
+ --expert-tensor-parallel-size: 1
+ --use-distributed-optimizer: true
+ # NOTE: uncomment if TE >= 2.9.0
+ # --overlap-grad-reduce: true
+ # --overlap-param-gather: true
+ # Use unfused attention since MLA with fused attention and deterministic mode leads to NaN
+ --attention-backend: unfused # TODO: switch back to fused attention after fix
+ # Training args
+ --use-mcore-models: true
+ --sequence-parallel: true
+ --disable-bias-linear: true
+ --micro-batch-size: 1
+ --global-batch-size: 8
+ --train-iters: 50
+ --exit-duration-in-mins: 230
+ --no-check-for-nan-in-loss-and-grad: true
+ --no-rope-fusion: true
+ --manual-gc: true
+ --manual-gc-interval: 100
+ --recompute-granularity: selective
+ --recompute-modules: "[layernorm mla_up_proj mlp moe_act]"
+ --fine-grained-activation-offloading: true
+ --offload-modules: "[expert_fc1 moe_act attn_norm mlp_norm qkv_linear core_attn attn_proj]"
+ # Transformer Engine args
+ --transformer-impl: transformer_engine
+ # Data args
+ --seq-length: 4096
+ --data-cache-path: ${DATA_CACHE_PATH}
+ --data-path: ${DATA_PATH}/text/the_pile/shard00/my-gpt3_00_text_document
+ --vocab-file: ${DATA_PATH}/text/the_pile/shard00/bpe/vocab.json
+ --merge-file: ${DATA_PATH}/text/the_pile/shard00/bpe/merges.txt
+ --split: 949,50,1
+ # Add network size args
+ --num-layers: 15
+ --moe-layer-freq: ([0]*3+[1]*12)
+ --pipeline-model-parallel-layout: Et*3\\|\\(tt\\|\\)*6L # Et*3|(tt|)*6L
+ --hidden-size: 1024
+ --ffn-hidden-size: 4096
+ --num-attention-heads: 32
+ --kv-channels: 128
+ --max-position-embeddings: 4096
+ --position-embedding-type: rope
+ --rotary-base: 10000
+ --make-vocab-size-divisible-by: 3232
+ --normalization: RMSNorm
+ --norm-epsilon: 1e-6
+ --swiglu: true
+ --untie-embeddings-and-output-weights: true
+ --multi-latent-attention: true
+ # Add regularization args
+ --attention-dropout: 0.0
+ --hidden-dropout: 0.0
+ --clip-grad: 1.0
+ --weight-decay: 0.1
+ --qk-layernorm: true
+ # Add learning rate args
+ --lr-warmup-fraction: .01
+ --lr: 0.00015
+ --min-lr: 1.0e-5
+ --lr-decay-style: cosine
+ --adam-beta1: 0.9
+ --adam-beta2: 0.95
+ # Add MoE args
+ --num-experts: 32
+ --moe-ffn-hidden-size: 1024
+ --moe-shared-expert-intermediate-size: 1024
+ --moe-router-load-balancing-type: seq_aux_loss
+ --moe-router-topk: 4
+ --moe-token-dispatcher-type: alltoall
+ --moe-router-pre-softmax: true
+ --moe-grouped-gemm: true
+ --moe-aux-loss-coeff: 1e-4
+ --moe-router-group-topk: 2
+ --moe-router-num-groups: 4
+ --moe-router-topk-scaling-factor: 2.0
+ --moe-router-score-function: sigmoid
+ --moe-router-enable-expert-bias: true
+ --moe-router-bias-update-rate: 1e-3
+ --moe-router-dtype: fp32
+ --moe-permute-fusion: true
+ # Add MLA args
+ --q-lora-rank: 1536
+ --kv-lora-rank: 512
+ --qk-head-dim: 128
+ --qk-pos-emb-head-dim: 64
+ --v-head-dim: 128
+ --rotary-scaling-factor: 40
+ --mscale: 1.0
+ --mscale-all-dim: 1.0
+ # Add validation args
+ --eval-iters: 32
+ --eval-interval: 200
+ # Add checkpointing args
+ --save: ${CHECKPOINT_SAVE_PATH}
+ --load: ${CHECKPOINT_LOAD_PATH}
+ --save-interval: 25
+ # Add initialization args
+ --init-method-std: 0.02
+ # Add logging args
+ --log-timers-to-tensorboard: true
+ --log-memory-to-tensorboard: true
+ --log-num-zeros-in-grad: true
+ --log-params-norm: true
+ --log-validation-ppl-to-tensorboard: true
+ --log-throughput: true
+ --log-interval: 1
+ --logging-level: 40
+ --tensorboard-dir: ${TENSORBOARD_PATH}
+ # Add mixed precision args
+ --bf16: true
+ --exit-interval: 50
+TEST_TYPE: regular # Usually ckpt-resume, but as a WAR to #513 set to regular
+METRICS:
+ # - "iteration-time"
+ - "lm loss"
+ - "mem-allocated-bytes"
+ - "mem-max-allocated-bytes"
diff --git a/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_tp4_ep2_etp2_pp2_scoped_cudagraph/golden_values_dev_dgx_h100.json b/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_tp4_ep2_etp2_pp2_scoped_cudagraph/golden_values_dev_dgx_h100.json
new file mode 100644
index 00000000000..a77eac20664
--- /dev/null
+++ b/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_tp4_ep2_etp2_pp2_scoped_cudagraph/golden_values_dev_dgx_h100.json
@@ -0,0 +1,644 @@
+{
+ "lm loss": {
+ "start_step": 1,
+ "end_step": 100,
+ "step_interval": 1,
+ "values": {
+ "1": 10.93691,
+ "2": 10.93262,
+ "3": 10.94243,
+ "4": 10.95011,
+ "5": 10.9502,
+ "6": 10.94175,
+ "7": 10.94469,
+ "8": 10.93675,
+ "9": 10.94939,
+ "10": 10.9367,
+ "11": 10.94082,
+ "12": 10.93794,
+ "13": 10.92338,
+ "14": 10.93415,
+ "15": 10.88723,
+ "16": 10.87495,
+ "17": 10.86864,
+ "18": 10.86127,
+ "19": 10.86341,
+ "20": 10.78125,
+ "21": 10.73131,
+ "22": 10.60371,
+ "23": 10.73309,
+ "24": 10.61865,
+ "25": 10.55175,
+ "26": 10.62651,
+ "27": 10.63921,
+ "28": 10.59104,
+ "29": 10.5981,
+ "30": 10.37817,
+ "31": 10.12235,
+ "32": 10.46117,
+ "33": 10.45537,
+ "34": 10.20087,
+ "35": 10.25661,
+ "36": 10.20876,
+ "37": 10.33662,
+ "38": 10.16683,
+ "39": 10.40916,
+ "40": 10.05209,
+ "41": 10.09427,
+ "42": 10.17821,
+ "43": 9.74204,
+ "44": 9.89005,
+ "45": 9.74011,
+ "46": 9.72669,
+ "47": 10.09152,
+ "48": 9.75295,
+ "49": 9.40186,
+ "50": 9.83645,
+ "51": 9.77036,
+ "52": 9.65641,
+ "53": 10.03067,
+ "54": 9.87916,
+ "55": 9.79619,
+ "56": 9.52858,
+ "57": 9.36596,
+ "58": 9.75327,
+ "59": 9.48259,
+ "60": 9.40835,
+ "61": 9.60202,
+ "62": 9.90742,
+ "63": 9.25777,
+ "64": 9.68411,
+ "65": 8.79911,
+ "66": 9.60796,
+ "67": 9.25427,
+ "68": 9.71419,
+ "69": 9.71666,
+ "70": 9.6613,
+ "71": 9.52439,
+ "72": 9.4709,
+ "73": 9.38862,
+ "74": 8.80286,
+ "75": 9.34004,
+ "76": 8.93543,
+ "77": 9.99337,
+ "78": 9.64723,
+ "79": 9.28126,
+ "80": 9.29633,
+ "81": 9.39609,
+ "82": 9.60877,
+ "83": 9.21694,
+ "84": 9.34008,
+ "85": 9.53009,
+ "86": 8.95652,
+ "87": 9.51691,
+ "88": 9.68221,
+ "89": 9.50553,
+ "90": 9.753,
+ "91": 9.2347,
+ "92": 9.26019,
+ "93": 8.94568,
+ "94": 8.69194,
+ "95": 9.44616,
+ "96": 9.41008,
+ "97": 9.20125,
+ "98": 9.58169,
+ "99": 8.75946,
+ "100": 9.29483
+ }
+ },
+ "num-zeros": {
+ "start_step": 1,
+ "end_step": 100,
+ "step_interval": 1,
+ "values": {
+ "1": 22750372.0,
+ "2": 22953180.0,
+ "3": 22604424.0,
+ "4": 23266362.0,
+ "5": 22735560.0,
+ "6": 23061884.0,
+ "7": 22793368.0,
+ "8": 22960792.0,
+ "9": 22865612.0,
+ "10": 22950328.0,
+ "11": 22499656.0,
+ "12": 22456052.0,
+ "13": 22948014.0,
+ "14": 22384498.0,
+ "15": 22846334.0,
+ "16": 22856854.0,
+ "17": 22836340.0,
+ "18": 22590220.0,
+ "19": 22627128.0,
+ "20": 22712376.0,
+ "21": 22762744.0,
+ "22": 22816900.0,
+ "23": 22545168.0,
+ "24": 22794340.0,
+ "25": 22841898.0,
+ "26": 22549680.0,
+ "27": 22464852.0,
+ "28": 22453780.0,
+ "29": 22534588.0,
+ "30": 22636160.0,
+ "31": 22989382.0,
+ "32": 22594002.0,
+ "33": 22566000.0,
+ "34": 22855476.0,
+ "35": 22813640.0,
+ "36": 22595484.0,
+ "37": 22499348.0,
+ "38": 22926172.0,
+ "39": 22825344.0,
+ "40": 22675752.0,
+ "41": 22671542.0,
+ "42": 22682408.0,
+ "43": 23014140.0,
+ "44": 22768504.0,
+ "45": 22679044.0,
+ "46": 22912572.0,
+ "47": 23691904.0,
+ "48": 24003148.0,
+ "49": 23786764.0,
+ "50": 22931654.0,
+ "51": 23866164.0,
+ "52": 23807242.0,
+ "53": 24007504.0,
+ "54": 22867916.0,
+ "55": 23571280.0,
+ "56": 23954212.0,
+ "57": 24211680.0,
+ "58": 23914512.0,
+ "59": 22722820.0,
+ "60": 23813508.0,
+ "61": 23796364.0,
+ "62": 23739896.0,
+ "63": 24965914.0,
+ "64": 23898698.0,
+ "65": 24150860.0,
+ "66": 23796512.0,
+ "67": 25032960.0,
+ "68": 23673048.0,
+ "69": 23644684.0,
+ "70": 23903614.0,
+ "71": 24864656.0,
+ "72": 24766928.0,
+ "73": 24850636.0,
+ "74": 24133166.0,
+ "75": 24143912.0,
+ "76": 25025406.0,
+ "77": 24358344.0,
+ "78": 24910132.0,
+ "79": 23808164.0,
+ "80": 23772256.0,
+ "81": 25020440.0,
+ "82": 23851242.0,
+ "83": 23911824.0,
+ "84": 25143864.0,
+ "85": 24823592.0,
+ "86": 23153228.0,
+ "87": 24850332.0,
+ "88": 24749368.0,
+ "89": 22505174.0,
+ "90": 25108752.0,
+ "91": 23838548.0,
+ "92": 24923816.0,
+ "93": 24769484.0,
+ "94": 25041572.0,
+ "95": 25189350.0,
+ "96": 23909318.0,
+ "97": 23664104.0,
+ "98": 23832392.0,
+ "99": 23981812.0,
+ "100": 24101144.0
+ }
+ },
+ "mem-allocated-bytes": {
+ "start_step": 1,
+ "end_step": 100,
+ "step_interval": 1,
+ "values": {
+ "1": 773784064.0,
+ "2": 776621056.0,
+ "3": 764709888.0,
+ "4": 937392128.0,
+ "5": 935098368.0,
+ "6": 935098368.0,
+ "7": 935639040.0,
+ "8": 937392128.0,
+ "9": 935098368.0,
+ "10": 936785920.0,
+ "11": 937392128.0,
+ "12": 935098368.0,
+ "13": 935098368.0,
+ "14": 935639040.0,
+ "15": 937392128.0,
+ "16": 935098368.0,
+ "17": 935639040.0,
+ "18": 937392128.0,
+ "19": 937392128.0,
+ "20": 935098368.0,
+ "21": 936785920.0,
+ "22": 937392128.0,
+ "23": 936785920.0,
+ "24": 937392128.0,
+ "25": 935098368.0,
+ "26": 935098368.0,
+ "27": 936245248.0,
+ "28": 937392128.0,
+ "29": 937392128.0,
+ "30": 935098368.0,
+ "31": 935098368.0,
+ "32": 935639040.0,
+ "33": 936785920.0,
+ "34": 937392128.0,
+ "35": 937392128.0,
+ "36": 937392128.0,
+ "37": 935098368.0,
+ "38": 935098368.0,
+ "39": 935098368.0,
+ "40": 936785920.0,
+ "41": 937392128.0,
+ "42": 937392128.0,
+ "43": 937392128.0,
+ "44": 937392128.0,
+ "45": 937392128.0,
+ "46": 937392128.0,
+ "47": 935098368.0,
+ "48": 935098368.0,
+ "49": 937392128.0,
+ "50": 937392128.0,
+ "51": 935098368.0,
+ "52": 935639040.0,
+ "53": 936785920.0,
+ "54": 937392128.0,
+ "55": 937392128.0,
+ "56": 935098368.0,
+ "57": 935098368.0,
+ "58": 935098368.0,
+ "59": 935639040.0,
+ "60": 936245248.0,
+ "61": 936785920.0,
+ "62": 936785920.0,
+ "63": 937392128.0,
+ "64": 937392128.0,
+ "65": 937392128.0,
+ "66": 935098368.0,
+ "67": 935098368.0,
+ "68": 935639040.0,
+ "69": 936245248.0,
+ "70": 936785920.0,
+ "71": 937392128.0,
+ "72": 937392128.0,
+ "73": 937392128.0,
+ "74": 937392128.0,
+ "75": 935098368.0,
+ "76": 937392128.0,
+ "77": 937392128.0,
+ "78": 935098368.0,
+ "79": 935639040.0,
+ "80": 937392128.0,
+ "81": 937392128.0,
+ "82": 935098368.0,
+ "83": 936785920.0,
+ "84": 937392128.0,
+ "85": 937392128.0,
+ "86": 935098368.0,
+ "87": 936785920.0,
+ "88": 937392128.0,
+ "89": 935098368.0,
+ "90": 935639040.0,
+ "91": 937392128.0,
+ "92": 937392128.0,
+ "93": 937392128.0,
+ "94": 935098368.0,
+ "95": 935098368.0,
+ "96": 935639040.0,
+ "97": 936245248.0,
+ "98": 937392128.0,
+ "99": 935098368.0,
+ "100": 936785920.0
+ }
+ },
+ "mem-max-allocated-bytes": {
+ "start_step": 1,
+ "end_step": 100,
+ "step_interval": 1,
+ "values": {
+ "1": 936453632.0,
+ "2": 1158617088.0,
+ "3": 1158617088.0,
+ "4": 1246761472.0,
+ "5": 1247365632.0,
+ "6": 1247365632.0,
+ "7": 1247765504.0,
+ "8": 1247765504.0,
+ "9": 1247765504.0,
+ "10": 1252415488.0,
+ "11": 1252415488.0,
+ "12": 1252415488.0,
+ "13": 1252415488.0,
+ "14": 1252415488.0,
+ "15": 1252415488.0,
+ "16": 1252415488.0,
+ "17": 1252415488.0,
+ "18": 1252415488.0,
+ "19": 1252415488.0,
+ "20": 1252415488.0,
+ "21": 1252415488.0,
+ "22": 1252415488.0,
+ "23": 1252415488.0,
+ "24": 1252415488.0,
+ "25": 1252415488.0,
+ "26": 1252415488.0,
+ "27": 1252415488.0,
+ "28": 1252415488.0,
+ "29": 1252415488.0,
+ "30": 1252415488.0,
+ "31": 1252415488.0,
+ "32": 1252415488.0,
+ "33": 1252415488.0,
+ "34": 1252415488.0,
+ "35": 1252415488.0,
+ "36": 1252415488.0,
+ "37": 1252415488.0,
+ "38": 1252415488.0,
+ "39": 1252415488.0,
+ "40": 1252415488.0,
+ "41": 1252415488.0,
+ "42": 1252415488.0,
+ "43": 1252415488.0,
+ "44": 1252415488.0,
+ "45": 1252415488.0,
+ "46": 1252415488.0,
+ "47": 1252415488.0,
+ "48": 1252415488.0,
+ "49": 1252415488.0,
+ "50": 1252415488.0,
+ "51": 1252415488.0,
+ "52": 1252415488.0,
+ "53": 1252415488.0,
+ "54": 1252415488.0,
+ "55": 1252415488.0,
+ "56": 1252415488.0,
+ "57": 1252415488.0,
+ "58": 1252415488.0,
+ "59": 1252415488.0,
+ "60": 1252415488.0,
+ "61": 1252415488.0,
+ "62": 1252415488.0,
+ "63": 1252415488.0,
+ "64": 1252415488.0,
+ "65": 1252415488.0,
+ "66": 1252415488.0,
+ "67": 1252415488.0,
+ "68": 1252415488.0,
+ "69": 1252415488.0,
+ "70": 1252415488.0,
+ "71": 1252415488.0,
+ "72": 1252415488.0,
+ "73": 1252415488.0,
+ "74": 1252415488.0,
+ "75": 1252415488.0,
+ "76": 1252415488.0,
+ "77": 1252415488.0,
+ "78": 1252415488.0,
+ "79": 1252415488.0,
+ "80": 1252415488.0,
+ "81": 1252415488.0,
+ "82": 1252415488.0,
+ "83": 1252415488.0,
+ "84": 1252415488.0,
+ "85": 1252415488.0,
+ "86": 1252415488.0,
+ "87": 1252415488.0,
+ "88": 1252415488.0,
+ "89": 1252415488.0,
+ "90": 1252415488.0,
+ "91": 1252415488.0,
+ "92": 1252415488.0,
+ "93": 1252415488.0,
+ "94": 1252415488.0,
+ "95": 1252415488.0,
+ "96": 1252415488.0,
+ "97": 1252415488.0,
+ "98": 1252415488.0,
+ "99": 1252415488.0,
+ "100": 1252415488.0
+ }
+ },
+ "mtp_1 loss": {
+ "start_step": 1,
+ "end_step": 100,
+ "step_interval": 1,
+ "values": {
+ "1": 10.88691,
+ "2": 10.90544,
+ "3": 10.90868,
+ "4": 10.86912,
+ "5": 10.91636,
+ "6": 10.90651,
+ "7": 10.90278,
+ "8": 10.88975,
+ "9": 10.90453,
+ "10": 10.89162,
+ "11": 10.93392,
+ "12": 10.91634,
+ "13": 10.91136,
+ "14": 10.91999,
+ "15": 10.88538,
+ "16": 10.90717,
+ "17": 10.87525,
+ "18": 10.91409,
+ "19": 10.90936,
+ "20": 10.87835,
+ "21": 10.8786,
+ "22": 10.85481,
+ "23": 10.87937,
+ "24": 10.87208,
+ "25": 10.85798,
+ "26": 10.86991,
+ "27": 10.87718,
+ "28": 10.88667,
+ "29": 10.88859,
+ "30": 10.85479,
+ "31": 10.79701,
+ "32": 10.86609,
+ "33": 10.87789,
+ "34": 10.8397,
+ "35": 10.84184,
+ "36": 10.85,
+ "37": 10.85585,
+ "38": 10.83714,
+ "39": 10.86361,
+ "40": 10.82866,
+ "41": 10.83386,
+ "42": 10.84447,
+ "43": 10.78747,
+ "44": 10.82127,
+ "45": 10.78826,
+ "46": 10.78323,
+ "47": 10.82894,
+ "48": 10.7901,
+ "49": 10.71201,
+ "50": 10.77359,
+ "51": 10.76681,
+ "52": 10.74029,
+ "53": 10.8027,
+ "54": 10.77345,
+ "55": 10.76133,
+ "56": 10.71153,
+ "57": 10.66673,
+ "58": 10.74318,
+ "59": 10.69182,
+ "60": 10.66418,
+ "61": 10.70712,
+ "62": 10.77164,
+ "63": 10.61759,
+ "64": 10.71667,
+ "65": 10.4936,
+ "66": 10.67118,
+ "67": 10.57515,
+ "68": 10.68716,
+ "69": 10.68277,
+ "70": 10.66908,
+ "71": 10.64566,
+ "72": 10.60905,
+ "73": 10.56507,
+ "74": 10.37106,
+ "75": 10.5114,
+ "76": 10.39856,
+ "77": 10.75192,
+ "78": 10.62708,
+ "79": 10.4675,
+ "80": 10.47474,
+ "81": 10.51003,
+ "82": 10.58819,
+ "83": 10.43946,
+ "84": 10.45015,
+ "85": 10.55142,
+ "86": 10.2831,
+ "87": 10.51182,
+ "88": 10.60318,
+ "89": 10.50948,
+ "90": 10.60407,
+ "91": 10.38208,
+ "92": 10.38708,
+ "93": 10.23019,
+ "94": 10.08381,
+ "95": 10.4259,
+ "96": 10.4489,
+ "97": 10.32133,
+ "98": 10.49668,
+ "99": 10.04795,
+ "100": 10.33446
+ }
+ },
+ "iteration-time": {
+ "start_step": 1,
+ "end_step": 100,
+ "step_interval": 1,
+ "values": {
+ "1": 74.16337,
+ "2": 1.6487,
+ "3": 1.45105,
+ "4": 4.39166,
+ "5": 0.72113,
+ "6": 0.82637,
+ "7": 0.7985,
+ "8": 0.73623,
+ "9": 0.7398,
+ "10": 0.74065,
+ "11": 0.73395,
+ "12": 0.73395,
+ "13": 0.79806,
+ "14": 0.7251,
+ "15": 0.7312,
+ "16": 0.75102,
+ "17": 0.72379,
+ "18": 0.72614,
+ "19": 0.73367,
+ "20": 0.73334,
+ "21": 0.72408,
+ "22": 0.74787,
+ "23": 0.75535,
+ "24": 0.72783,
+ "25": 0.7314,
+ "26": 0.71985,
+ "27": 0.7246,
+ "28": 0.72236,
+ "29": 0.71945,
+ "30": 0.72182,
+ "31": 0.72292,
+ "32": 0.71754,
+ "33": 0.7157,
+ "34": 0.70975,
+ "35": 0.72388,
+ "36": 0.71455,
+ "37": 0.71511,
+ "38": 0.71163,
+ "39": 0.71376,
+ "40": 0.72067,
+ "41": 0.71279,
+ "42": 0.70858,
+ "43": 0.7086,
+ "44": 0.70995,
+ "45": 0.70901,
+ "46": 0.70881,
+ "47": 0.71115,
+ "48": 0.72369,
+ "49": 0.73908,
+ "50": 0.81598,
+ "51": 0.73667,
+ "52": 0.71381,
+ "53": 0.72282,
+ "54": 0.73549,
+ "55": 0.70748,
+ "56": 0.7102,
+ "57": 0.70853,
+ "58": 0.70998,
+ "59": 0.71846,
+ "60": 0.70825,
+ "61": 0.70848,
+ "62": 0.70734,
+ "63": 0.7097,
+ "64": 0.72007,
+ "65": 0.71061,
+ "66": 0.7223,
+ "67": 0.71411,
+ "68": 0.71437,
+ "69": 0.70943,
+ "70": 0.70895,
+ "71": 0.71052,
+ "72": 0.70672,
+ "73": 0.72725,
+ "74": 0.70761,
+ "75": 0.7334,
+ "76": 0.7387,
+ "77": 0.72758,
+ "78": 0.72748,
+ "79": 0.73386,
+ "80": 0.72774,
+ "81": 0.71859,
+ "82": 0.71526,
+ "83": 0.75425,
+ "84": 0.72064,
+ "85": 0.72017,
+ "86": 0.72277,
+ "87": 0.73635,
+ "88": 0.72228,
+ "89": 0.73388,
+ "90": 0.74435,
+ "91": 0.7281,
+ "92": 0.71839,
+ "93": 0.71175,
+ "94": 0.71437,
+ "95": 0.71311,
+ "96": 0.71386,
+ "97": 0.71412,
+ "98": 0.72944,
+ "99": 0.7486,
+ "100": 0.74015
+ }
+ }
+}
\ No newline at end of file
diff --git a/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_tp4_ep2_etp2_pp2_scoped_cudagraph/golden_values_dev_dgxh100_coreweave.json b/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_tp4_ep2_etp2_pp2_scoped_cudagraph/golden_values_dev_dgxh100_coreweave.json
new file mode 100644
index 00000000000..309b2533461
--- /dev/null
+++ b/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_tp4_ep2_etp2_pp2_scoped_cudagraph/golden_values_dev_dgxh100_coreweave.json
@@ -0,0 +1,644 @@
+{
+ "lm loss": {
+ "start_step": 1,
+ "end_step": 100,
+ "step_interval": 1,
+ "values": {
+ "1": 10.93663,
+ "2": 10.9327,
+ "3": 10.94263,
+ "4": 10.94969,
+ "5": 10.95052,
+ "6": 10.94157,
+ "7": 10.94484,
+ "8": 10.93674,
+ "9": 10.94996,
+ "10": 10.93686,
+ "11": 10.94102,
+ "12": 10.93763,
+ "13": 10.9235,
+ "14": 10.93428,
+ "15": 10.88791,
+ "16": 10.87434,
+ "17": 10.86896,
+ "18": 10.86065,
+ "19": 10.86311,
+ "20": 10.78063,
+ "21": 10.73125,
+ "22": 10.60283,
+ "23": 10.73278,
+ "24": 10.61888,
+ "25": 10.55212,
+ "26": 10.62704,
+ "27": 10.6391,
+ "28": 10.5908,
+ "29": 10.59809,
+ "30": 10.37777,
+ "31": 10.1201,
+ "32": 10.46078,
+ "33": 10.45538,
+ "34": 10.20107,
+ "35": 10.25779,
+ "36": 10.20889,
+ "37": 10.33688,
+ "38": 10.16827,
+ "39": 10.40875,
+ "40": 10.05239,
+ "41": 10.09432,
+ "42": 10.17894,
+ "43": 9.74205,
+ "44": 9.8904,
+ "45": 9.74009,
+ "46": 9.72707,
+ "47": 10.09139,
+ "48": 9.75298,
+ "49": 9.40106,
+ "50": 9.83667,
+ "51": 9.77071,
+ "52": 9.65705,
+ "53": 10.03051,
+ "54": 9.87899,
+ "55": 9.79604,
+ "56": 9.52924,
+ "57": 9.36583,
+ "58": 9.75331,
+ "59": 9.48065,
+ "60": 9.40785,
+ "61": 9.60145,
+ "62": 9.90753,
+ "63": 9.2583,
+ "64": 9.68397,
+ "65": 8.80003,
+ "66": 9.60779,
+ "67": 9.25408,
+ "68": 9.71438,
+ "69": 9.71682,
+ "70": 9.6617,
+ "71": 9.52466,
+ "72": 9.47116,
+ "73": 9.38822,
+ "74": 8.80223,
+ "75": 9.33966,
+ "76": 8.93574,
+ "77": 9.99333,
+ "78": 9.64731,
+ "79": 9.28114,
+ "80": 9.29588,
+ "81": 9.39589,
+ "82": 9.60893,
+ "83": 9.21629,
+ "84": 9.33891,
+ "85": 9.52979,
+ "86": 8.95817,
+ "87": 9.51641,
+ "88": 9.68228,
+ "89": 9.50664,
+ "90": 9.75348,
+ "91": 9.23465,
+ "92": 9.25972,
+ "93": 8.94517,
+ "94": 8.69188,
+ "95": 9.44591,
+ "96": 9.4101,
+ "97": 9.20087,
+ "98": 9.58175,
+ "99": 8.75818,
+ "100": 9.29466
+ }
+ },
+ "num-zeros": {
+ "start_step": 1,
+ "end_step": 100,
+ "step_interval": 1,
+ "values": {
+ "1": 22750260.0,
+ "2": 22953110.0,
+ "3": 22604450.0,
+ "4": 23266322.0,
+ "5": 22735560.0,
+ "6": 23061920.0,
+ "7": 22793342.0,
+ "8": 22960820.0,
+ "9": 22865664.0,
+ "10": 22950364.0,
+ "11": 22499674.0,
+ "12": 22456088.0,
+ "13": 22948060.0,
+ "14": 22384512.0,
+ "15": 22846272.0,
+ "16": 22856858.0,
+ "17": 22836412.0,
+ "18": 22590058.0,
+ "19": 22627048.0,
+ "20": 22712308.0,
+ "21": 22762624.0,
+ "22": 22816888.0,
+ "23": 22545124.0,
+ "24": 22794440.0,
+ "25": 22841936.0,
+ "26": 22549680.0,
+ "27": 22464820.0,
+ "28": 22453684.0,
+ "29": 22534640.0,
+ "30": 22636152.0,
+ "31": 22989488.0,
+ "32": 22594070.0,
+ "33": 22566010.0,
+ "34": 22855504.0,
+ "35": 22813688.0,
+ "36": 22595396.0,
+ "37": 22499360.0,
+ "38": 22926126.0,
+ "39": 22825392.0,
+ "40": 22675666.0,
+ "41": 22671586.0,
+ "42": 22682140.0,
+ "43": 23013940.0,
+ "44": 22764458.0,
+ "45": 22678992.0,
+ "46": 22915276.0,
+ "47": 22642868.0,
+ "48": 22954190.0,
+ "49": 23786668.0,
+ "50": 22934008.0,
+ "51": 23866222.0,
+ "52": 23807290.0,
+ "53": 24007532.0,
+ "54": 22871610.0,
+ "55": 23571284.0,
+ "56": 23954310.0,
+ "57": 24211632.0,
+ "58": 23914404.0,
+ "59": 23771838.0,
+ "60": 23813560.0,
+ "61": 23797288.0,
+ "62": 23739984.0,
+ "63": 23916692.0,
+ "64": 23895952.0,
+ "65": 24150562.0,
+ "66": 23796504.0,
+ "67": 25032232.0,
+ "68": 23673188.0,
+ "69": 23648580.0,
+ "70": 23903504.0,
+ "71": 24864636.0,
+ "72": 24767108.0,
+ "73": 24850612.0,
+ "74": 24132990.0,
+ "75": 24146528.0,
+ "76": 25025540.0,
+ "77": 24358472.0,
+ "78": 24910064.0,
+ "79": 23810516.0,
+ "80": 24821440.0,
+ "81": 25020512.0,
+ "82": 23851244.0,
+ "83": 24961024.0,
+ "84": 25144020.0,
+ "85": 24823608.0,
+ "86": 23153096.0,
+ "87": 24850204.0,
+ "88": 24749150.0,
+ "89": 22505554.0,
+ "90": 24059620.0,
+ "91": 23839038.0,
+ "92": 23874568.0,
+ "93": 24769548.0,
+ "94": 23992452.0,
+ "95": 25189838.0,
+ "96": 23909262.0,
+ "97": 24713068.0,
+ "98": 23832506.0,
+ "99": 23983474.0,
+ "100": 24101108.0
+ }
+ },
+ "mem-allocated-bytes": {
+ "start_step": 1,
+ "end_step": 100,
+ "step_interval": 1,
+ "values": {
+ "1": 763142656.0,
+ "2": 778734592.0,
+ "3": 772525056.0,
+ "4": 803593216.0,
+ "5": 803593216.0,
+ "6": 803593216.0,
+ "7": 801299456.0,
+ "8": 803593216.0,
+ "9": 801840128.0,
+ "10": 803593216.0,
+ "11": 802987008.0,
+ "12": 803593216.0,
+ "13": 802987008.0,
+ "14": 801299456.0,
+ "15": 803593216.0,
+ "16": 801840128.0,
+ "17": 803593216.0,
+ "18": 802987008.0,
+ "19": 801299456.0,
+ "20": 803593216.0,
+ "21": 801299456.0,
+ "22": 803593216.0,
+ "23": 801299456.0,
+ "24": 803593216.0,
+ "25": 801299456.0,
+ "26": 803593216.0,
+ "27": 801299456.0,
+ "28": 803593216.0,
+ "29": 801299456.0,
+ "30": 803593216.0,
+ "31": 801299456.0,
+ "32": 803593216.0,
+ "33": 801840128.0,
+ "34": 803593216.0,
+ "35": 801840128.0,
+ "36": 803593216.0,
+ "37": 802987008.0,
+ "38": 801299456.0,
+ "39": 803593216.0,
+ "40": 801299456.0,
+ "41": 803593216.0,
+ "42": 801840128.0,
+ "43": 803593216.0,
+ "44": 801840128.0,
+ "45": 803593216.0,
+ "46": 801840128.0,
+ "47": 803593216.0,
+ "48": 801840128.0,
+ "49": 803593216.0,
+ "50": 801840128.0,
+ "51": 801299456.0,
+ "52": 803593216.0,
+ "53": 801299456.0,
+ "54": 803593216.0,
+ "55": 801840128.0,
+ "56": 803593216.0,
+ "57": 801840128.0,
+ "58": 803593216.0,
+ "59": 801840128.0,
+ "60": 803593216.0,
+ "61": 801299456.0,
+ "62": 803593216.0,
+ "63": 801299456.0,
+ "64": 802987008.0,
+ "65": 803593216.0,
+ "66": 801299456.0,
+ "67": 803593216.0,
+ "68": 801299456.0,
+ "69": 803593216.0,
+ "70": 801840128.0,
+ "71": 803593216.0,
+ "72": 801299456.0,
+ "73": 803593216.0,
+ "74": 803593216.0,
+ "75": 802987008.0,
+ "76": 803593216.0,
+ "77": 801840128.0,
+ "78": 803593216.0,
+ "79": 801299456.0,
+ "80": 802987008.0,
+ "81": 803593216.0,
+ "82": 801840128.0,
+ "83": 803593216.0,
+ "84": 801299456.0,
+ "85": 802987008.0,
+ "86": 803593216.0,
+ "87": 801840128.0,
+ "88": 803593216.0,
+ "89": 801299456.0,
+ "90": 802987008.0,
+ "91": 803593216.0,
+ "92": 801299456.0,
+ "93": 803593216.0,
+ "94": 801299456.0,
+ "95": 803593216.0,
+ "96": 801299456.0,
+ "97": 803593216.0,
+ "98": 801299456.0,
+ "99": 802987008.0,
+ "100": 803593216.0
+ }
+ },
+ "mem-max-allocated-bytes": {
+ "start_step": 1,
+ "end_step": 100,
+ "step_interval": 1,
+ "values": {
+ "1": 993582592.0,
+ "2": 1210942464.0,
+ "3": 1210942464.0,
+ "4": 1210942464.0,
+ "5": 1210942464.0,
+ "6": 1210942464.0,
+ "7": 1210942464.0,
+ "8": 1210942464.0,
+ "9": 1210942464.0,
+ "10": 1210942464.0,
+ "11": 1210942464.0,
+ "12": 1210942464.0,
+ "13": 1210942464.0,
+ "14": 1210942464.0,
+ "15": 1210942464.0,
+ "16": 1210942464.0,
+ "17": 1210942464.0,
+ "18": 1210942464.0,
+ "19": 1210942464.0,
+ "20": 1210942464.0,
+ "21": 1210942464.0,
+ "22": 1210942464.0,
+ "23": 1210942464.0,
+ "24": 1210942464.0,
+ "25": 1210942464.0,
+ "26": 1210942464.0,
+ "27": 1210942464.0,
+ "28": 1210942464.0,
+ "29": 1210942464.0,
+ "30": 1210942464.0,
+ "31": 1210942464.0,
+ "32": 1210942464.0,
+ "33": 1210942464.0,
+ "34": 1210942464.0,
+ "35": 1210942464.0,
+ "36": 1210942464.0,
+ "37": 1210942464.0,
+ "38": 1210942464.0,
+ "39": 1210942464.0,
+ "40": 1210942464.0,
+ "41": 1210942464.0,
+ "42": 1210942464.0,
+ "43": 1210942464.0,
+ "44": 1210942464.0,
+ "45": 1210942464.0,
+ "46": 1210942464.0,
+ "47": 1210942464.0,
+ "48": 1210942464.0,
+ "49": 1210942464.0,
+ "50": 1210942464.0,
+ "51": 1210942464.0,
+ "52": 1210942464.0,
+ "53": 1210942464.0,
+ "54": 1210942464.0,
+ "55": 1210942464.0,
+ "56": 1210942464.0,
+ "57": 1210942464.0,
+ "58": 1210942464.0,
+ "59": 1210942464.0,
+ "60": 1210942464.0,
+ "61": 1210942464.0,
+ "62": 1210942464.0,
+ "63": 1210942464.0,
+ "64": 1210942464.0,
+ "65": 1210942464.0,
+ "66": 1210942464.0,
+ "67": 1210942464.0,
+ "68": 1210942464.0,
+ "69": 1210942464.0,
+ "70": 1210942464.0,
+ "71": 1210942464.0,
+ "72": 1210942464.0,
+ "73": 1210942464.0,
+ "74": 1210942464.0,
+ "75": 1210942464.0,
+ "76": 1210942464.0,
+ "77": 1210942464.0,
+ "78": 1210942464.0,
+ "79": 1210942464.0,
+ "80": 1210942464.0,
+ "81": 1210942464.0,
+ "82": 1210942464.0,
+ "83": 1210942464.0,
+ "84": 1210942464.0,
+ "85": 1210942464.0,
+ "86": 1210942464.0,
+ "87": 1210942464.0,
+ "88": 1210942464.0,
+ "89": 1210942464.0,
+ "90": 1210942464.0,
+ "91": 1210942464.0,
+ "92": 1210942464.0,
+ "93": 1210942464.0,
+ "94": 1210942464.0,
+ "95": 1210942464.0,
+ "96": 1210942464.0,
+ "97": 1210942464.0,
+ "98": 1210942464.0,
+ "99": 1210942464.0,
+ "100": 1210942464.0
+ }
+ },
+ "mtp_1 loss": {
+ "start_step": 1,
+ "end_step": 100,
+ "step_interval": 1,
+ "values": {
+ "1": 10.88689,
+ "2": 10.90485,
+ "3": 10.90869,
+ "4": 10.86903,
+ "5": 10.91601,
+ "6": 10.906,
+ "7": 10.90268,
+ "8": 10.88984,
+ "9": 10.90425,
+ "10": 10.89144,
+ "11": 10.93384,
+ "12": 10.91647,
+ "13": 10.91108,
+ "14": 10.91974,
+ "15": 10.88488,
+ "16": 10.9077,
+ "17": 10.87571,
+ "18": 10.91379,
+ "19": 10.9092,
+ "20": 10.87837,
+ "21": 10.87896,
+ "22": 10.85583,
+ "23": 10.88007,
+ "24": 10.87245,
+ "25": 10.85859,
+ "26": 10.8696,
+ "27": 10.87702,
+ "28": 10.88641,
+ "29": 10.88866,
+ "30": 10.85422,
+ "31": 10.79713,
+ "32": 10.86631,
+ "33": 10.8781,
+ "34": 10.83982,
+ "35": 10.84165,
+ "36": 10.85012,
+ "37": 10.85556,
+ "38": 10.83674,
+ "39": 10.86355,
+ "40": 10.82887,
+ "41": 10.8341,
+ "42": 10.84469,
+ "43": 10.78828,
+ "44": 10.82123,
+ "45": 10.78831,
+ "46": 10.7823,
+ "47": 10.82898,
+ "48": 10.78985,
+ "49": 10.71269,
+ "50": 10.77382,
+ "51": 10.76639,
+ "52": 10.7397,
+ "53": 10.80285,
+ "54": 10.77365,
+ "55": 10.76066,
+ "56": 10.71068,
+ "57": 10.66686,
+ "58": 10.74378,
+ "59": 10.69209,
+ "60": 10.66474,
+ "61": 10.7073,
+ "62": 10.77206,
+ "63": 10.61812,
+ "64": 10.7178,
+ "65": 10.49439,
+ "66": 10.67106,
+ "67": 10.57534,
+ "68": 10.6873,
+ "69": 10.6816,
+ "70": 10.66836,
+ "71": 10.64586,
+ "72": 10.60925,
+ "73": 10.56508,
+ "74": 10.37144,
+ "75": 10.51183,
+ "76": 10.39914,
+ "77": 10.75182,
+ "78": 10.6268,
+ "79": 10.46827,
+ "80": 10.47524,
+ "81": 10.51083,
+ "82": 10.58769,
+ "83": 10.4381,
+ "84": 10.45057,
+ "85": 10.55084,
+ "86": 10.28076,
+ "87": 10.51088,
+ "88": 10.60323,
+ "89": 10.50794,
+ "90": 10.60274,
+ "91": 10.38238,
+ "92": 10.38703,
+ "93": 10.23076,
+ "94": 10.08438,
+ "95": 10.42616,
+ "96": 10.44905,
+ "97": 10.32215,
+ "98": 10.4966,
+ "99": 10.04765,
+ "100": 10.33491
+ }
+ },
+ "iteration-time": {
+ "start_step": 1,
+ "end_step": 100,
+ "step_interval": 1,
+ "values": {
+ "1": 51.30209,
+ "2": 1.41746,
+ "3": 1.28029,
+ "4": 10.57024,
+ "5": 0.66643,
+ "6": 0.67893,
+ "7": 0.65727,
+ "8": 0.66196,
+ "9": 0.66227,
+ "10": 0.65877,
+ "11": 0.65828,
+ "12": 0.65862,
+ "13": 0.65727,
+ "14": 0.65896,
+ "15": 0.65851,
+ "16": 0.66826,
+ "17": 0.65878,
+ "18": 0.65573,
+ "19": 0.65631,
+ "20": 0.65579,
+ "21": 0.65091,
+ "22": 0.65603,
+ "23": 0.65158,
+ "24": 0.65266,
+ "25": 0.65816,
+ "26": 0.65194,
+ "27": 0.6541,
+ "28": 0.65515,
+ "29": 0.65439,
+ "30": 0.65241,
+ "31": 0.65597,
+ "32": 0.65551,
+ "33": 0.65318,
+ "34": 0.6553,
+ "35": 0.65725,
+ "36": 0.65926,
+ "37": 0.65606,
+ "38": 0.65571,
+ "39": 0.65846,
+ "40": 0.65642,
+ "41": 0.65509,
+ "42": 0.66105,
+ "43": 0.65448,
+ "44": 0.65534,
+ "45": 0.65304,
+ "46": 0.65227,
+ "47": 0.64871,
+ "48": 0.65257,
+ "49": 0.65485,
+ "50": 0.65054,
+ "51": 0.67883,
+ "52": 0.6571,
+ "53": 0.65671,
+ "54": 0.65877,
+ "55": 0.65584,
+ "56": 0.65072,
+ "57": 0.64951,
+ "58": 0.65703,
+ "59": 0.65106,
+ "60": 0.64536,
+ "61": 0.64416,
+ "62": 0.64816,
+ "63": 0.64084,
+ "64": 0.6396,
+ "65": 0.64182,
+ "66": 0.64004,
+ "67": 0.64101,
+ "68": 0.63928,
+ "69": 0.65723,
+ "70": 0.6828,
+ "71": 0.64052,
+ "72": 0.64287,
+ "73": 0.64136,
+ "74": 0.64252,
+ "75": 0.64617,
+ "76": 0.64857,
+ "77": 0.64304,
+ "78": 0.64068,
+ "79": 0.64048,
+ "80": 0.64091,
+ "81": 0.64179,
+ "82": 0.64793,
+ "83": 0.641,
+ "84": 0.64077,
+ "85": 0.64011,
+ "86": 0.64018,
+ "87": 0.64132,
+ "88": 0.63901,
+ "89": 0.6407,
+ "90": 0.64277,
+ "91": 0.64132,
+ "92": 0.64123,
+ "93": 0.65051,
+ "94": 0.65036,
+ "95": 0.64542,
+ "96": 0.64561,
+ "97": 0.6504,
+ "98": 0.64563,
+ "99": 0.64524,
+ "100": 0.65049
+ }
+ }
+}
\ No newline at end of file
diff --git a/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_tp4_ep2_etp2_pp2_scoped_cudagraph/golden_values_dev_dgxh100_eos.json b/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_tp4_ep2_etp2_pp2_scoped_cudagraph/golden_values_dev_dgxh100_eos.json
new file mode 100644
index 00000000000..e8c2bae571f
--- /dev/null
+++ b/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_tp4_ep2_etp2_pp2_scoped_cudagraph/golden_values_dev_dgxh100_eos.json
@@ -0,0 +1,644 @@
+{
+ "lm loss": {
+ "start_step": 1,
+ "end_step": 100,
+ "step_interval": 1,
+ "values": {
+ "1": 10.93663,
+ "2": 10.9327,
+ "3": 10.94263,
+ "4": 10.94969,
+ "5": 10.95052,
+ "6": 10.94157,
+ "7": 10.94484,
+ "8": 10.93674,
+ "9": 10.94996,
+ "10": 10.93686,
+ "11": 10.94102,
+ "12": 10.93763,
+ "13": 10.9235,
+ "14": 10.93428,
+ "15": 10.88791,
+ "16": 10.87434,
+ "17": 10.86896,
+ "18": 10.86065,
+ "19": 10.86311,
+ "20": 10.78063,
+ "21": 10.73125,
+ "22": 10.60283,
+ "23": 10.73278,
+ "24": 10.61888,
+ "25": 10.55212,
+ "26": 10.62704,
+ "27": 10.6391,
+ "28": 10.5908,
+ "29": 10.59809,
+ "30": 10.37777,
+ "31": 10.1201,
+ "32": 10.46078,
+ "33": 10.45538,
+ "34": 10.20107,
+ "35": 10.25779,
+ "36": 10.20889,
+ "37": 10.33688,
+ "38": 10.16827,
+ "39": 10.40875,
+ "40": 10.05239,
+ "41": 10.09432,
+ "42": 10.17894,
+ "43": 9.74205,
+ "44": 9.8904,
+ "45": 9.74009,
+ "46": 9.72707,
+ "47": 10.09139,
+ "48": 9.75298,
+ "49": 9.40106,
+ "50": 9.83667,
+ "51": 9.77071,
+ "52": 9.65705,
+ "53": 10.03051,
+ "54": 9.87899,
+ "55": 9.79604,
+ "56": 9.52924,
+ "57": 9.36583,
+ "58": 9.75331,
+ "59": 9.48065,
+ "60": 9.40785,
+ "61": 9.60145,
+ "62": 9.90753,
+ "63": 9.2583,
+ "64": 9.68397,
+ "65": 8.80003,
+ "66": 9.60779,
+ "67": 9.25408,
+ "68": 9.71438,
+ "69": 9.71682,
+ "70": 9.6617,
+ "71": 9.52466,
+ "72": 9.47116,
+ "73": 9.38822,
+ "74": 8.80223,
+ "75": 9.33966,
+ "76": 8.93574,
+ "77": 9.99333,
+ "78": 9.64731,
+ "79": 9.28114,
+ "80": 9.29588,
+ "81": 9.39589,
+ "82": 9.60893,
+ "83": 9.21629,
+ "84": 9.33891,
+ "85": 9.52979,
+ "86": 8.95817,
+ "87": 9.51641,
+ "88": 9.68228,
+ "89": 9.50664,
+ "90": 9.75348,
+ "91": 9.23465,
+ "92": 9.25972,
+ "93": 8.94517,
+ "94": 8.69188,
+ "95": 9.44591,
+ "96": 9.4101,
+ "97": 9.20087,
+ "98": 9.58175,
+ "99": 8.75818,
+ "100": 9.29466
+ }
+ },
+ "num-zeros": {
+ "start_step": 1,
+ "end_step": 100,
+ "step_interval": 1,
+ "values": {
+ "1": 22750260.0,
+ "2": 22953110.0,
+ "3": 22604450.0,
+ "4": 23266322.0,
+ "5": 22735560.0,
+ "6": 23061920.0,
+ "7": 22793342.0,
+ "8": 22960820.0,
+ "9": 22865664.0,
+ "10": 22950364.0,
+ "11": 22499674.0,
+ "12": 22456088.0,
+ "13": 22948060.0,
+ "14": 22384512.0,
+ "15": 22846272.0,
+ "16": 22856858.0,
+ "17": 22836412.0,
+ "18": 22590058.0,
+ "19": 22627048.0,
+ "20": 22712308.0,
+ "21": 22762624.0,
+ "22": 22816888.0,
+ "23": 22545124.0,
+ "24": 22794440.0,
+ "25": 22841936.0,
+ "26": 22549680.0,
+ "27": 22464820.0,
+ "28": 22453684.0,
+ "29": 22534640.0,
+ "30": 22636152.0,
+ "31": 22989488.0,
+ "32": 22594070.0,
+ "33": 22566010.0,
+ "34": 22855504.0,
+ "35": 22813688.0,
+ "36": 22595396.0,
+ "37": 22499360.0,
+ "38": 22926126.0,
+ "39": 22825392.0,
+ "40": 22675666.0,
+ "41": 22671586.0,
+ "42": 22682140.0,
+ "43": 23013940.0,
+ "44": 22764458.0,
+ "45": 22678992.0,
+ "46": 22915276.0,
+ "47": 22642868.0,
+ "48": 22954190.0,
+ "49": 23786668.0,
+ "50": 22934008.0,
+ "51": 23866222.0,
+ "52": 23807290.0,
+ "53": 24007532.0,
+ "54": 22871610.0,
+ "55": 23571284.0,
+ "56": 23954310.0,
+ "57": 24211632.0,
+ "58": 23914404.0,
+ "59": 23771838.0,
+ "60": 23813560.0,
+ "61": 23797288.0,
+ "62": 23739984.0,
+ "63": 23916692.0,
+ "64": 23895952.0,
+ "65": 24150562.0,
+ "66": 23796504.0,
+ "67": 25032232.0,
+ "68": 23673188.0,
+ "69": 23648580.0,
+ "70": 23903504.0,
+ "71": 24864636.0,
+ "72": 24767108.0,
+ "73": 24850612.0,
+ "74": 24132990.0,
+ "75": 24146528.0,
+ "76": 25025540.0,
+ "77": 24358472.0,
+ "78": 24910064.0,
+ "79": 23810516.0,
+ "80": 24821440.0,
+ "81": 25020512.0,
+ "82": 23851244.0,
+ "83": 24961024.0,
+ "84": 25144020.0,
+ "85": 24823608.0,
+ "86": 23153096.0,
+ "87": 24850204.0,
+ "88": 24749150.0,
+ "89": 22505554.0,
+ "90": 24059620.0,
+ "91": 23839038.0,
+ "92": 23874568.0,
+ "93": 24769548.0,
+ "94": 23992452.0,
+ "95": 25189838.0,
+ "96": 23909262.0,
+ "97": 24713068.0,
+ "98": 23832506.0,
+ "99": 23983474.0,
+ "100": 24101108.0
+ }
+ },
+ "mem-allocated-bytes": {
+ "start_step": 1,
+ "end_step": 100,
+ "step_interval": 1,
+ "values": {
+ "1": 769688064.0,
+ "2": 775359488.0,
+ "3": 769690624.0,
+ "4": 801299456.0,
+ "5": 803593216.0,
+ "6": 801299456.0,
+ "7": 803593216.0,
+ "8": 803593216.0,
+ "9": 801299456.0,
+ "10": 803593216.0,
+ "11": 801299456.0,
+ "12": 803593216.0,
+ "13": 801299456.0,
+ "14": 803593216.0,
+ "15": 803593216.0,
+ "16": 801299456.0,
+ "17": 803593216.0,
+ "18": 801299456.0,
+ "19": 803593216.0,
+ "20": 801299456.0,
+ "21": 803593216.0,
+ "22": 803593216.0,
+ "23": 801840128.0,
+ "24": 803593216.0,
+ "25": 802987008.0,
+ "26": 801299456.0,
+ "27": 802987008.0,
+ "28": 801299456.0,
+ "29": 801299456.0,
+ "30": 803593216.0,
+ "31": 801299456.0,
+ "32": 803593216.0,
+ "33": 801299456.0,
+ "34": 803593216.0,
+ "35": 801299456.0,
+ "36": 801299456.0,
+ "37": 803593216.0,
+ "38": 801299456.0,
+ "39": 803593216.0,
+ "40": 801299456.0,
+ "41": 803593216.0,
+ "42": 801299456.0,
+ "43": 801299456.0,
+ "44": 803593216.0,
+ "45": 802987008.0,
+ "46": 801299456.0,
+ "47": 803593216.0,
+ "48": 801299456.0,
+ "49": 803593216.0,
+ "50": 801299456.0,
+ "51": 801299456.0,
+ "52": 803593216.0,
+ "53": 802446336.0,
+ "54": 801299456.0,
+ "55": 803593216.0,
+ "56": 802987008.0,
+ "57": 801299456.0,
+ "58": 801840128.0,
+ "59": 801299456.0,
+ "60": 803593216.0,
+ "61": 801840128.0,
+ "62": 801299456.0,
+ "63": 803593216.0,
+ "64": 802446336.0,
+ "65": 803593216.0,
+ "66": 801840128.0,
+ "67": 801299456.0,
+ "68": 803593216.0,
+ "69": 801840128.0,
+ "70": 801299456.0,
+ "71": 803593216.0,
+ "72": 803593216.0,
+ "73": 802987008.0,
+ "74": 801299456.0,
+ "75": 803593216.0,
+ "76": 803593216.0,
+ "77": 801299456.0,
+ "78": 801299456.0,
+ "79": 803593216.0,
+ "80": 801840128.0,
+ "81": 801299456.0,
+ "82": 803593216.0,
+ "83": 801299456.0,
+ "84": 801299456.0,
+ "85": 803593216.0,
+ "86": 801299456.0,
+ "87": 801299456.0,
+ "88": 803593216.0,
+ "89": 801840128.0,
+ "90": 803593216.0,
+ "91": 802987008.0,
+ "92": 801299456.0,
+ "93": 803593216.0,
+ "94": 801299456.0,
+ "95": 801299456.0,
+ "96": 803593216.0,
+ "97": 801840128.0,
+ "98": 803593216.0,
+ "99": 802987008.0,
+ "100": 801299456.0
+ }
+ },
+ "mem-max-allocated-bytes": {
+ "start_step": 1,
+ "end_step": 100,
+ "step_interval": 1,
+ "values": {
+ "1": 988765184.0,
+ "2": 1206831616.0,
+ "3": 1210116096.0,
+ "4": 1210116096.0,
+ "5": 1210116096.0,
+ "6": 1210116096.0,
+ "7": 1210116096.0,
+ "8": 1210116096.0,
+ "9": 1210116096.0,
+ "10": 1210116096.0,
+ "11": 1210116096.0,
+ "12": 1210116096.0,
+ "13": 1210116096.0,
+ "14": 1210116096.0,
+ "15": 1210116096.0,
+ "16": 1210116096.0,
+ "17": 1210116096.0,
+ "18": 1210116096.0,
+ "19": 1210116096.0,
+ "20": 1210116096.0,
+ "21": 1210116096.0,
+ "22": 1210116096.0,
+ "23": 1210116096.0,
+ "24": 1210116096.0,
+ "25": 1210116096.0,
+ "26": 1210116096.0,
+ "27": 1210116096.0,
+ "28": 1210116096.0,
+ "29": 1210116096.0,
+ "30": 1210116096.0,
+ "31": 1210116096.0,
+ "32": 1210116096.0,
+ "33": 1210116096.0,
+ "34": 1210116096.0,
+ "35": 1210116096.0,
+ "36": 1210116096.0,
+ "37": 1210116096.0,
+ "38": 1210116096.0,
+ "39": 1210116096.0,
+ "40": 1210116096.0,
+ "41": 1210116096.0,
+ "42": 1210116096.0,
+ "43": 1210116096.0,
+ "44": 1210116096.0,
+ "45": 1210116096.0,
+ "46": 1210116096.0,
+ "47": 1210116096.0,
+ "48": 1210116096.0,
+ "49": 1210116096.0,
+ "50": 1210116096.0,
+ "51": 1210116096.0,
+ "52": 1210116096.0,
+ "53": 1210116096.0,
+ "54": 1210116096.0,
+ "55": 1210116096.0,
+ "56": 1210116096.0,
+ "57": 1210116096.0,
+ "58": 1210116096.0,
+ "59": 1210116096.0,
+ "60": 1210116096.0,
+ "61": 1210116096.0,
+ "62": 1210116096.0,
+ "63": 1210116096.0,
+ "64": 1210116096.0,
+ "65": 1210116096.0,
+ "66": 1210116096.0,
+ "67": 1210116096.0,
+ "68": 1210116096.0,
+ "69": 1210116096.0,
+ "70": 1210116096.0,
+ "71": 1210116096.0,
+ "72": 1210116096.0,
+ "73": 1210116096.0,
+ "74": 1210116096.0,
+ "75": 1210116096.0,
+ "76": 1210116096.0,
+ "77": 1210116096.0,
+ "78": 1210116096.0,
+ "79": 1210116096.0,
+ "80": 1210116096.0,
+ "81": 1210116096.0,
+ "82": 1210116096.0,
+ "83": 1210116096.0,
+ "84": 1210116096.0,
+ "85": 1210116096.0,
+ "86": 1210116096.0,
+ "87": 1210116096.0,
+ "88": 1210116096.0,
+ "89": 1210116096.0,
+ "90": 1210116096.0,
+ "91": 1210116096.0,
+ "92": 1210116096.0,
+ "93": 1210116096.0,
+ "94": 1210116096.0,
+ "95": 1210116096.0,
+ "96": 1210116096.0,
+ "97": 1210116096.0,
+ "98": 1210116096.0,
+ "99": 1210116096.0,
+ "100": 1210116096.0
+ }
+ },
+ "mtp_1 loss": {
+ "start_step": 1,
+ "end_step": 100,
+ "step_interval": 1,
+ "values": {
+ "1": 10.88689,
+ "2": 10.90485,
+ "3": 10.90869,
+ "4": 10.86903,
+ "5": 10.91601,
+ "6": 10.906,
+ "7": 10.90268,
+ "8": 10.88984,
+ "9": 10.90425,
+ "10": 10.89144,
+ "11": 10.93384,
+ "12": 10.91647,
+ "13": 10.91108,
+ "14": 10.91974,
+ "15": 10.88488,
+ "16": 10.9077,
+ "17": 10.87571,
+ "18": 10.91379,
+ "19": 10.9092,
+ "20": 10.87837,
+ "21": 10.87896,
+ "22": 10.85583,
+ "23": 10.88007,
+ "24": 10.87245,
+ "25": 10.85859,
+ "26": 10.8696,
+ "27": 10.87702,
+ "28": 10.88641,
+ "29": 10.88866,
+ "30": 10.85422,
+ "31": 10.79713,
+ "32": 10.86631,
+ "33": 10.8781,
+ "34": 10.83982,
+ "35": 10.84165,
+ "36": 10.85012,
+ "37": 10.85556,
+ "38": 10.83674,
+ "39": 10.86355,
+ "40": 10.82887,
+ "41": 10.8341,
+ "42": 10.84469,
+ "43": 10.78828,
+ "44": 10.82123,
+ "45": 10.78831,
+ "46": 10.7823,
+ "47": 10.82898,
+ "48": 10.78985,
+ "49": 10.71269,
+ "50": 10.77382,
+ "51": 10.76639,
+ "52": 10.7397,
+ "53": 10.80285,
+ "54": 10.77365,
+ "55": 10.76066,
+ "56": 10.71068,
+ "57": 10.66686,
+ "58": 10.74378,
+ "59": 10.69209,
+ "60": 10.66474,
+ "61": 10.7073,
+ "62": 10.77206,
+ "63": 10.61812,
+ "64": 10.7178,
+ "65": 10.49439,
+ "66": 10.67106,
+ "67": 10.57534,
+ "68": 10.6873,
+ "69": 10.6816,
+ "70": 10.66836,
+ "71": 10.64586,
+ "72": 10.60925,
+ "73": 10.56508,
+ "74": 10.37144,
+ "75": 10.51183,
+ "76": 10.39914,
+ "77": 10.75182,
+ "78": 10.6268,
+ "79": 10.46827,
+ "80": 10.47524,
+ "81": 10.51083,
+ "82": 10.58769,
+ "83": 10.4381,
+ "84": 10.45057,
+ "85": 10.55084,
+ "86": 10.28076,
+ "87": 10.51088,
+ "88": 10.60323,
+ "89": 10.50794,
+ "90": 10.60274,
+ "91": 10.38238,
+ "92": 10.38703,
+ "93": 10.23076,
+ "94": 10.08438,
+ "95": 10.42616,
+ "96": 10.44905,
+ "97": 10.32215,
+ "98": 10.4966,
+ "99": 10.04765,
+ "100": 10.33491
+ }
+ },
+ "iteration-time": {
+ "start_step": 1,
+ "end_step": 100,
+ "step_interval": 1,
+ "values": {
+ "1": 58.67467,
+ "2": 1.49483,
+ "3": 1.38721,
+ "4": 11.78499,
+ "5": 0.75759,
+ "6": 0.75678,
+ "7": 0.76144,
+ "8": 0.80382,
+ "9": 0.74706,
+ "10": 0.74893,
+ "11": 0.75091,
+ "12": 0.75087,
+ "13": 0.74803,
+ "14": 0.75316,
+ "15": 0.80396,
+ "16": 0.75267,
+ "17": 0.75378,
+ "18": 0.75457,
+ "19": 0.75484,
+ "20": 0.75428,
+ "21": 0.75639,
+ "22": 0.81363,
+ "23": 0.75607,
+ "24": 0.75553,
+ "25": 0.75564,
+ "26": 0.75334,
+ "27": 0.75722,
+ "28": 0.76027,
+ "29": 0.8113,
+ "30": 0.75278,
+ "31": 0.75471,
+ "32": 0.75104,
+ "33": 0.75271,
+ "34": 0.74877,
+ "35": 0.74765,
+ "36": 0.80549,
+ "37": 0.75089,
+ "38": 0.75395,
+ "39": 0.75254,
+ "40": 0.76025,
+ "41": 0.75356,
+ "42": 0.75573,
+ "43": 0.79632,
+ "44": 0.77927,
+ "45": 0.75515,
+ "46": 0.75759,
+ "47": 0.75978,
+ "48": 0.75749,
+ "49": 0.75504,
+ "50": 0.75616,
+ "51": 0.77974,
+ "52": 0.76581,
+ "53": 0.76997,
+ "54": 0.76705,
+ "55": 0.76737,
+ "56": 0.77352,
+ "57": 0.77833,
+ "58": 0.81195,
+ "59": 0.77251,
+ "60": 0.7711,
+ "61": 0.77181,
+ "62": 0.77006,
+ "63": 0.76957,
+ "64": 0.77251,
+ "65": 0.82259,
+ "66": 0.77112,
+ "67": 0.7683,
+ "68": 0.77335,
+ "69": 0.77022,
+ "70": 0.77335,
+ "71": 0.77822,
+ "72": 0.77769,
+ "73": 0.79476,
+ "74": 0.7728,
+ "75": 0.7711,
+ "76": 0.76863,
+ "77": 0.77228,
+ "78": 0.77031,
+ "79": 0.76995,
+ "80": 0.77286,
+ "81": 0.76616,
+ "82": 0.76752,
+ "83": 0.76583,
+ "84": 0.77264,
+ "85": 0.76732,
+ "86": 0.76873,
+ "87": 0.77239,
+ "88": 0.77971,
+ "89": 0.76112,
+ "90": 0.76225,
+ "91": 0.75814,
+ "92": 0.76144,
+ "93": 0.75796,
+ "94": 0.76412,
+ "95": 0.777,
+ "96": 0.77207,
+ "97": 0.7628,
+ "98": 0.76325,
+ "99": 0.76204,
+ "100": 0.7668
+ }
+ }
+}
\ No newline at end of file
diff --git a/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_tp4_ep2_etp2_pp2_scoped_cudagraph/model_config.yaml b/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_tp4_ep2_etp2_pp2_scoped_cudagraph/model_config.yaml
new file mode 100644
index 00000000000..f0d1cc0afd3
--- /dev/null
+++ b/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_tp4_ep2_etp2_pp2_scoped_cudagraph/model_config.yaml
@@ -0,0 +1,96 @@
+ENV_VARS:
+ CUDA_DEVICE_MAX_CONNECTIONS: 1
+ NVTE_ALLOW_NONDETERMINISTIC_ALGO: 0
+ NCCL_ALGO: Ring
+ CUBLAS_WORKSPACE_CONFIG: :4096:8
+MODEL_ARGS:
+ --num-layers: 13
+ --hidden-size: 512
+ --num-attention-heads: 8
+ --mtp-num-layers: 1
+ --micro-batch-size: 2
+ --global-batch-size: 32
+ --seq-length: 1024
+ --max-position-embeddings: 1024
+ --position-embedding-type: rope
+ --rotary-base: 10000
+ --untie-embeddings-and-output-weights: true
+ --disable-bias-linear: true
+ --attention-dropout: 0.0
+ --hidden-dropout: 0.0
+ --train-iters: 100
+ --lr-decay-iters: 320000
+ --split: 949,50,1
+ --distributed-backend: nccl
+ --lr: 0.00015
+ --lr-decay-style: cosine
+ --min-lr: 1.0e-5
+ --weight-decay: 1e-2
+ --clip-grad: 1.0
+ --lr-warmup-fraction: .01
+ --transformer-impl: transformer_engine
+ --tensor-model-parallel-size: 4
+ --pipeline-model-parallel-size: 2
+ --expert-model-parallel-size: 2
+ --expert-tensor-parallel-size: 2
+ --pipeline-model-parallel-layout: Et\\|\\(tt\\|\\)*6mL # Et|(tt|)*6mL
+ --sequence-parallel: true
+ --num-experts: 8
+ --use-distributed-optimizer: true
+ --overlap-grad-reduce: true
+ --overlap-param-gather: true
+ --moe-token-dispatcher-type: alltoall
+ --moe-router-load-balancing-type: global_aux_loss
+ --moe-router-topk: 2
+ --moe-router-dtype: fp32
+ --moe-router-fusion: true
+ --moe-router-enable-expert-bias: true
+ --moe-router-score-function: sigmoid
+ --moe-router-pre-softmax: true
+ --moe-ffn-hidden-size: 1024
+ --moe-shared-expert-intermediate-size: 512
+ --moe-grouped-gemm: true
+ --moe-layer-freq: ([0]*4+[1]*9)
+ --moe-permute-fusion: true
+ --deterministic-mode: true
+ --no-gradient-accumulation-fusion: true
+ --attention-softmax-in-fp32: true
+ --use-checkpoint-opt_param-scheduler: true
+ --use-mcore-models: true
+ --bf16: true
+ --fp8-format: hybrid
+ --fp8-recipe: blockwise
+ --first-last-layers-bf16: true
+ --no-bias-gelu-fusion: true
+ --recompute-granularity: selective
+ --recompute-modules: "[moe_act]"
+ --cuda-graph-impl: transformer_engine
+ --cuda-graph-scope: "[attn mlp moe_router moe_preprocess]"
+ --log-memory-to-tensorboard: true
+ --log-params-norm: true
+ --log-num-zeros-in-grad: true
+ --log-validation-ppl-to-tensorboard: true
+ --log-timers-to-tensorboard: true
+ --tensorboard-dir: ${TENSORBOARD_PATH}
+ --log-interval: 1
+ --timing-log-level: 0
+ --save-interval: 50
+ --eval-interval: 1000
+ --eval-iters: 10
+ --data-path: ${DATA_PATH}/text/the_pile/shard00/my-gpt3_00_text_document
+ --data-cache-path: ${DATA_CACHE_PATH}
+ --vocab-file: ${DATA_PATH}/text/the_pile/shard00/bpe/vocab.json
+ --merge-file: ${DATA_PATH}/text/the_pile/shard00/bpe/merges.txt
+ --save: ${CHECKPOINT_SAVE_PATH}
+ --load: ${CHECKPOINT_LOAD_PATH}
+ --ckpt-fully-parallel-load: true
+ --ckpt-format: torch_dist
+ --ckpt-assume-constant-structure: true
+TEST_TYPE: ckpt-resume
+METRICS:
+ # - "iteration-time"
+ - "lm loss"
+ - "num-zeros"
+ - "mem-allocated-bytes"
+ - "mem-max-allocated-bytes"
+ - "mtp_1 loss"
diff --git a/tests/functional_tests/test_cases/moe2.0/golden_values/dsv3_tp1pp1ep8/golden_values_dev_dgx_h100.json b/tests/functional_tests/test_cases/moe2.0/golden_values/dsv3_tp1pp1ep8/golden_values_dev_dgx_h100.json
new file mode 100644
index 00000000000..e69de29bb2d
diff --git a/tests/functional_tests/test_cases/moe2.0/golden_values/dsv3_tp2pp2ep4/golden_values_dev_dgx_h100.json b/tests/functional_tests/test_cases/moe2.0/golden_values/dsv3_tp2pp2ep4/golden_values_dev_dgx_h100.json
new file mode 100644
index 00000000000..e69de29bb2d
diff --git a/tests/functional_tests/test_cases/moe2.0/model_configs/dsv3_proxy.yaml b/tests/functional_tests/test_cases/moe2.0/model_configs/dsv3_proxy.yaml
new file mode 100644
index 00000000000..70924aed0cc
--- /dev/null
+++ b/tests/functional_tests/test_cases/moe2.0/model_configs/dsv3_proxy.yaml
@@ -0,0 +1,85 @@
+MODEL_ARGS:
+ # Data args
+ --seq-length: 4096
+ --data-cache-path: ${DATA_CACHE_PATH}
+ --data-path: ${DATA_PATH}/text/the_pile/shard00/my-gpt3_00_text_document
+ --vocab-file: ${DATA_PATH}/text/the_pile/shard00/bpe/vocab.json
+ --merge-file: ${DATA_PATH}/text/the_pile/shard00/bpe/merges.txt
+ --split: 949,50,1
+ # Add transformer base args
+ --num-layers: 16
+ --hidden-size: 1024
+ --normalization: RMSNorm
+ --norm-epsilon: 1e-6
+ --disable-bias-linear: true
+ --max-position-embeddings: 4096
+ --make-vocab-size-divisible-by: 3232
+ --untie-embeddings-and-output-weights: true
+ # Add attention related args
+ --multi-latent-attention: true
+ --num-attention-heads: 32
+ --kv-channels: 128
+ --qk-layernorm: true
+ --position-embedding-type: rope
+ --rotary-base: 10000
+ --q-lora-rank: 1536
+ --kv-lora-rank: 512
+ --qk-head-dim: 128
+ --qk-pos-emb-head-dim: 64
+ --v-head-dim: 128
+ --rotary-scaling-factor: 40
+ --mscale: 1.0
+ --mscale-all-dim: 1.0
+ # Add MLP related args
+ --swiglu: true
+ --ffn-hidden-size: 4096
+ # Add MoE args
+ --num-experts: 32
+ --moe-layer-freq: ([0]*1+[1]*15)
+ --moe-ffn-hidden-size: 1024
+ --moe-shared-expert-intermediate-size: 1024
+ --moe-router-load-balancing-type: seq_aux_loss
+ --moe-router-topk: 4
+ --moe-router-pre-softmax: true
+ --moe-grouped-gemm: true
+ --moe-aux-loss-coeff: 1e-4
+ --moe-router-group-topk: 2
+ --moe-router-num-groups: 4
+ --moe-router-topk-scaling-factor: 2.0
+ --moe-router-score-function: sigmoid
+ --moe-router-enable-expert-bias: true
+ --moe-router-bias-update-rate: 1e-3
+ --moe-router-dtype: fp32
+ # Comment out the following MTP args to disable MTP
+ --mtp-num-layers: 1
+ --mtp-loss-scaling-factor: 0.1
+ # Add regularization args
+ --attention-dropout: 0.0
+ --hidden-dropout: 0.0
+ --clip-grad: 1.0
+ --weight-decay: 0.1
+ # Add learning rate args
+ --lr-warmup-fraction: .01
+ --lr: 0.00015
+ --min-lr: 1.0e-5
+ --lr-decay-style: cosine
+ --adam-beta1: 0.9
+ --adam-beta2: 0.95
+ # Add validation args
+ --eval-iters: 32
+ --eval-interval: 200
+ # Add initialization args
+ --init-method-std: 0.02
+ # Training args
+ --global-batch-size: 32
+ --train-iters: 50
+ --exit-duration-in-mins: 230
+ --no-check-for-nan-in-loss-and-grad: true
+
+METRICS:
+ - "lm loss"
+ - "num-zeros"
+ - "mem-allocated-bytes"
+ - "mem-max-allocated-bytes"
+ - "mtp_1 loss"
+ - "seq_load_balancing_loss"
diff --git a/tests/functional_tests/test_cases/moe2.0/model_configs/qwen3_proxy.yaml b/tests/functional_tests/test_cases/moe2.0/model_configs/qwen3_proxy.yaml
new file mode 100644
index 00000000000..46e298ec971
--- /dev/null
+++ b/tests/functional_tests/test_cases/moe2.0/model_configs/qwen3_proxy.yaml
@@ -0,0 +1,74 @@
+MODEL_ARGS:
+ # Data args
+ --seq-length: 4096
+ --data-cache-path: ${DATA_CACHE_PATH}
+ --data-path: ${DATA_PATH}/text/the_pile/shard00/my-gpt3_00_text_document
+ --vocab-file: ${DATA_PATH}/text/the_pile/shard00/bpe/vocab.json
+ --merge-file: ${DATA_PATH}/text/the_pile/shard00/bpe/merges.txt
+ --split: 949,50,1
+ # Add transformer base args
+ --num-layers: 16
+ --hidden-size: 1024
+ --normalization: RMSNorm
+ --norm-epsilon: 1e-6
+ --disable-bias-linear: true
+ --max-position-embeddings: 4096
+ --make-vocab-size-divisible-by: 3232
+ --untie-embeddings-and-output-weights: true
+ # Add attention related args
+ --group-query-attention: true
+ --num-query-groups: 4
+ --kv-channels: 128
+ --qk-layernorm: true
+ --position-embedding-type: rope
+ --rotary-percent: 1.0
+ --rotary-base: 1000000
+ # Add MLP related args
+ --swiglu: true
+ --ffn-hidden-size: 4096
+ # Add MoE args
+ --num-experts: 32
+ --moe-layer-freq: ([0]*1+[1]*15)
+ --moe-ffn-hidden-size: 1024
+ --moe-shared-expert-intermediate-size: 1024
+ --moe-router-load-balancing-type: aux_loss
+ --moe-router-topk: 4
+ --moe-router-pre-softmax: true
+ --moe-grouped-gemm: true
+ --moe-aux-loss-coeff: 1e-4
+ --moe-router-group-topk: 2
+ --moe-router-num-groups: 4
+ --moe-router-topk-scaling-factor: 2.0
+ --moe-router-score-function: sigmoid
+ --moe-router-enable-expert-bias: true
+ --moe-router-bias-update-rate: 1e-3
+ --moe-router-dtype: fp32
+ # Add regularization args
+ --attention-dropout: 0.0
+ --hidden-dropout: 0.0
+ --clip-grad: 1.0
+ --weight-decay: 0.1
+ # Add learning rate args
+ --lr-warmup-fraction: .01
+ --lr: 0.00015
+ --min-lr: 1.0e-5
+ --lr-decay-style: cosine
+ --adam-beta1: 0.9
+ --adam-beta2: 0.95
+ # Add validation args
+ --eval-iters: 32
+ --eval-interval: 200
+ # Add initialization args
+ --init-method-std: 0.02
+ # Training args
+ --global-batch-size: 32
+ --train-iters: 50
+ --exit-duration-in-mins: 230
+ --no-check-for-nan-in-loss-and-grad: true
+
+METRICS:
+ - "lm loss"
+ - "num-zeros"
+ - "mem-allocated-bytes"
+ - "mem-max-allocated-bytes"
+ - "load_balancing_loss"
diff --git a/tests/functional_tests/test_cases/moe2.0/runtime_configs/tp1pp1ep8.yaml b/tests/functional_tests/test_cases/moe2.0/runtime_configs/tp1pp1ep8.yaml
new file mode 100644
index 00000000000..305e2847305
--- /dev/null
+++ b/tests/functional_tests/test_cases/moe2.0/runtime_configs/tp1pp1ep8.yaml
@@ -0,0 +1,41 @@
+ENV_VARS:
+ CUDA_DEVICE_MAX_CONNECTIONS: 1
+ NVTE_ALLOW_NONDETERMINISTIC_ALGO: 0
+ PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True
+ NCCL_NVLS_ENABLE: 0
+ PYTHONWARNINGS: ignore
+ NCCL_DEBUG: VERSION
+
+MODEL_ARGS:
+ # Transformer Engine args
+ --transformer-impl: transformer_engine
+ # Distributed args
+ --distributed-timeout-minutes: 60
+ --tensor-model-parallel-size: 1
+ --pipeline-model-parallel-size: 1
+ --expert-model-parallel-size: 8
+ --context-parallel-size: 1
+ --expert-tensor-parallel-size: 1
+ --use-distributed-optimizer: true
+ --overlap-grad-reduce: true
+ --overlap-param-gather: true
+ # Use unfused attention since MLA with fused attention and deterministic mode leads to NaN
+ --attention-backend: unfused # TODO: switch back to fused attention after fix
+ --use-mcore-models: true
+ --sequence-parallel: true
+ --micro-batch-size: 4
+ # MoE training related args
+ --moe-token-dispatcher-type: alltoall
+ --moe-permute-fusion: true
+ --save-interval: 25
+ # Add mixed precision args
+ --bf16: true
+ --exit-interval: 50
+ # kernel fusion related args
+ --no-rope-fusion: true
+ --cross-entropy-loss-fusion: true
+ --cross-entropy-fusion-impl: native
+ # MISC
+ --manual-gc: true
+ --manual-gc-interval: 100
+TEST_TYPE: resume-ckpt
diff --git a/tests/functional_tests/test_cases/moe2.0/runtime_configs/tp2pp2ep4.yaml b/tests/functional_tests/test_cases/moe2.0/runtime_configs/tp2pp2ep4.yaml
new file mode 100644
index 00000000000..b93862aff8c
--- /dev/null
+++ b/tests/functional_tests/test_cases/moe2.0/runtime_configs/tp2pp2ep4.yaml
@@ -0,0 +1,55 @@
+ENV_VARS:
+ CUDA_DEVICE_MAX_CONNECTIONS: 1
+ NVTE_ALLOW_NONDETERMINISTIC_ALGO: 0
+ PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True
+ NCCL_NVLS_ENABLE: 0
+ PYTHONWARNINGS: ignore
+ NCCL_DEBUG: VERSION
+
+MODEL_ARGS:
+ # Transformer Engine args
+ --transformer-impl: transformer_engine
+ # Distributed args
+ --distributed-timeout-minutes: 60
+ --tensor-model-parallel-size: 2
+ --pipeline-model-parallel-size: 2
+ --num-virtual-stages-per-pipeline-rank: 4
+ --expert-model-parallel-size: 4
+ --context-parallel-size: 1
+ --expert-tensor-parallel-size: 1
+ --use-distributed-optimizer: true
+ --overlap-grad-reduce: true
+ --overlap-param-gather: true
+ # Use unfused attention since MLA with fused attention and deterministic mode leads to NaN
+ --attention-backend: unfused # TODO: switch back to fused attention after fix
+ --use-mcore-models: true
+ --sequence-parallel: true
+ --micro-batch-size: 4
+ # MoE training related args
+ --moe-token-dispatcher-type: alltoall
+ --moe-permute-fusion: true
+ # Add checkpointing args
+ --save: ${CHECKPOINT_SAVE_PATH}
+ --load: ${CHECKPOINT_LOAD_PATH}
+ --save-interval: 25
+ # Add logging args
+ --log-timers-to-tensorboard: true
+ --log-memory-to-tensorboard: true
+ --log-num-zeros-in-grad: true
+ --log-params-norm: true
+ --log-validation-ppl-to-tensorboard: true
+ --log-throughput: true
+ --log-interval: 1
+ --logging-level: 40
+ --tensorboard-dir: ${TENSORBOARD_PATH}
+ # Add mixed precision args
+ --bf16: true
+ --exit-interval: 50
+ # kernel fusion related args
+ --no-rope-fusion: true
+ --cross-entropy-loss-fusion: true
+ --cross-entropy-fusion-impl: native
+ # MISC
+ --manual-gc: true
+ --manual-gc-interval: 100
+TEST_TYPE: resume-ckpt
\ No newline at end of file
diff --git a/tests/functional_tests/test_cases/t5/t5_release/model_config.yaml b/tests/functional_tests/test_cases/t5/t5_release/model_config.yaml
index 852fbf9819d..b684a2ebb54 100644
--- a/tests/functional_tests/test_cases/t5/t5_release/model_config.yaml
+++ b/tests/functional_tests/test_cases/t5/t5_release/model_config.yaml
@@ -37,7 +37,7 @@ MODEL_ARGS:
--pipeline-model-parallel-size: 1
# Data args
--data-path: ${DATA_BLEND}
- --vocab-file: ${DATA_PATH}/text/the_pile/t5_shard00/bert-large-cased-vocab.txt
+ --vocab-file: ${DATA_PATH}/bert-large-cased-vocab.txt
--tokenizer-type: BertWordPieceCase
--split: 99982,9,9
--data-cache-path: ${DATA_CACHE_PATH}
diff --git a/tests/test_utils/python_scripts/launch_jet_workload.py b/tests/test_utils/python_scripts/launch_jet_workload.py
index 6ecd98a06c1..7f60ceb12d6 100644
--- a/tests/test_utils/python_scripts/launch_jet_workload.py
+++ b/tests/test_utils/python_scripts/launch_jet_workload.py
@@ -8,6 +8,7 @@
import signal
import sys
import time
+import uuid
import zipfile
from typing import Dict, List, Optional
@@ -111,15 +112,12 @@ def launch_and_wait_for_completion(
"HF_HUB_CACHE": "/lustre/fsw/coreai_dlalgo_mcore/hf_hub",
"TRANSFORMERS_OFFLINE": "1",
"CLUSTER": cluster,
+ "RUN_ID": str(uuid.uuid4()),
}
}
}
}
},
- "outputs": {
- "enabled": True,
- "artifacts_storages": [recipe_parser.resolve_artifact_config(cluster)],
- },
},
wait_for_validation=True,
max_wait_time=(60 * 60),
diff --git a/tests/test_utils/python_scripts/launch_nemo_run_workload.py b/tests/test_utils/python_scripts/launch_nemo_run_workload.py
index 6e2b73e430f..26a7dbd79f5 100644
--- a/tests/test_utils/python_scripts/launch_nemo_run_workload.py
+++ b/tests/test_utils/python_scripts/launch_nemo_run_workload.py
@@ -115,6 +115,7 @@ def main(
"ENABLE_LIGHTWEIGHT_MODE": str(enable_lightweight_mode).lower(),
"N_REPEAT": "1",
"CLUSTER": "dgxh100_dgxc",
+ "NCCL_DEBUG": "INFO",
},
packager=run.Packager(),
volumes=artifacts,
diff --git a/tests/test_utils/python_scripts/merge_config.py b/tests/test_utils/python_scripts/merge_config.py
new file mode 100644
index 00000000000..176706038b7
--- /dev/null
+++ b/tests/test_utils/python_scripts/merge_config.py
@@ -0,0 +1,92 @@
+"""
+Merges base_config, runtime_config and model_config into one final config that the CI can launch.
+
+Starting Dec 19th 2025 MCore CI supports a new format of defining tests. We are decoupling the test
+config into a modular system of base_config, model_config and runtime_config. This allows us to
+re-use and parametrize a given model easily with multiple runtime configs, like parallelism settings.
+
+With this DRY principle, we simplify test maintenance and reduce the amount of code duplication.
+
+This refactoring is fully compliant with the original CI system as we merge the three configs into one
+final config that the CI can launch.
+
+Precendence: Base config > Model config > Runtime config.
+
+Usage:
+
+python merge_config.py \
+ --model_config model_config.yaml \
+ --base_config base_config.yaml \
+ --runtime_config runtime_config.yaml \
+ --output_config output_config.yaml
+"""
+
+import logging
+
+import click
+import yaml
+
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+
+@click.command()
+@click.option("--model_config", type=str, help="Model config to merge")
+@click.option("--base_config", type=str, help="Base config to merge")
+@click.option("--runtime_config", type=str, help="Run time config to merge")
+@click.option("--output_config", type=str, help="Output config to merge")
+def main(model_config, base_config, runtime_config, output_config):
+
+ with open(model_config, "r") as f:
+ model_config = yaml.safe_load(f)
+ with open(base_config, "r") as f:
+ base_config = yaml.safe_load(f)
+ with open(runtime_config, "r") as f:
+ runtime_config = yaml.safe_load(f)
+
+ config = {}
+
+ # Collect all top-level keys (ENV_VARS, MODEL_ARGS, etc.)
+ all_keys = set(base_config.keys()) | set(model_config.keys()) | set(runtime_config.keys())
+
+ for key in all_keys:
+ base_val = base_config.get(key)
+ model_val = model_config.get(key)
+ runtime_val = runtime_config.get(key)
+
+ # Get first non-None value to check type
+ first_val = base_val or model_val or runtime_val
+
+ if isinstance(first_val, dict):
+ # Merge dicts
+ config[key] = {}
+ for val in [base_val, model_val, runtime_val]:
+ if val:
+ config[key].update(val)
+ elif isinstance(first_val, list):
+ # Concatenate lists (deduplicate while preserving order)
+ config[key] = []
+ seen = set()
+ for val in [base_val, model_val, runtime_val]:
+ if val:
+ for item in val:
+ if item not in seen:
+ config[key].append(item)
+ seen.add(item)
+ else:
+ # Scalar value (string, int, bool, etc.) - use last defined
+ if runtime_val is not None:
+ config[key] = runtime_val
+ elif model_val is not None:
+ config[key] = model_val
+ else:
+ config[key] = base_val
+
+ with open(output_config, "w") as f:
+ yaml.dump(config, f)
+
+ logger.info(f"Config merged and saved to {output_config}")
+
+
+if __name__ == "__main__":
+ main()
diff --git a/tests/test_utils/python_scripts/recipe_parser.py b/tests/test_utils/python_scripts/recipe_parser.py
index e26d04d6f20..c6e7c5517e8 100644
--- a/tests/test_utils/python_scripts/recipe_parser.py
+++ b/tests/test_utils/python_scripts/recipe_parser.py
@@ -1,3 +1,4 @@
+# Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
import copy
import itertools
import logging
@@ -23,6 +24,8 @@ class dotdict(dict):
def resolve_cluster_config(cluster: str) -> str:
if cluster == "dgxh100_eos":
return "eos"
+ if cluster == "dgxgb200_oci-hsg":
+ return "oci-hsg"
if cluster == "dgxa100_dracooci":
return "draco-oci-iad"
if cluster == "dgxa100_dracooci-ord":
@@ -34,28 +37,36 @@ def resolve_cluster_config(cluster: str) -> str:
raise ValueError(f"Unknown cluster {cluster} provided.")
-def resolve_artifact_config(cluster: str) -> str:
- if cluster == "dgxh100_eos":
- return "eos_lustre"
- if cluster == "dgxa100_dracooci":
- return "draco-oci_lustre"
- if cluster == "dgxa100_dracooci-ord":
- return "draco-oci-ord_lustre"
- if cluster == "dgxh100_coreweave":
- return "coreweave_lustre"
- raise ValueError(f"Unknown cluster {cluster} provided.")
-
-
def flatten_products(workload_manifest: dotdict) -> dotdict:
"""Flattens a nested dict of products"""
- workload_manifest.products = [
- dict(**dict(zip(inp.keys(), values)), **{"test_case": product["test_case"][0]})
- for product in (workload_manifest.products or [])
- if "products" in product
- for inp in product["products"]
- for values in itertools.product(*inp.values())
- ]
-
+ expanded_products = []
+
+ for product in workload_manifest.products or []:
+ # Skip products that don't have nested product specifications
+ if "products" not in product:
+ continue
+
+ test_case = product["test_case"][0]
+
+ # Iterate over each input specification in the product
+ for inp in product["products"]:
+ # Generate all combinations of the input values (Cartesian product)
+ model_config = inp.pop("model_config", None)
+ runtime_config = inp.pop("runtime_config", None)
+ keys = inp.keys()
+ value_combinations = itertools.product(*inp.values())
+
+ # Create a flattened product dict for each combination
+ for values in value_combinations:
+ product_dict = dict(zip(keys, values))
+ product_dict["test_case"] = test_case
+ if model_config:
+ product_dict["model_config"] = model_config
+ if runtime_config:
+ product_dict["runtime_config"] = runtime_config
+ expanded_products.append(product_dict)
+
+ workload_manifest.products = expanded_products
return workload_manifest
@@ -98,11 +109,16 @@ def load_and_flatten(config_path: str) -> List[dotdict]:
def filter_by_test_case(workload_manifests: List[dotdict], test_case: str) -> Optional[dotdict]:
"""Returns a workload with matching name. Raises an error if there no or more than a single workload."""
+ print(len(workload_manifests))
workload_manifests = list(
workload_manifest
for workload_manifest in workload_manifests
if workload_manifest["spec"]["test_case"] == test_case
)
+ print(len(workload_manifests))
+
+ for w in workload_manifests:
+ print(w["spec"]["test_case"])
if len(workload_manifests) > 1:
logger.info("Duplicate test_case found!")
diff --git a/tests/test_utils/recipes/_build-mcore-dev.yaml b/tests/test_utils/recipes/_build-mcore-dev.yaml
index 123250d7469..d82417ea5e3 100644
--- a/tests/test_utils/recipes/_build-mcore-dev.yaml
+++ b/tests/test_utils/recipes/_build-mcore-dev.yaml
@@ -3,7 +3,7 @@ format_version: 1
maintainers: [maanug]
spec:
name: mcore-pyt-dev
- platforms: [linux/amd64]
+ platforms: [linux/amd64,linux/arm64]
source:
# The image tag will be added via `jet-tests.yaml`
# Tags are one of {buildcache, $CI_PIPELINE_ID}
diff --git a/tests/test_utils/recipes/_build-mcore-lts.yaml b/tests/test_utils/recipes/_build-mcore-lts.yaml
index d017b71c101..8efa6faa1e5 100644
--- a/tests/test_utils/recipes/_build-mcore-lts.yaml
+++ b/tests/test_utils/recipes/_build-mcore-lts.yaml
@@ -3,7 +3,7 @@ format_version: 1
maintainers: [maanug]
spec:
name: mcore-pyt-lts
- platforms: [linux/amd64]
+ platforms: [linux/amd64,linux/arm64]
source:
# The image tag will be added via `jet-tests.yaml`
# Tags are one of {buildcache, $CI_PIPELINE_ID}
diff --git a/tests/test_utils/recipes/bert.yaml b/tests/test_utils/recipes/gpt-gb200.yaml
similarity index 60%
rename from tests/test_utils/recipes/bert.yaml
rename to tests/test_utils/recipes/gpt-gb200.yaml
index 49fed7f5542..70b89e31a0e 100644
--- a/tests/test_utils/recipes/bert.yaml
+++ b/tests/test_utils/recipes/gpt-gb200.yaml
@@ -3,14 +3,13 @@ format_version: 1
maintainers: [mcore]
loggers: [stdout]
spec:
- name: '{test_case}_{environment}_{platforms}'
- model: bert
- nodes: 1
+ name: "{test_case}_{environment}_{platforms}"
+ model: gpt
build: mcore-pyt-{environment}
- gpus: 8
+ nodes: 2
+ gpus: 4
+ n_repeat: 5
platforms: dgx_a100
- time_limit:
- n_repeat:
script_setup: |
unset https_proxy
echo "machine gitlab-master.nvidia.com login okoenig password $RO_API_TOKEN" | tee -a /root/.netrc
@@ -37,15 +36,18 @@ spec:
script: |-
ls
cd /opt/megatron-lm
+
NAME=$(echo {test_case}_{environment} | sed 's/dgx_h100/dgx_a100/g')
+ export GPUS_PER_NODE={gpus}
+
ARGUMENTS=(
"DATA_PATH=/mnt/artifacts"
- "DATA_CACHE_PATH=/workspace/data/cache"
+ "DATA_CACHE_PATH=/lustre/fsw/coreai_dlalgo_mcore/mcore_ci/data/$RUN_ID/cache/"
"OUTPUT_PATH={assets_dir}"
"TENSORBOARD_PATH={assets_dir}/tensorboard"
"CHECKPOINT_SAVE_PATH={artifacts_dir}/checkpoints"
- "CHECKPOINT_LOAD_PATH=/mnt/artifacts"
- "TRAINING_SCRIPT_PATH=pretrain_bert.py"
+ "CHECKPOINT_LOAD_PATH=/mnt/artifacts/"
+ "TRAINING_SCRIPT_PATH=pretrain_gpt.py"
"TRAINING_PARAMS_PATH=./tests/functional_tests/test_cases/{model}/{test_case}/model_config.yaml"
"GOLDEN_VALUES_PATH=./tests/functional_tests/test_cases/{model}/{test_case}/golden_values_{environment}_{platforms}.json"
"N_REPEAT={n_repeat}"
@@ -53,41 +55,17 @@ spec:
"RECORD_CHECKPOINTS=${{RECORD_CHECKPOINTS}}"
)
- bash ./tests/functional_tests/shell_test_utils/run_ci_test.sh ${{ARGUMENTS[@]}}
+ set +x
+ bash ./tests/functional_tests/shell_test_utils/run_ci_test.sh ${{ARGUMENTS[@]}}
+ exit_code=$?
+ echo "Exit code: $exit_code"
+ rm -rf /lustre/fsw/coreai_dlalgo_mcore/mcore_ci/data/$RUN_ID || true
+ set -x
+ exit $exit_code
products:
- - test_case: [bert_mcore_tp2_pp2]
- products:
- - environment: [dev]
- scope: [mr]
- platforms: [dgx_h100]
- - test_case: [bert_mcore_tp2_pp2_local_spec]
- products:
- - environment: [dev]
- scope: [mr]
- platforms: [dgx_h100]
- - test_case: [bert_mcore_tp2_pp2_resume_torch_dist]
- products:
- - environment: [dev]
- scope: [mr]
- platforms: [dgx_h100]
- - test_case: [bert_mcore_tp2_pp2_resume_torch_dist_local_spec]
- products:
- - environment: [dev]
- scope: [mr]
- platforms: [dgx_h100]
- - test_case: [bert_mcore_tp1_pp2]
- products:
- - environment: [dev]
- scope: [nightly]
- platforms: [dgx_h100]
- - test_case: [bert_mcore_tp1_pp4_vp2]
- products:
- - environment: [dev]
- scope: [nightly]
- platforms: [dgx_h100]
- - test_case: [bert_mcore_tp4_pp1]
+ - test_case: [gpt3_mcore_te_tp1_pp4_vp1_dist_optimizer_overlap_grad_reduce_param_gather_overlap_optimizer]
products:
- environment: [dev]
scope: [nightly]
- platforms: [dgx_h100]
+ platforms: [dgx_gb200]
diff --git a/tests/test_utils/recipes/gpt-grpo.yaml b/tests/test_utils/recipes/gpt-grpo.yaml
index 76f1ea2d3a9..90e9815c5fe 100644
--- a/tests/test_utils/recipes/gpt-grpo.yaml
+++ b/tests/test_utils/recipes/gpt-grpo.yaml
@@ -54,11 +54,11 @@ spec:
bash ./tests/functional_tests/shell_test_utils/run_ci_test.sh ${{ARGUMENTS[@]}}
products:
- - test_case: [gpt_grpo_tp1_pp1_dp8_583m_throughputtest]
- products:
- - environment: [dev]
- scope: [mr]
- platforms: [dgx_h100]
+ # - test_case: [gpt_grpo_tp1_pp1_dp8_583m_throughputtest]
+ # products:
+ # - environment: [dev]
+ # scope: [mr]
+ # platforms: [dgx_h100]
- test_case: [gpt_grpo_tp1_pp1_dp8_583m_throughputtest_github]
products:
- environment: [dev]
diff --git a/tests/test_utils/recipes/gpt.yaml b/tests/test_utils/recipes/gpt.yaml
index 1b1ab41231d..90eddc55c27 100644
--- a/tests/test_utils/recipes/gpt.yaml
+++ b/tests/test_utils/recipes/gpt.yaml
@@ -3,7 +3,7 @@ format_version: 1
maintainers: [mcore]
loggers: [stdout]
spec:
- name: '{test_case}_{environment}_{platforms}'
+ name: "{test_case}_{environment}_{platforms}"
model: gpt
build: mcore-pyt-{environment}
nodes: 1
@@ -342,6 +342,11 @@ products:
platforms: [dgx_h100]
- environment: [lts]
scope: [nightly]
+ - test_case: [gpt3_mcore_te_tp2_pp1_gdn]
+ products:
+ - environment: [dev]
+ scope: [mr, mr-github, mr-github-slim]
+ platforms: [dgx_h100]
- test_case: [gpt3_mcore_te_tp2_pp2_mla]
products:
- environment: [dev]
@@ -457,7 +462,7 @@ products:
- test_case: [gpt3_mcore_te_tp1_pp4_vp1_dist_optimizer_overlap_grad_reduce_param_gather_overlap_optimizer]
products:
- environment: [lts]
- scope: [mr]
+ scope: [nightly]
- environment: [dev]
scope: [mr, mr-github, mr-github-slim]
platforms: [dgx_h100]
@@ -467,11 +472,11 @@ products:
scope: [mr]
platforms: [dgx_h100]
- environment: [lts]
- scope: [mr]
+ scope: [nightly]
- test_case: [gpt3_mcore_te_tp4_pp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce_param_gather]
products:
- environment: [lts]
- scope: [mr]
+ scope: [nightly]
- environment: [dev]
scope: [mr, mr-github, mr-github-slim]
platforms: [dgx_h100]
@@ -481,7 +486,7 @@ products:
scope: [mr]
platforms: [dgx_h100]
- environment: [lts]
- scope: [mr]
+ scope: [nightly]
# - test_case: [gpt3_mcore_te_tp4_pp2_frozen_resume_torch_dist_reshard_8x1xNone]
# products:
# - environment: [dev]
diff --git a/tests/test_utils/recipes/mamba-static-inference.yaml b/tests/test_utils/recipes/mamba-static-inference.yaml
index be15f02a462..7cee0a47f56 100644
--- a/tests/test_utils/recipes/mamba-static-inference.yaml
+++ b/tests/test_utils/recipes/mamba-static-inference.yaml
@@ -63,4 +63,4 @@ products:
products:
- environment: [dev]
scope: [mr]
- platforms: [dg x_h100]
+ platforms: [dgx_h100]
diff --git a/tests/test_utils/recipes/moe.yaml b/tests/test_utils/recipes/moe.yaml
index 86d2b248e39..02c3f68b5f1 100644
--- a/tests/test_utils/recipes/moe.yaml
+++ b/tests/test_utils/recipes/moe.yaml
@@ -60,51 +60,16 @@ products:
#######################################################################
# Nightly tests: Run both DEV and LTS unless something is flaky #
#######################################################################
- - test_case: [gpt3_mcore_tp2_pp2_ep2_te_4experts2parallel]
- products:
- - environment: [dev]
- scope: [nightly]
- platforms: [dgx_a100, dgx_h100]
- - environment: [lts]
- scope: [nightly]
- - test_case: [gpt3_mcore_tp2_cp2_pp2_ep2_te_4experts2parallel]
- products:
- - environment: [dev]
- scope: [nightly]
- platforms: [dgx_a100, dgx_h100]
- test_case: [gpt3_mcore_tp2_pp2_ep2_etp2_te_4experts2parallel]
products:
- environment: [dev]
scope: [nightly]
platforms: [dgx_a100, dgx_h100]
- - environment: [lts]
- scope: [nightly]
- - test_case: [gpt3_mcore_tp2_pp2_ep2_etp2_te_4experts2parallel_dp_last]
- products:
- - environment: [dev]
- scope: [nightly]
- platforms: [dgx_a100, dgx_h100]
- - environment: [lts]
- scope: [nightly]
- test_case: [gpt3_mcore_tp2_cp2_pp2_ep2_te_4experts2parallel_dp_last]
products:
- environment: [dev]
scope: [nightly]
platforms: [dgx_a100, dgx_h100]
- - test_case: [gpt3_mcore_cp2_pp2_ep2_te_4experts2parallel_nondeterministic]
- products:
- - environment: [dev]
- scope: [nightly]
- platforms: [dgx_a100, dgx_h100]
- - environment: [lts]
- scope: [nightly]
- - test_case: [gpt3_mcore_cp2_pp2_ep2_te_4experts2parallel_nondeterministic_dp_last]
- products:
- - environment: [dev]
- scope: [nightly]
- platforms: [dgx_a100, dgx_h100]
- - environment: [lts]
- scope: [nightly]
# - test_case: [gpt3_mcore_tp2_pp2_resume_torch_dist_te_2experts]
# products: # non-determinism: #478
# - environment: [dev, lts]
@@ -121,15 +86,10 @@ products:
- environment: [dev]
scope: [mr]
platforms: [dgx_h100]
- # - test_case: [gpt3_mcore_te_tp2_pp2_ep4_etp1_mtp_resume_torch_dist_fp8]
- # products:
- # - environment: [dev]
- # scope: [mr]
- # platforms: [dgx_h100] # hang: #513
- test_case: [gpt3_mcore_te_tp2_pp2_ep4_etp1_resume_torch_dist_attn_cudagraph]
products:
- environment: [dev]
- scope: [mr]
+ scope: [mr-broken]
platforms: [dgx_h100] # hang: #513
# - test_case: [gpt3_mcore_te_tp2_pp2_ep4_etp1_selective_recompute_experimental]
# products:
@@ -166,20 +126,43 @@ products:
- test_case: [gpt3_mcore_te_tp2_zp_z3_resume_torch_dist_te_8experts2parallel_top2router]
products:
- environment: [dev]
- scope: [mr]
+ scope: [mr, mr-github]
platforms: [dgx_h100]
- test_case: [gpt3_mcore_te_tp2_pp1_te_8experts2parallel_ddp_average_in_collective]
products:
- environment: [dev]
scope: [mr]
platforms: [dgx_h100]
- - environment: [lts]
- scope: [nightly]
- test_case: [gpt3_moe_mcore_te_ep8_resume_torch_dist_dist_optimizer]
products:
- environment: [dev]
scope: [mr]
platforms: [dgx_h100]
+ - test_case: [gpt3_moe_mcore_te_ep8_resume_torch_dist_dist_muon]
+ products:
+ - environment: [dev]
+ scope: [mr, mr-github]
+ platforms: [dgx_h100]
+ - test_case: [gpt3_moe_mcore_te_ep8_resume_torch_dist_muon]
+ products:
+ - environment: [dev]
+ scope: [mr, mr-github]
+ platforms: [dgx_h100]
+ - test_case: [gpt3_moe_mcore_te_tp2_pp2_ep4_etp1_no_mtp_no_a2a_ovlp_fine_grained_offloading]
+ products:
+ - environment: [dev]
+ scope: [mr-broken, mr-github]
+ platforms: [dgx_h100]
+ - test_case: [gpt3_moe_mcore_te_tp2_pp2_ep4_etp1_fine_grained_offloading]
+ products:
+ - environment: [dev]
+ scope: [mr, mr-github]
+ platforms: [dgx_h100]
+ - test_case: [gpt3_moe_mcore_te_tp4_ep2_etp2_pp2_scoped_cudagraph]
+ products:
+ - environment: [dev]
+ scope: [mr, mr-github]
+ platforms: [dgx_h100]
#######################################################################
# Super important mr, mr-github tests that run for both DEV and LTS per mr, mr-github #
#######################################################################
@@ -204,5 +187,13 @@ products:
- test_case: [gpt3_mcore_te_tp2_pp2_ep4_etp1_memory_speed]
products:
- environment: [dev]
- scope: [mr]
+ scope: [mr-broken]
+ platforms: [dgx_h100]
+ - test_case: [gpt3_mcore_te_tp2_pp2_ep4_etp1_mtp_resume_torch_dist_fp8]
+ products:
+ - environment: [dev]
+ scope: [mr-broken]
+ platforms: [dgx_h100] # hang: #513
+ - environment: [dev]
+ scope: [mr-slim-broken]
platforms: [dgx_h100]
diff --git a/tests/test_utils/recipes/t5.yaml b/tests/test_utils/recipes/moe2.0.yaml
similarity index 58%
rename from tests/test_utils/recipes/t5.yaml
rename to tests/test_utils/recipes/moe2.0.yaml
index 96b560c6427..e3249dd6ad1 100644
--- a/tests/test_utils/recipes/t5.yaml
+++ b/tests/test_utils/recipes/moe2.0.yaml
@@ -4,10 +4,11 @@ maintainers: [mcore]
loggers: [stdout]
spec:
name: '{test_case}_{environment}_{platforms}'
- model: t5
+ model: moe2.0
build: mcore-pyt-{environment}
nodes: 1
gpus: 8
+ n_repeat: 5
platforms: dgx_a100
script_setup: |
unset https_proxy
@@ -38,6 +39,13 @@ spec:
NAME=$(echo {test_case}_{environment} | sed 's/dgx_h100/dgx_a100/g')
+ mkdir -p $(dirname ./tests/functional_tests/test_cases/{model}/{test_case}/model_config.yaml)
+ python ./tests/test_utils/python_scripts/merge_config.py \
+ --base_config ./tests/functional_tests/test_cases/ci_base_config.yml \
+ --model_config ./tests/functional_tests/test_cases/{model}/model_configs/{model_config}.yaml \
+ --runtime_config ./tests/functional_tests/test_cases/{model}/runtime_configs/{runtime_config}.yaml \
+ --output_config ./tests/functional_tests/test_cases/{model}/{test_case}/model_config.yaml
+
ARGUMENTS=(
"DATA_PATH=/mnt/artifacts"
"DATA_CACHE_PATH=/workspace/data/cache"
@@ -45,7 +53,7 @@ spec:
"TENSORBOARD_PATH={assets_dir}/tensorboard"
"CHECKPOINT_SAVE_PATH={artifacts_dir}/checkpoints"
"CHECKPOINT_LOAD_PATH=/mnt/artifacts"
- "TRAINING_SCRIPT_PATH=pretrain_t5.py"
+ "TRAINING_SCRIPT_PATH=pretrain_gpt.py"
"TRAINING_PARAMS_PATH=./tests/functional_tests/test_cases/{model}/{test_case}/model_config.yaml"
"GOLDEN_VALUES_PATH=./tests/functional_tests/test_cases/{model}/{test_case}/golden_values_{environment}_{platforms}.json"
"N_REPEAT={n_repeat}"
@@ -56,58 +64,34 @@ spec:
bash ./tests/functional_tests/shell_test_utils/run_ci_test.sh ${{ARGUMENTS[@]}}
products:
- - test_case: [t5_11b_mcore_tp4_pp1]
- products:
- - environment: [dev]
- scope: [mr]
- platforms: [dgx_h100]
- - test_case: [t5_mcore_te_tp4_pp1]
+ ###########################
+ # Merge train tests #
+ ###########################
+ - test_case: [dsv3_tp1pp1ep8]
products:
- - environment: [dev]
- scope: [mr]
+ - model_config: dsv3_proxy
+ runtime_config: tp1pp1ep8
+ environment: [dev]
+ scope: [broken]
platforms: [dgx_h100]
- - test_case: [t5_mcore_te_tp4_pp1_resume_torch_dist]
+ - test_case: [dsv3_tp2pp2ep4]
products:
- - environment: [dev]
- scope: [mr]
+ - model_config: dsv3_proxy
+ runtime_config: tp2pp2ep4
+ environment: [dev]
+ scope: [broken]
platforms: [dgx_h100]
- - test_case: [t5_mcore_tp4_pp1]
+ - test_case: [qwen3_tp1pp1ep1]
products:
- - environment: [dev]
- scope: [mr]
+ - model_config: qwen3_proxy
+ runtime_config: tp1pp1ep1
+ environment: [dev]
+ scope: [broken]
platforms: [dgx_h100]
- - test_case: [t5_mcore_tp4_pp1_resume_torch_dist]
+ - test_case: [qwen3_tp2pp2ep4]
products:
- - environment: [dev]
- scope: [mr]
+ - model_config: qwen3_proxy
+ runtime_config: tp2pp2ep4
+ environment: [dev]
+ scope: [broken]
platforms: [dgx_h100]
- - test_case: [t5_mcore_te_tp1_pp1_vp1_resume_torch]
- products:
- - environment: [dev]
- scope: [nightly]
- platforms: [dgx_a100, dgx_h100]
- - test_case: [t5_mcore_te_tp2_pp1_vp1]
- products:
- - environment: [dev]
- scope: [nightly]
- platforms: [dgx_a100, dgx_h100]
- - test_case: [t5_mcore_te_tp2_pp1_vp1_sequence_parallel]
- products:
- - environment: [dev]
- scope: [nightly]
- platforms: [dgx_a100, dgx_h100]
- - test_case: [t5_mcore_tp1_pp1_vp1]
- products:
- - environment: [dev]
- scope: [nightly]
- platforms: [dgx_a100, dgx_h100]
- - test_case: [t5_mcore_tp1_pp1_vp1_resume_torch]
- products:
- - environment: [dev]
- scope: [nightly]
- platforms: [dgx_a100, dgx_h100]
- - test_case: [t5_mcore_tp2_pp1_vp1]
- products:
- - environment: [dev]
- scope: [nightly]
- platforms: [dgx_a100, dgx_h100]
diff --git a/tests/unit_tests/a2a_overlap/test_cuda_graphed_schedule_chunk_1f1b.py b/tests/unit_tests/a2a_overlap/test_cuda_graphed_schedule_chunk_1f1b.py
new file mode 100644
index 00000000000..719bd5df18f
--- /dev/null
+++ b/tests/unit_tests/a2a_overlap/test_cuda_graphed_schedule_chunk_1f1b.py
@@ -0,0 +1,377 @@
+# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved.
+import gc
+import os
+import sys
+
+import pytest
+import torch
+
+from megatron.core.enums import ModelType
+from megatron.core.models.gpt.gpt_layer_specs import (
+ get_gpt_decoder_block_spec,
+ get_gpt_mtp_block_spec,
+)
+from megatron.core.models.gpt.gpt_model import GPTModel
+from megatron.core.num_microbatches_calculator import destroy_num_microbatches_calculator
+from megatron.core.pipeline_parallel.utils import set_streams
+from megatron.core.tensor_parallel.random import HAVE_TE, model_parallel_cuda_manual_seed
+from megatron.core.transformer.enums import CudaGraphScope
+from megatron.core.transformer.module import float16_to_fp32
+from megatron.core.utils import is_te_min_version, unwrap_model
+from megatron.training.arguments import core_transformer_config_from_args, parse_args, validate_args
+from megatron.training.global_vars import (
+ destroy_global_vars,
+ get_args,
+ set_args,
+ set_global_variables,
+)
+from megatron.training.training import setup_model_and_optimizer
+from tests.unit_tests.test_utilities import Utils
+
+
+def is_deep_ep_available():
+ from megatron.core.transformer.moe.fused_a2a import HAVE_DEEP_EP
+
+ return HAVE_DEEP_EP
+
+
+def is_hybrid_ep_available():
+ from megatron.core.transformer.moe.fused_a2a import HAVE_HYBRIDEP
+
+ return HAVE_HYBRIDEP
+
+
+def save(fn, message):
+ with open(fn, 'w') as f:
+ f.write(message)
+
+
+class TestPartialCudaGraphedA2AOverlap:
+ """Test that CUDA graph outputs match ep-overlapped CUDA graph outputs for various scopes."""
+
+ def setup_method(self, method):
+ self.seq_length = 512
+ self.micro_batch_size = 2
+ # Store original environment variable values
+ self.original_env = {
+ 'CUDA_DEVICE_MAX_CONNECTIONS': os.environ.get('CUDA_DEVICE_MAX_CONNECTIONS'),
+ 'NVTE_ALLOW_NONDETERMINISTIC_ALGO': os.environ.get('NVTE_ALLOW_NONDETERMINISTIC_ALGO'),
+ }
+ self.cuda_graph_helper = None
+ os.environ['CUDA_DEVICE_MAX_CONNECTIONS'] = '1'
+ os.environ['NVTE_ALLOW_NONDETERMINISTIC_ALGO'] = '0'
+
+ def teardown_method(self, method):
+ # Restore original environment variable values
+ for key, value in self.original_env.items():
+ if value is None:
+ os.environ.pop(key, None)
+ else:
+ os.environ[key] = value
+ Utils.destroy_model_parallel()
+ destroy_global_vars()
+ destroy_num_microbatches_calculator()
+ self.delete_cuda_graphs()
+
+ gc.collect()
+
+ def delete_cuda_graphs(self):
+ if self.cuda_graph_helper is not None and self.cuda_graph_helper.graphs_created():
+ self.cuda_graph_helper.delete_cuda_graphs()
+ self.cuda_graph_helper = None
+
+ def model_provider(
+ self,
+ pre_process=True,
+ post_process=True,
+ layer_spec_fn=get_gpt_decoder_block_spec,
+ **config_kwargs,
+ ):
+ model_parallel_cuda_manual_seed(123)
+ args = get_args()
+ config = core_transformer_config_from_args(args)
+ transformer_layer_spec = layer_spec_fn(
+ config,
+ use_transformer_engine=True,
+ normalization=args.normalization,
+ qk_l2_norm=args.qk_l2_norm,
+ )
+ if args.mtp_num_layers:
+ mtp_block_spec = get_gpt_mtp_block_spec(
+ config, transformer_layer_spec, use_transformer_engine=True
+ )
+ else:
+ mtp_block_spec = None
+ return GPTModel(
+ config=config,
+ transformer_layer_spec=transformer_layer_spec,
+ vocab_size=args.vocab_size,
+ max_sequence_length=args.max_position_embeddings,
+ pre_process=pre_process,
+ post_process=post_process,
+ fp16_lm_cross_entropy=args.fp16_lm_cross_entropy,
+ parallel_output=True,
+ share_embeddings_and_output_weights=not args.untie_embeddings_and_output_weights,
+ position_embedding_type=args.position_embedding_type,
+ rotary_percent=args.rotary_percent,
+ mtp_block_spec=mtp_block_spec,
+ )
+
+ def create_test_args(
+ self, cuda_graph_impl, cuda_graph_scope, cuda_graph_warmup_steps, ep_size, **kwargs
+ ):
+ destroy_global_vars()
+ destroy_num_microbatches_calculator()
+
+ sys.argv = ['test_cuda_graphs.py']
+ args = parse_args()
+ args.num_layers = 1
+ args.mtp_num_layers = None
+ args.vocab_size = 1024
+ args.hidden_size = 128
+ args.num_attention_heads = 8
+ args.max_position_embeddings = 512
+ args.global_batch_size = self.micro_batch_size * 8
+ args.micro_batch_size = self.micro_batch_size
+ args.create_attention_mask_in_dataloader = True
+ args.seq_length = self.seq_length
+ args.tensor_model_parallel_size = 2
+ args.sequence_parallel = True
+ args.pipeline_model_parallel_size = 1
+ args.context_parallel_size = 1
+ args.expert_model_parallel_size = ep_size
+ args.train_iters = 10
+ args.lr = 3e-5
+ args.bf16 = True
+ args.add_bias_linear = False
+ args.swiglu = True
+ args.use_distributed_optimizer = True
+ args.position_embedding_type = "rope"
+ args.rotary_percent = 1.0
+ args.hidden_dropout = 0.0
+ args.attention_dropout = 0.0
+ args.untie_embeddings_and_output_weights = True
+
+ # MoE settings
+ args.num_experts = 16
+ args.expert_model_parallel_size = ep_size
+ args.moe_shared_expert_intermediate_size = 1024
+ args.moe_layer_freq = kwargs.get("moe_layer_freq", "[0,0,1,1]")
+ args.moe_permute_fusion = True
+ args.moe_router_fusion = True
+ args.moe_router_topk = 2
+
+ # CUDA graph settings
+ args.cuda_graph_impl = cuda_graph_impl
+ args.cuda_graph_scope = cuda_graph_scope
+ args.cuda_graph_warmup_steps = cuda_graph_warmup_steps
+ args.use_te_rng_tracker = cuda_graph_impl != "none"
+
+ for key, value in kwargs.items():
+ assert hasattr(args, key)
+ setattr(args, key, value)
+
+ validate_args(args)
+ set_global_variables(args, False)
+ return args
+
+ def get_batch(self, seq_length, micro_batch_size):
+ data = list(range(seq_length))
+ input_ids = torch.tensor(data, dtype=torch.int64).repeat((micro_batch_size, 1)).cuda()
+ labels = 1 + torch.tensor(data, dtype=torch.int64).repeat((micro_batch_size, 1)).cuda()
+ position_ids = torch.tensor(data, dtype=torch.int64).repeat((micro_batch_size, 1)).cuda()
+ attention_mask = torch.ones(
+ (micro_batch_size, 1, seq_length, seq_length), dtype=bool
+ ).cuda()
+ loss_mask = torch.ones(seq_length).repeat((micro_batch_size, 1)).cuda()
+ return input_ids, labels, position_ids, attention_mask, loss_mask
+
+ def _run_1f1b_helper(self, gpt_model, optimizer, data, num_iters, cuda_graph_warmup_steps):
+ from megatron.core.models.common.model_chunk_schedule_plan import (
+ TransformerModelChunkSchedulePlan,
+ )
+ from megatron.core.pipeline_parallel.schedules import set_current_microbatch
+
+ schedule_plans = []
+ losses = []
+ set_current_microbatch(gpt_model[0], 1)
+
+ gpt_model[0].zero_grad_buffer()
+ optimizer.zero_grad()
+ assert cuda_graph_warmup_steps > 0, "cuda_graph_warmup_steps must be greater than 0"
+ for fwd_mb_idx in range(num_iters + 1):
+ # Capture CUDA graphs after warmup if helper is provided
+ if self.cuda_graph_helper is not None and fwd_mb_idx == cuda_graph_warmup_steps:
+ self.cuda_graph_helper.create_cudagraphs()
+
+ if fwd_mb_idx < cuda_graph_warmup_steps:
+ gpt_model[0].zero_grad_buffer()
+ optimizer.zero_grad()
+ output = gpt_model[0].forward(**data)
+ schedule_plans.append(None)
+ else:
+ if fwd_mb_idx == cuda_graph_warmup_steps:
+ extra_schedule_plan = unwrap_model(gpt_model[0]).build_schedule_plan(**data)
+ TransformerModelChunkSchedulePlan.run(extra_schedule_plan, None)
+ schedule_plans[-1] = extra_schedule_plan
+ f_schedule_plan = unwrap_model(gpt_model[0]).build_schedule_plan(**data)
+ b_schedule_plan = schedule_plans[-1]
+ schedule_plans.append(f_schedule_plan)
+ if b_schedule_plan is not None:
+ gpt_model[0].zero_grad_buffer()
+ optimizer.zero_grad()
+ output = TransformerModelChunkSchedulePlan.run(
+ f_schedule_plan,
+ b_schedule_plan,
+ b_grad=torch.ones_like(output) if fwd_mb_idx > 0 else None,
+ )
+ # Check output shapes
+ if fwd_mb_idx < num_iters:
+ assert output is not None
+ assert output.shape[0] == self.micro_batch_size
+ assert output.shape[1] == self.seq_length
+ losses.append(output)
+
+ if fwd_mb_idx < cuda_graph_warmup_steps:
+ output.backward(torch.ones_like(output))
+
+ for param in gpt_model[0].parameters():
+ assert param.main_grad is not None
+
+ update_successful, _, _ = optimizer.step()
+ assert update_successful
+
+ return losses
+
+ def _run_test_helper(
+ self,
+ ep_size,
+ cuda_graph_impl,
+ cuda_graph_scope,
+ cuda_graph_warmup_steps,
+ ep_overlap=False,
+ **kwargs,
+ ):
+ """Test fp8_param with gpt_model."""
+ args = self.create_test_args(
+ cuda_graph_impl,
+ cuda_graph_scope,
+ cuda_graph_warmup_steps,
+ ep_size,
+ overlap_moe_expert_parallel_comm=ep_overlap,
+ **kwargs,
+ )
+ if ep_overlap:
+ set_streams()
+ set_args(args)
+ torch.manual_seed(123)
+ Utils.initialize_model_parallel(
+ tensor_model_parallel_size=2, expert_model_parallel_size=ep_size
+ )
+
+ input_ids, labels, position_ids, attention_mask, loss_mask = self.get_batch(
+ self.seq_length, self.micro_batch_size
+ )
+
+ gpt_model, optimizer, _ = setup_model_and_optimizer(
+ self.model_provider, ModelType.encoder_or_decoder
+ )
+ assert len(gpt_model) == 1 # Assume only one model in the model provider.
+
+ loss_list = []
+
+ if cuda_graph_impl == "transformer_engine":
+ from megatron.core.transformer.cuda_graphs import TECudaGraphHelper
+
+ self.cuda_graph_helper = TECudaGraphHelper(
+ model=gpt_model,
+ config=gpt_model[0].config,
+ seq_length=self.seq_length,
+ micro_batch_size=self.micro_batch_size,
+ optimizers=[optimizer],
+ )
+
+ num_iters = cuda_graph_warmup_steps + 2
+ data = {
+ "input_ids": input_ids,
+ "position_ids": position_ids,
+ "attention_mask": attention_mask,
+ "labels": labels,
+ "loss_mask": loss_mask,
+ }
+ if not ep_overlap:
+ for i in range(num_iters):
+ gpt_model[0].zero_grad_buffer()
+ optimizer.zero_grad()
+
+ # Capture CUDA graphs after warmup if helper is provided
+ if self.cuda_graph_helper is not None and i == cuda_graph_warmup_steps:
+ self.cuda_graph_helper.create_cudagraphs()
+
+ output = unwrap_model(gpt_model[0]).forward(**data)
+ output = float16_to_fp32(output)
+
+ # Check output shapes
+ assert output.shape[0] == self.micro_batch_size
+ assert output.shape[1] == self.seq_length
+
+ # Verify gradients
+ output.backward(torch.ones_like(output))
+ for param in gpt_model[0].parameters():
+ assert param.main_grad is not None
+
+ update_successful, _, _ = optimizer.step()
+ assert update_successful
+
+ loss_list.append(output)
+ else:
+ loss_list = self._run_1f1b_helper(
+ gpt_model, optimizer, data, num_iters, cuda_graph_warmup_steps
+ )
+
+ self.delete_cuda_graphs()
+
+ return loss_list
+
+ @pytest.mark.skipif(
+ not (HAVE_TE and is_te_min_version("2.10.0")),
+ reason="Partial CUDA graph support requires TransformerEngine version >= 2.10.0",
+ )
+ @pytest.mark.parametrize("moe_dispatcher_type", ["alltoall", "deepep"])
+ def test_moe_partial_cudagraph_with_ep_overlap(self, moe_dispatcher_type):
+ extra_kwargs = {"moe_layer_freq": 1}
+ if moe_dispatcher_type == "deepep":
+ if not is_deep_ep_available():
+ pytest.skip("Deep EP is not available")
+ extra_kwargs["moe_token_dispatcher_type"] = "flex"
+ extra_kwargs["moe_flex_dispatcher_backend"] = "deepep"
+ extra_kwargs["moe_router_dtype"] = "fp32"
+ elif moe_dispatcher_type == "hybridep":
+ if not is_hybrid_ep_available():
+ pytest.skip("Hybrid EP is not available")
+ extra_kwargs["moe_token_dispatcher_type"] = "flex"
+ extra_kwargs["moe_flex_dispatcher_backend"] = "hybridep"
+ else:
+ extra_kwargs["moe_token_dispatcher_type"] = moe_dispatcher_type
+
+ loss_list_ref = self._run_test_helper(4, "none", None, 3, **extra_kwargs)
+ for cuda_graph_scope in [
+ [CudaGraphScope.attn],
+ [CudaGraphScope.attn, CudaGraphScope.moe_router],
+ [CudaGraphScope.attn, CudaGraphScope.moe_router, CudaGraphScope.moe_preprocess],
+ ]:
+ cuda_graph_warmup_steps = 3
+ loss_list = self._run_test_helper(
+ 4,
+ "transformer_engine",
+ cuda_graph_scope,
+ cuda_graph_warmup_steps,
+ ep_overlap=True,
+ **extra_kwargs,
+ )
+ assert len(loss_list) == len(loss_list_ref)
+ for i in range(len(loss_list)):
+ assert torch.equal(
+ loss_list[i].mean(), loss_list_ref[i].mean()
+ ), f"scope={cuda_graph_scope}, i={i},loss_list={loss_list[i]}, loss_list_ref={loss_list_ref[i]}"
+ print(f"[DEBUG] Pass {cuda_graph_scope}")
diff --git a/tests/unit_tests/a2a_overlap/test_schedule_chunk_1f1b.py b/tests/unit_tests/a2a_overlap/test_schedule_chunk_1f1b.py
index 81e61a3404a..6c59dd3f9e3 100644
--- a/tests/unit_tests/a2a_overlap/test_schedule_chunk_1f1b.py
+++ b/tests/unit_tests/a2a_overlap/test_schedule_chunk_1f1b.py
@@ -23,7 +23,7 @@
from tests.unit_tests.test_utilities import Utils
-def build_model(config):
+def build_model(config, use_padding_mask=False):
seq_len = 32
max_seq_len = 300
# ids = random.sample([i for i in range(max_seq_len)], seq_len)
@@ -39,6 +39,12 @@ def build_model(config):
"attention_mask": torch.ones((1, 1, seq_len, seq_len), dtype=bool).cuda(),
}
+ # Optionally add padding_mask with same shape as input_ids
+ if use_padding_mask:
+ padding_mask = torch.zeros((1, seq_len), dtype=torch.bool).cuda()
+ padding_mask[0, -8:] = True
+ data["padding_mask"] = padding_mask
+
# build layer spec
transformer_layer_spec = get_gpt_decoder_block_spec(config=config, use_transformer_engine=True)
mtp_block_spec = get_gpt_mtp_block_spec(config, transformer_layer_spec.layer_specs[-1], True)
@@ -48,7 +54,7 @@ def build_model(config):
config=config,
transformer_layer_spec=transformer_layer_spec,
mtp_block_spec=mtp_block_spec,
- vocab_size=100,
+ vocab_size=128,
pre_process=True,
post_process=True,
max_sequence_length=max_seq_len,
@@ -174,3 +180,109 @@ def test_1f1b_schedule_model_chunk(self, mtp_layers, dispatcher_type, fp8_flag,
gpt_models[i] = None
gc.collect()
torch.cuda.empty_cache()
+
+ @pytest.mark.skipif(not is_te_min_version("1.9.0.dev0"), reason="Requires TE >= 1.9.0.dev0")
+ @pytest.mark.parametrize("dispatcher_type", get_valid_token_dispatcher_types())
+ @pytest.mark.parametrize("layers", [[2, 1], [1, 1]])
+ @pytest.mark.parametrize("tp_size", [1, 2, 4, 8])
+ def test_1f1b_schedule_model_chunk_with_padding_mask(self, dispatcher_type, layers, tp_size):
+ """
+ Verifies all-to-all overlap optimization with padding_mask produces
+ the same results as the reference implementation with various TP/EP/CP combinations.
+ """
+ # Re-initialize model parallel with the specified configuration
+ Utils.destroy_model_parallel()
+ Utils.initialize_model_parallel(
+ tensor_model_parallel_size=tp_size,
+ pipeline_model_parallel_size=1,
+ expert_model_parallel_size=4,
+ expert_tensor_parallel_size=1,
+ )
+ set_streams()
+
+ microbatches = 1
+
+ gpt_models = []
+ schedule_plans = []
+ ref_captures = []
+ datas = []
+
+ # create TransformerConfig
+ extra_kwargs = {
+ "moe_token_dispatcher_type": dispatcher_type,
+ "tensor_model_parallel_size": tp_size,
+ "sequence_parallel": tp_size > 1,
+ }
+ if dispatcher_type == "flex":
+ extra_kwargs["moe_flex_dispatcher_backend"] = "deepep"
+ extra_kwargs["moe_router_dtype"] = "fp32"
+ with deterministic_mode():
+ for layer_num in layers:
+ output_tensors = []
+ # build config
+ config = get_test_config(num_layers=layer_num, extra_kwargs=extra_kwargs)
+ # build model with padding_mask
+ gpt_model, schedule_plan, data = build_model(config, use_padding_mask=True)
+ gpt_model.cuda()
+ gpt_models.append(gpt_model)
+ datas.append(data)
+ schedule_plans.append(schedule_plan)
+
+ # run reference
+ for _ in range(microbatches):
+ loss = gpt_model.forward(**data)
+ loss = float16_to_fp32(loss)
+ loss.backward(torch.ones_like(loss))
+ output_tensors.append(loss)
+
+ capture = {"outputs": output_tensors}
+ for name, param in gpt_model.named_parameters():
+ capture[name] = param.grad
+ ref_captures.append(capture)
+ gpt_model.zero_grad()
+ assert gpt_models[0].embedding is not None
+ assert gpt_models[1].embedding is not None
+ # run a2a overlap
+ capture_0 = {"outputs": []}
+ capture_1 = {"outputs": []}
+ a2a_captures = [capture_0, capture_1]
+ for i in range(microbatches):
+ # 1st forward
+ if i > 0:
+ assert (
+ schedule_plans[0].pre_process is None
+ ), "pre_process should be released after backward"
+ schedule_plans[0] = gpt_models[0].build_schedule_plan(**datas[0])
+ schedule_plans[1] = gpt_models[1].build_schedule_plan(**datas[1])
+ f_input_0 = TransformerModelChunkSchedulePlan.run(schedule_plans[0], None)
+ capture_0["outputs"].append(f_input_0)
+ # overlap
+ f_input_1 = TransformerModelChunkSchedulePlan.run(
+ schedule_plans[1], schedule_plans[0], b_grad=torch.ones_like(f_input_0)
+ )
+ capture_1["outputs"].append(f_input_1)
+ # last backward
+ TransformerModelChunkSchedulePlan.run(
+ None, schedule_plans[1], b_grad=torch.ones_like(f_input_1)
+ )
+ for i in range(len(gpt_models)):
+ for name, param in gpt_models[i].named_parameters():
+ a2a_captures[i][name] = param.grad
+
+ # compare results
+ for i in range(len(ref_captures)):
+ comp_res = compare_captures(ref_captures[i], a2a_captures[i], True, True)
+ assert comp_res[0], f"[rank {torch.distributed.get_rank()}] {comp_res[1]}"
+
+ # release resources is necessary, otherwise later testcases will oom
+ for i in range(len(schedule_plans)):
+ schedule_plans[i] = None
+ ref_captures[i] = None
+ a2a_captures[i] = None
+ for k in datas[i]:
+ datas[i][k] = None
+ datas[i] = None
+ gpt_models[i].zero_grad()
+ gpt_models[i] = None
+ gc.collect()
+ torch.cuda.empty_cache()
diff --git a/tests/unit_tests/a2a_overlap/test_schedule_layer_1f1b.py b/tests/unit_tests/a2a_overlap/test_schedule_layer_1f1b.py
index 3ebffb810e5..c6c4a75af99 100644
--- a/tests/unit_tests/a2a_overlap/test_schedule_layer_1f1b.py
+++ b/tests/unit_tests/a2a_overlap/test_schedule_layer_1f1b.py
@@ -306,7 +306,59 @@ def test_transformer_layer_overlap_shared_expert(self):
"moe_shared_expert_intermediate_size": 512,
}
overlap_config = get_test_config(extra_kwargs=extra_kwargs)
- extra_kwargs["moe_shared_expert_overlap"] = True
+ extra_kwargs["moe_shared_expert_overlap"] = False
+ ref_config = get_test_config(extra_kwargs=extra_kwargs)
+ microbatches = 4
+ with deterministic_mode():
+ transformer_layer_spec = get_gpt_decoder_block_spec(
+ config=ref_config, use_transformer_engine=True
+ )
+ gpt_model = GPTModel(
+ config=ref_config,
+ transformer_layer_spec=transformer_layer_spec,
+ vocab_size=100,
+ pre_process=True,
+ post_process=True,
+ max_sequence_length=300,
+ )
+
+ params = reset_model(gpt_model)
+ input_tensors = [build_data() for _ in range(microbatches)]
+
+ fp8_context = get_fp8_context(ref_config, 0) if ref_config.fp8 else nullcontext()
+ with fp8_context:
+ capture_ref = run_transformer_layer_ref_with_capture(
+ gpt_model, input_tensors, microbatches
+ )
+ del gpt_model
+
+ gpt_model = GPTModel(
+ config=overlap_config,
+ transformer_layer_spec=transformer_layer_spec,
+ vocab_size=100,
+ pre_process=True,
+ post_process=True,
+ max_sequence_length=300,
+ )
+ reset_model(gpt_model, params)
+ capture_a2a_overlap = run_transformer_layer_a2a_overlap_with_capture(
+ gpt_model, input_tensors, microbatches
+ )
+ comp_res = compare_captures(capture_ref, capture_a2a_overlap, True)
+ assert comp_res[0], f"[rank {torch.distributed.get_rank()}] {comp_res[1]}"
+
+ @pytest.mark.skipif(not is_te_min_version("1.9.0.dev0"), reason="Requires TE >= 1.9.0.dev0")
+ def test_transformer_layer_overlap_early_attn_memory_release(self):
+ """
+ Verifies all-to-all overlap optimization in transformer layer with early attn memory release
+ produces the same results as the reference implementation.
+ """
+ extra_kwargs = {
+ "moe_token_dispatcher_type": "alltoall",
+ "ep_overlap_early_attn_memory_release": True,
+ "overlap_moe_expert_parallel_comm": True,
+ }
+ overlap_config = get_test_config(extra_kwargs=extra_kwargs)
ref_config = get_test_config(extra_kwargs=extra_kwargs)
microbatches = 4
with deterministic_mode():
@@ -450,8 +502,8 @@ def test_mtp_layer_overlap(self, dispatcher_type, fp8_flag):
position_ids = torch.tensor(data, dtype=torch.int64).repeat((1, 1)).cuda()
attention_mask = torch.ones((1, 1, seq_len, seq_len), dtype=bool).cuda()
# get rotary pos emb
- _, rotary_pos_emb, rotary_pos_cos, rotary_pos_sin, _ = gpt_model._preprocess(
- input_ids, position_ids
+ _, rotary_pos_emb, rotary_pos_cos, rotary_pos_sin, _, _padding_mask = (
+ gpt_model._preprocess(input_ids, position_ids)
)
# reset model
params = reset_model(gpt_model)
diff --git a/tests/unit_tests/a2a_overlap/utils.py b/tests/unit_tests/a2a_overlap/utils.py
index 7db4256a849..a52843956df 100644
--- a/tests/unit_tests/a2a_overlap/utils.py
+++ b/tests/unit_tests/a2a_overlap/utils.py
@@ -1,3 +1,4 @@
+# Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
import os
from contextlib import contextmanager
from dataclasses import dataclass
diff --git a/tests/unit_tests/dist_checkpointing/test_layer_wise_optimizer.py b/tests/unit_tests/dist_checkpointing/test_layer_wise_optimizer.py
new file mode 100644
index 00000000000..54e12b9e7b7
--- /dev/null
+++ b/tests/unit_tests/dist_checkpointing/test_layer_wise_optimizer.py
@@ -0,0 +1,610 @@
+# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved.
+
+from copy import deepcopy
+from functools import partial
+from unittest import mock
+
+import pytest
+import torch
+
+from megatron.core import parallel_state
+from megatron.core.dist_checkpointing import load, save
+from megatron.core.dist_checkpointing.dict_utils import nested_values
+from megatron.core.models.gpt.gpt_layer_specs import get_gpt_decoder_block_spec
+from megatron.core.models.gpt.gpt_layer_specs import (
+ get_gpt_layer_with_transformer_engine_spec as gpt_te_spec,
+)
+from megatron.core.models.gpt.gpt_model import GPTModel
+from megatron.core.optimizer import ChainedOptimizer
+from megatron.core.optimizer.layer_wise_optimizer import LayerWiseDistributedOptimizer
+from megatron.core.process_groups_config import ProcessGroupCollection
+from megatron.core.tensor_parallel import model_parallel_cuda_manual_seed
+from megatron.core.transformer import MLATransformerConfig, TransformerConfig
+from megatron.core.utils import get_pg_size
+from megatron.training.arguments import parse_args
+from megatron.training.checkpointing import load_checkpoint, save_checkpoint
+from tests.unit_tests.dist_checkpointing import (
+ TempNamedDir,
+ init_basic_mock_args,
+ init_checkpointing_mock_args,
+ initialize_gpt_model,
+ setup_model_and_optimizer,
+ setup_moe_model_and_optimizer,
+)
+from tests.unit_tests.test_utilities import Utils
+
+
+def check_equal(input_1, input_2):
+ """Check if two inputs are equal, used for checking checkpointing."""
+ if isinstance(input_1, dict) and isinstance(input_2, dict):
+ assert input_1.keys() == input_2.keys()
+ for key in input_1.keys():
+ check_equal(input_1[key], input_2[key])
+ elif isinstance(input_1, list) and isinstance(input_2, list):
+ assert len(input_1) == len(input_2)
+ for i in range(len(input_1)):
+ check_equal(input_1[i], input_2[i])
+ elif isinstance(input_1, torch.Tensor) and isinstance(input_2, torch.Tensor):
+ assert torch.all(input_1 == input_2), f"Input 1: {input_1} != Input 2: {input_2}"
+ elif type(input_1) != type(input_2):
+ assert False, f"Input 1 type: {type(input_1)} != Input 2 type: {type(input_2)}"
+ else:
+ assert input_1 == input_2, f"Input 1: {input_1} != Input 2: {input_2}"
+
+
+def initialize_real_model(
+ seed,
+ pre_process,
+ post_process,
+ vp_stage=None,
+ is_moe=False,
+ is_mla=False,
+ virtual_pipeline_model_parallel_size=None,
+ **config_kwargs,
+):
+ # These kwargs are passed through training.get_model for model construction,
+ # but are not part of TransformerConfig; strip them before building config.
+ config_kwargs.pop("pg_collection", None)
+ config_kwargs.pop("config", None)
+
+ torch.manual_seed(seed)
+ model_parallel_cuda_manual_seed(seed)
+
+ default_config_kwargs = dict(
+ num_layers=6,
+ hidden_size=16,
+ num_attention_heads=8,
+ use_cpu_initialization=True,
+ pipeline_dtype=torch.bfloat16,
+ bf16=True,
+ virtual_pipeline_model_parallel_size=virtual_pipeline_model_parallel_size,
+ )
+ if is_moe:
+ default_config_kwargs["moe_ffn_hidden_size"] = 128
+ default_config_kwargs["num_moe_experts"] = 4
+ default_config_kwargs["add_bias_linear"] = False
+ # Pop unused fields
+ config_kwargs.pop("use_sp")
+ config_kwargs.pop("use_te")
+ config_kwargs.pop("use_grouped_mlp")
+ config_kwargs.pop("use_glu")
+ if is_mla:
+ default_config_kwargs["multi_latent_attention"] = True
+ default_config_kwargs["q_lora_rank"] = 96
+ default_config_kwargs["kv_lora_rank"] = 512
+ default_config_kwargs["qk_head_dim"] = 64
+ default_config_kwargs["qk_pos_emb_head_dim"] = 32
+ default_config_kwargs["v_head_dim"] = 64
+ default_config_kwargs.update(**config_kwargs)
+ config_cls = MLATransformerConfig if is_mla else TransformerConfig
+ transformer_config = config_cls(**default_config_kwargs)
+
+ if is_moe:
+ layer_spec = get_gpt_decoder_block_spec(
+ transformer_config, use_transformer_engine=True, vp_stage=vp_stage
+ )
+ else:
+ layer_spec = gpt_te_spec(multi_latent_attention=is_mla)
+ this_model = GPTModel(
+ config=transformer_config,
+ transformer_layer_spec=layer_spec,
+ vocab_size=128,
+ max_sequence_length=4,
+ pre_process=pre_process,
+ post_process=post_process,
+ vp_stage=vp_stage,
+ )
+
+ return this_model
+
+
+def load_checkpoint_no_arg_checks(*args, **kwargs):
+ with mock.patch('megatron.training.checkpointing.check_checkpoint_args'):
+ with mock.patch('megatron.training.checkpointing.update_num_microbatches'):
+ return load_checkpoint(*args, **kwargs)
+
+
+class TestLayerWiseOptimizer:
+ """Tests for LayerWiseDistributedOptimizer functionality."""
+
+ def setup_method(self, method):
+ pass
+
+ def teardown_method(self, method):
+ Utils.destroy_model_parallel()
+
+ def test_parameter_sharding(self):
+ """Test that parameters are correctly sharded across DP ranks."""
+ Utils.initialize_model_parallel(1, 1)
+
+ model, optimizer = setup_model_and_optimizer(
+ seed=2,
+ tp=1,
+ pp=1,
+ bf16=True,
+ dist_opt=False,
+ initialize_fn=initialize_gpt_model,
+ optimizer='dist_muon',
+ )
+
+ # Check if optimizer is ChainedOptimizer (expected for standard setup)
+ if isinstance(optimizer, ChainedOptimizer):
+ total_params = sum(
+ len(group['params'])
+ for opt in optimizer.chained_optimizers
+ for group in opt.param_groups
+ )
+ assert total_params > 0, "No parameters found in optimizer"
+
+ @pytest.mark.parametrize('tp', [1, 2, 4])
+ @pytest.mark.parametrize('pp', [1, 2, 4])
+ def test_broadcast_params(self, tp, pp):
+ """Test that parameter broadcasting works correctly across DP ranks."""
+ if tp * pp > 8:
+ pytest.skip(f"TP*PP > 8 is larger than world size")
+
+ Utils.initialize_model_parallel(tp, pp)
+
+ model, optimizer = setup_model_and_optimizer(
+ seed=2,
+ tp=tp,
+ pp=pp,
+ bf16=True,
+ dist_opt=False,
+ initialize_fn=initialize_gpt_model,
+ optimizer='dist_muon',
+ )
+
+ # If this is a LayerWiseDistributedOptimizer, test broadcast
+ if isinstance(optimizer, LayerWiseDistributedOptimizer):
+ # Store original param values
+ original_params = {}
+ for name, param in model[0].named_parameters():
+ original_params[name] = param.data.clone()
+
+ # Call broadcast (should be idempotent if no updates)
+ optimizer.broadcast_params()
+
+ # Check params are unchanged after broadcast without step
+ for name, param in model[0].named_parameters():
+ assert torch.allclose(param.data, original_params[name])
+
+ # TODO(@boxiangw): add PP=4 back and fix the test
+ @pytest.mark.parametrize('tp', [1, 2, 4])
+ @pytest.mark.parametrize('pp', [1, 2])
+ @pytest.mark.parametrize('bf16', [True, False])
+ def test_layer_wise_optimizer_save_load(self, tmp_path_dist_ckpt, tp, pp, bf16):
+ """Test save/load of LayerWiseDistributedOptimizer checkpoints."""
+ if tp * pp > 8:
+ pytest.skip(f"TP*PP > 8 is larger than world size")
+
+ Utils.initialize_model_parallel(tp, pp)
+
+ with TempNamedDir(
+ tmp_path_dist_ckpt / 'test_layer_wise_optimizer_A', sync=True
+ ) as ckpt_dir_A:
+ with TempNamedDir(
+ tmp_path_dist_ckpt / 'test_layer_wise_optimizer_B', sync=True
+ ) as ckpt_dir_B:
+ # Create model and optimizer A
+ model_A, optimizer_A = setup_model_and_optimizer(
+ seed=2,
+ tp=tp,
+ pp=pp,
+ bf16=bf16,
+ dist_opt=False,
+ initialize_fn=initialize_gpt_model,
+ optimizer='dist_muon',
+ )
+
+ # Save checkpoint A
+ model_sharded_sd_A = model_A[0].sharded_state_dict()
+ optim_sd_A = optimizer_A.sharded_state_dict(model_sharded_sd_A)
+ save(optim_sd_A, ckpt_dir_A)
+
+ # Create model and optimizer B with different seed
+ model_B, optimizer_B = setup_model_and_optimizer(
+ seed=3,
+ tp=tp,
+ pp=pp,
+ bf16=bf16,
+ dist_opt=False,
+ initialize_fn=initialize_gpt_model,
+ optimizer='dist_muon',
+ )
+
+ # Load checkpoint A into optimizer B
+ model_sharded_sd_B = model_B[0].sharded_state_dict()
+ load_sharded_sd = optimizer_B.sharded_state_dict(
+ model_sharded_sd_B, is_loading=True
+ )
+ state_dict = load(load_sharded_sd, ckpt_dir_A)
+ optimizer_B.load_state_dict(state_dict)
+
+ # Save as checkpoint B
+ optim_sd_B = optimizer_B.sharded_state_dict(model_sharded_sd_B)
+ save(optim_sd_B, ckpt_dir_B)
+
+ Utils.destroy_model_parallel()
+
+ # Compare checkpoints
+ Utils.initialize_model_parallel(1, 1)
+ from megatron.core.dist_checkpointing import load_plain_tensors
+
+ plain_sd_A = load_plain_tensors(ckpt_dir_A)
+ plain_sd_B = load_plain_tensors(ckpt_dir_B)
+
+ check_equal(plain_sd_A, plain_sd_B)
+
+ @pytest.mark.parametrize('tp', [1, 2, 4])
+ @pytest.mark.parametrize('pp', [1, 2, 4])
+ def test_layer_wise_optimizer_grad_norm(self, tp, pp):
+ """Test that gradient norm calculation works correctly."""
+ if tp * pp > 8:
+ pytest.skip(f"TP*PP > 8 is larger than world size")
+
+ Utils.initialize_model_parallel(tp, pp)
+
+ model, optimizer = setup_model_and_optimizer(
+ seed=2,
+ tp=tp,
+ pp=pp,
+ bf16=True,
+ dist_opt=False,
+ initialize_fn=initialize_gpt_model,
+ optimizer='dist_muon',
+ )
+
+ # Create dummy gradients
+ for param in model[0].parameters():
+ if param.requires_grad:
+ param.grad = torch.randn_like(param.data)
+
+ # Test grad norm calculation
+ if isinstance(optimizer, LayerWiseDistributedOptimizer):
+ grad_norm = optimizer.get_grad_norm()
+ assert grad_norm is not None
+ assert grad_norm >= 0
+
+ @pytest.mark.parametrize('tp', [1, 2, 4])
+ @pytest.mark.parametrize('pp', [1, 2, 4])
+ def test_layer_wise_optimizer_count_zeros(self, tp, pp):
+ """Test that zero counting in gradients works correctly."""
+ if tp * pp > 8:
+ pytest.skip(f"TP*PP > 8 is larger than world size")
+
+ Utils.initialize_model_parallel(tp, pp)
+
+ model, optimizer = setup_model_and_optimizer(
+ seed=2,
+ tp=tp,
+ pp=pp,
+ bf16=True,
+ dist_opt=False,
+ initialize_fn=initialize_gpt_model,
+ optimizer='dist_muon',
+ )
+
+ # Create dummy gradients with some zeros
+ for param in model[0].parameters():
+ if param.requires_grad:
+ grad = torch.randn_like(param.data)
+ # Set some values to zero
+ grad[grad < 0] = 0
+ param.grad = grad
+
+ # Test zero counting
+ if isinstance(optimizer, LayerWiseDistributedOptimizer):
+ num_zeros = optimizer.count_zeros()
+ assert num_zeros >= 0
+
+ # TODO(@boxiangw): add PP=4 back and fix the test
+ @pytest.mark.parametrize('src_tp', [1, 2, 4])
+ @pytest.mark.parametrize('src_pp', [1, 2])
+ @pytest.mark.parametrize('dest_tp', [1, 2, 4])
+ @pytest.mark.parametrize('dest_pp', [1, 2])
+ def test_layer_wise_optimizer_resharding(
+ self, tmp_path_dist_ckpt, src_tp, src_pp, dest_tp, dest_pp
+ ):
+ """Test resharding of LayerWiseDistributedOptimizer across different TP/PP."""
+ if src_tp * src_pp > 8:
+ pytest.skip(f"SRC_TP*SRC_PP > 8 is larger than world size")
+
+ if dest_tp * dest_pp > 8:
+ pytest.skip(f"DEST_TP*DEST_PP > 8 is larger than world size")
+
+ Utils.initialize_model_parallel(src_tp, src_pp)
+
+ with TempNamedDir(
+ tmp_path_dist_ckpt / 'test_layer_wise_resharding_A', sync=True
+ ) as ckpt_dir:
+ # Create and save with source configuration
+ model_A, optimizer_A = setup_model_and_optimizer(
+ seed=2,
+ tp=src_tp,
+ pp=src_pp,
+ bf16=True,
+ dist_opt=False,
+ initialize_fn=initialize_gpt_model,
+ optimizer='dist_muon',
+ )
+
+ model_sharded_sd = model_A[0].sharded_state_dict()
+ optim_sd = optimizer_A.sharded_state_dict(model_sharded_sd)
+ save(optim_sd, ckpt_dir)
+
+ Utils.destroy_model_parallel()
+
+ # Load with destination configuration
+ Utils.initialize_model_parallel(dest_tp, dest_pp)
+ model_B, optimizer_B = setup_model_and_optimizer(
+ seed=3,
+ tp=dest_tp,
+ pp=dest_pp,
+ bf16=True,
+ dist_opt=False,
+ initialize_fn=initialize_gpt_model,
+ optimizer='dist_muon',
+ )
+
+ model_sharded_sd = model_B[0].sharded_state_dict()
+ load_sharded_sd = optimizer_B.sharded_state_dict(model_sharded_sd, is_loading=True)
+
+ state_dict = load(load_sharded_sd, ckpt_dir)
+ optimizer_B.load_state_dict(state_dict)
+
+ @pytest.mark.parametrize('tp', [1, 2, 4])
+ @pytest.mark.parametrize('pp', [1, 2, 4])
+ @pytest.mark.parametrize('ep', [1, 2, 4])
+ def test_layer_wise_optimizer_with_moe(self, tmp_path_dist_ckpt, tp, pp, ep):
+ """Test LayerWiseDistributedOptimizer with MoE models."""
+ if tp * pp * ep > 8:
+ pytest.skip(f"TP*PP > 8 is larger than world size")
+
+ Utils.initialize_model_parallel(
+ tensor_model_parallel_size=tp,
+ pipeline_model_parallel_size=pp,
+ expert_model_parallel_size=ep,
+ )
+
+ with TempNamedDir(tmp_path_dist_ckpt / 'test_layer_wise_moe', sync=True) as ckpt_dir:
+ # Create MoE model with optimizer
+ model, optimizer = setup_moe_model_and_optimizer(
+ seed=2, tp=tp, pp=pp, ep=ep, bf16=True, dist_opt=False, optimizer='dist_muon'
+ )
+
+ # Test that optimizer handles expert parallel parameters
+ if isinstance(optimizer, LayerWiseDistributedOptimizer):
+ # Check that expt_dp_params_list exists if EP > 1
+ if ep > 1:
+ assert hasattr(optimizer, 'expt_dp_params_list')
+
+ # Test save/load
+ model_sharded_sd = model[0].sharded_state_dict()
+ optim_sd = optimizer.sharded_state_dict(model_sharded_sd)
+ save(optim_sd, ckpt_dir)
+
+ # Create new optimizer and load
+ model_new, optimizer_new = setup_moe_model_and_optimizer(
+ seed=3, tp=tp, pp=pp, ep=ep, bf16=True, dist_opt=False, optimizer='dist_muon'
+ )
+
+ model_sharded_sd = model_new[0].sharded_state_dict()
+ load_sharded_sd = optimizer_new.sharded_state_dict(model_sharded_sd, is_loading=True)
+ state_dict = load(load_sharded_sd, ckpt_dir)
+ optimizer_new.load_state_dict(state_dict)
+
+ def test_layer_wise_optimizer_replica_id(self):
+ """Test that LayerWiseDistributedOptimizer sets replica_id correctly."""
+ Utils.initialize_model_parallel(2, 2)
+
+ model, optimizer = setup_model_and_optimizer(
+ seed=2,
+ tp=2,
+ pp=2,
+ bf16=True,
+ dist_opt=False,
+ initialize_fn=initialize_gpt_model,
+ optimizer='dist_muon',
+ )
+
+ if isinstance(optimizer, LayerWiseDistributedOptimizer):
+ model_sharded_sd = model[0].sharded_state_dict()
+ optim_sd = optimizer.sharded_state_dict(model_sharded_sd)
+
+ # Extract ShardedTensors and check replica_id
+ from megatron.core.dist_checkpointing import ShardedTensor
+
+ for sh_base in nested_values(optim_sd):
+ if isinstance(sh_base, ShardedTensor):
+ # Check that replica_id has been modified
+ assert len(sh_base.replica_id) == 3
+ # DP component should be 0 for layer-wise optimizer
+ assert sh_base.replica_id[2] == 0
+
+ @pytest.mark.parametrize('dp_size', [1, 2, 4])
+ def test_layer_wise_optimizer_dp_sizes(self, dp_size):
+ """Test LayerWiseDistributedOptimizer with different DP sizes."""
+ # Use TP to vary DP size while keeping world size constant
+ world_size = 8
+ if world_size % dp_size != 0:
+ pytest.skip(f"World size {world_size} not divisible by DP size {dp_size}")
+
+ pp = 1
+ tp = world_size // dp_size
+
+ if tp == 0:
+ pytest.skip(f"Invalid TP configuration")
+
+ Utils.initialize_model_parallel(tp, pp)
+
+ model, optimizer = setup_model_and_optimizer(
+ seed=2,
+ tp=tp,
+ pp=1,
+ bf16=True,
+ dist_opt=False,
+ initialize_fn=initialize_gpt_model,
+ optimizer='dist_muon',
+ )
+
+ if isinstance(optimizer, LayerWiseDistributedOptimizer):
+ # Check parameter sharding based on DP size
+ pg_collection = ProcessGroupCollection.use_mpu_process_groups()
+ pg_collection.dp_cp = parallel_state.get_data_parallel_group(with_context_parallel=True)
+
+ actual_dp_size = get_pg_size(pg_collection.dp_cp)
+
+ if actual_dp_size > 1:
+ assert optimizer.dp_cp_params_list is not None
+ assert len(optimizer.dp_cp_params_list) == actual_dp_size
+ else:
+ assert optimizer.dp_cp_params_list is None
+
+ def test_layer_wise_optimizer_step(self):
+ """Test that step function works and returns expected values."""
+ Utils.initialize_model_parallel(2, 2)
+
+ model, optimizer = setup_model_and_optimizer(
+ seed=2,
+ tp=2,
+ pp=2,
+ bf16=True,
+ dist_opt=False,
+ initialize_fn=initialize_gpt_model,
+ optimizer='dist_muon',
+ )
+
+ # Create dummy gradients
+ for param in model[0].parameters():
+ if param.requires_grad:
+ param.grad = torch.randn_like(param.data)
+
+ if isinstance(optimizer, LayerWiseDistributedOptimizer):
+ # Perform step
+ update_successful, grad_norm, num_zeros = optimizer.step()
+
+ # Check return values
+ assert isinstance(update_successful, bool)
+ assert grad_norm is None or grad_norm >= 0
+ assert num_zeros is None or num_zeros >= 0
+
+ # TODO(@boxiangw): Add test for loading with different TP/PP sizes
+ @pytest.mark.parametrize("fully_parallel", [True, False])
+ @pytest.mark.parametrize('optimizer_type', ['dist_muon', 'muon'])
+ @pytest.mark.parametrize('tp', [1, 2, 4])
+ @pytest.mark.parametrize('pp', [1, 2])
+ @pytest.mark.parametrize('ep', [1, 2, 4])
+ @pytest.mark.parametrize('is_moe', [True, False])
+ @pytest.mark.parametrize('is_mla', [True, False])
+ def test_optimizer_common_state_dict(
+ self, tmp_path_dist_ckpt, fully_parallel, tp, pp, ep, is_moe, is_mla, optimizer_type
+ ):
+ if tp * pp * ep > 8:
+ pytest.skip(f"TP*PP*EP > 8 is larger than world size")
+
+ if ep > 1 and not is_moe:
+ pytest.skip(f"EP > 1 needs to be used with MoE")
+
+ initialize_fn = partial(initialize_real_model, is_moe=is_moe, is_mla=is_mla)
+
+ # Initialize parallel
+ Utils.initialize_model_parallel(
+ tensor_model_parallel_size=tp,
+ pipeline_model_parallel_size=pp,
+ expert_model_parallel_size=ep,
+ )
+ rank = torch.distributed.get_rank()
+
+ with TempNamedDir(tmp_path_dist_ckpt / 'test_dp_sharding', sync=True) as ckpt_dir:
+ mock_args = parse_args(ignore_unknown_args=True)
+ mock_args.use_distributed_optimizer = False
+ with mock.patch('megatron.training.checkpointing.get_args', new=lambda: mock_args):
+ # Initialize model and optimizer A
+ if is_moe:
+ model, optimizer_A = setup_moe_model_and_optimizer(
+ seed=2,
+ tp=tp,
+ pp=pp,
+ ep=ep,
+ initialize_fn=initialize_fn,
+ dist_opt=False,
+ optimizer=optimizer_type,
+ )
+ else:
+ model, optimizer_A = setup_model_and_optimizer(
+ seed=2,
+ tp=tp,
+ pp=pp,
+ initialize_fn=initialize_fn,
+ dist_opt=False,
+ optimizer=optimizer_type,
+ )
+
+ # Save checkpoint
+ init_checkpointing_mock_args(mock_args, ckpt_dir, fully_parallel=fully_parallel)
+ from megatron.training.training import preprocess_common_state_dict
+
+ save_checkpoint(
+ 10,
+ model,
+ optimizer_A,
+ None,
+ 0,
+ preprocess_common_state_dict_fn=preprocess_common_state_dict,
+ )
+
+ # Get optimizer A param state
+ optim_param_state_A = optimizer_A.state_dict()
+
+ # Initialize model and optimizer B
+ if is_moe:
+ model, optimizer_B = setup_moe_model_and_optimizer(
+ seed=3,
+ tp=tp,
+ pp=pp,
+ ep=ep,
+ initialize_fn=initialize_fn,
+ dist_opt=False,
+ optimizer=optimizer_type,
+ )
+ else:
+ model, optimizer_B = setup_model_and_optimizer(
+ seed=3,
+ tp=tp,
+ pp=pp,
+ initialize_fn=initialize_fn,
+ dist_opt=False,
+ optimizer=optimizer_type,
+ )
+
+ # Load optimizer B from checkpoint
+ load_checkpoint_no_arg_checks(model, optimizer_B, None)
+
+ # Get optimizer B param state
+ optim_param_state_B = optimizer_B.state_dict()
+
+ # Test both param state dicts are equal
+ check_equal(optim_param_state_A, optim_param_state_B)
+
+ Utils.destroy_model_parallel()
diff --git a/tests/unit_tests/dist_checkpointing/utils.py b/tests/unit_tests/dist_checkpointing/utils.py
index b5725df1e14..ddbb78e0a61 100644
--- a/tests/unit_tests/dist_checkpointing/utils.py
+++ b/tests/unit_tests/dist_checkpointing/utils.py
@@ -12,6 +12,7 @@
get_gpt_layer_with_transformer_engine_spec,
)
from megatron.core.optimizer import OptimizerConfig, get_megatron_optimizer
+from megatron.core.optimizer.muon import get_megatron_muon_optimizer
from megatron.core.tensor_parallel import model_parallel_cuda_manual_seed
from megatron.core.transformer import TransformerConfig
from megatron.training.arguments import parse_args
@@ -169,14 +170,13 @@ def init_checkpointing_mock_args(args, ckpt_dir, fully_parallel=False):
def setup_model_and_optimizer(
- seed,
- tp,
- pp,
- initialize_fn=initialize_gpt_model,
- bf16=True,
- dist_opt=True,
- data_parallel_sharding_strategy="optim_grads_params",
+ seed, tp, pp, initialize_fn=initialize_gpt_model, bf16=True, dist_opt=True, optimizer='adam'
):
+ if 'muon' in optimizer and dist_opt:
+ raise ValueError(
+ "Layer-wise distributed optimizer with Muon is not supported with distributed optimizer."
+ )
+
mock_args = parse_args(ignore_unknown_args=True)
with mock.patch('megatron.training.training.get_args', new=lambda: mock_args):
init_basic_mock_args(mock_args, tp, pp, bf16=bf16)
@@ -195,17 +195,39 @@ def setup_model_and_optimizer(
bf16=bf16,
params_dtype=torch.bfloat16 if bf16 else torch.float,
use_distributed_optimizer=dist_opt,
+ optimizer=optimizer,
)
- optimizer = get_megatron_optimizer(config, model)
+
+ if 'muon' in optimizer:
+ # Use layer-wise distributed optimizer with Muon
+ optimizer_type = optimizer
+ optimizer = get_megatron_muon_optimizer(config, model)
+ else:
+ optimizer_type = optimizer
+ optimizer = get_megatron_optimizer(config, model)
torch.manual_seed(seed + 1)
model_parallel_cuda_manual_seed(seed + 1)
- for group in optimizer.optimizer.param_groups:
- for p in group['params']:
- if len(optimizer.optimizer.state[p]) == 0:
- optimizer.optimizer.state[p]['exp_avg'] = torch.rand_like(p.data)
- optimizer.optimizer.state[p]['exp_avg_sq'] = torch.rand_like(p.data)
+ if not 'muon' in optimizer_type:
+ for group in optimizer.optimizer.param_groups:
+ for p in group['params']:
+ if len(optimizer.optimizer.state[p]) == 0:
+ optimizer.optimizer.state[p]['exp_avg'] = torch.rand_like(p.data)
+ optimizer.optimizer.state[p]['exp_avg_sq'] = torch.rand_like(p.data)
+ else:
+ for group in optimizer.chained_optimizers[0].param_groups:
+ for p in group['params']:
+ if len(optimizer.chained_optimizers[0].state[p]) == 0:
+ optimizer.chained_optimizers[0].state[p]['momentum_buffer'] = torch.rand_like(
+ p.data
+ )
+
+ for group in optimizer.chained_optimizers[1].param_groups:
+ for p in group['params']:
+ if len(optimizer.chained_optimizers[1].state[p]) == 0:
+ optimizer.chained_optimizers[1].state[p]['exp_avg'] = torch.rand_like(p.data)
+ optimizer.chained_optimizers[1].state[p]['exp_avg_sq'] = torch.rand_like(p.data)
optimizer.reload_model_params()
@@ -248,7 +270,12 @@ def setup_moe_model_and_optimizer(
use_te=False,
use_grouped_mlp=False,
use_glu=False,
+ optimizer='adam',
):
+ if 'muon' in optimizer and dist_opt:
+ raise ValueError(
+ "Layer-wise distributed optimizer with Muon is not supported with distributed optimizer."
+ )
mock_args = parse_args(ignore_unknown_args=True)
with mock.patch('megatron.training.training.get_args', new=lambda: mock_args):
init_basic_mock_args(mock_args, tp, pp, bf16=bf16)
@@ -272,18 +299,39 @@ def setup_moe_model_and_optimizer(
bf16=bf16,
params_dtype=torch.bfloat16 if bf16 else torch.float,
use_distributed_optimizer=dist_opt,
+ optimizer=optimizer,
)
- optimizer = get_megatron_optimizer(config, model)
+
+ if 'muon' in optimizer:
+ optimizer_type = optimizer
+ optimizer = get_megatron_muon_optimizer(config, model)
+ else:
+ optimizer_type = optimizer
+ optimizer = get_megatron_optimizer(config, model)
torch.manual_seed(seed + 1)
model_parallel_cuda_manual_seed(seed + 1)
- for opt in optimizer.chained_optimizers:
- for group in opt.param_groups:
+ if not 'muon' in optimizer_type:
+ for opt in optimizer.chained_optimizers:
+ for group in opt.param_groups:
+ for p in group['params']:
+ if len(opt.state[p]) == 0:
+ opt.state[p]['exp_avg'] = torch.rand_like(p.data)
+ opt.state[p]['exp_avg_sq'] = torch.rand_like(p.data)
+ else:
+ for group in optimizer.chained_optimizers[0].param_groups:
+ for p in group['params']:
+ if len(optimizer.chained_optimizers[0].state[p]) == 0:
+ optimizer.chained_optimizers[0].state[p]['momentum_buffer'] = torch.rand_like(
+ p.data
+ )
+
+ for group in optimizer.chained_optimizers[1].param_groups:
for p in group['params']:
- if len(opt.state[p]) == 0:
- opt.state[p]['exp_avg'] = torch.rand_like(p.data)
- opt.state[p]['exp_avg_sq'] = torch.rand_like(p.data)
+ if len(optimizer.chained_optimizers[1].state[p]) == 0:
+ optimizer.chained_optimizers[1].state[p]['exp_avg'] = torch.rand_like(p.data)
+ optimizer.chained_optimizers[1].state[p]['exp_avg_sq'] = torch.rand_like(p.data)
optimizer.reload_model_params()
diff --git a/tests/unit_tests/distributed/test_mcore_fully_sharded_data_parallel.py b/tests/unit_tests/distributed/test_mcore_fully_sharded_data_parallel.py
index 3b41daf58ef..3f0cce4e40b 100644
--- a/tests/unit_tests/distributed/test_mcore_fully_sharded_data_parallel.py
+++ b/tests/unit_tests/distributed/test_mcore_fully_sharded_data_parallel.py
@@ -220,13 +220,16 @@ def train_step(model, optimizer, inputs):
# Testing fsdp_double_buffer with and without nccl_ub
@pytest.mark.parametrize(
- ("dp_size", "nccl_ub", "fsdp_double_buffer"), [(8, False, True), (8, True, True)]
+ ("dp_size", "nccl_ub", "fsdp_double_buffer", "fsdp_manual_registration"),
+ [(8, False, True, False), (8, True, True, False), (8, True, True, True)],
)
- def test_fsdp_user_buffer_registration(self, dp_size, nccl_ub, fsdp_double_buffer):
+ def test_fsdp_user_buffer_registration(
+ self, dp_size, nccl_ub, fsdp_double_buffer, fsdp_manual_registration
+ ):
"""Test that FSDP works correctly with user buffer registration.
This test compares the training results of the baseline fsdp with the target fsdp config.
- Baseline fsdp: nccl_ub=False, fsdp_double_buffer=False
- Target fsdp: nccl_ub=[True, False], fsdp_double_buffer=[True, False]
+ Baseline fsdp: nccl_ub=False, fsdp_double_buffer=False, fsdp_manual_registration=False
+ Target fsdp: nccl_ub=[True, False], fsdp_double_buffer=[True, False], fsdp_manual_registration=[True, False]
"""
if not is_torch_min_version("2.4.0"):
pytest.skip("Megatron FSDP requires torch >= 2.4.0")
@@ -264,6 +267,7 @@ def test_fsdp_user_buffer_registration(self, dp_size, nccl_ub, fsdp_double_buffe
use_megatron_fsdp=True,
nccl_ub=False,
fsdp_double_buffer=False,
+ fsdp_manual_registration=False,
)
# Setup FSDP config - target fsdp config
@@ -275,6 +279,7 @@ def test_fsdp_user_buffer_registration(self, dp_size, nccl_ub, fsdp_double_buffe
use_megatron_fsdp=True,
nccl_ub=nccl_ub,
fsdp_double_buffer=fsdp_double_buffer,
+ fsdp_manual_registration=fsdp_manual_registration,
)
# Create two identical models
@@ -354,6 +359,13 @@ def train_step(model, optimizer, inputs):
out1, loss1 = train_step(baseline_fsdp_model, optimizer1, input_data)
out2, loss2 = train_step(target_fsdp_model, optimizer2, input_data)
+ # In case of manual registration, we need to manually register the buffer
+ # And proceed one more step to check the results
+ if fsdp_manual_registration:
+ out1, loss1 = train_step(baseline_fsdp_model, optimizer1, input_data)
+ target_fsdp_model.manual_buffer_registration()
+ out2, loss2 = train_step(target_fsdp_model, optimizer2, input_data)
+
testing.assert_close(out1, out2, rtol=0, atol=0)
testing.assert_close(loss1, loss2, rtol=0, atol=0)
diff --git a/tests/unit_tests/fusions/test_fused_linear_cross_entropy.py b/tests/unit_tests/fusions/test_fused_linear_cross_entropy.py
new file mode 100644
index 00000000000..3ac8e7f6200
--- /dev/null
+++ b/tests/unit_tests/fusions/test_fused_linear_cross_entropy.py
@@ -0,0 +1,1509 @@
+# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved.
+
+import contextlib
+import os
+import typing
+from contextlib import ExitStack
+from dataclasses import dataclass
+
+import numpy as np
+import pytest
+import torch
+import torch.distributed as dist
+from torch.utils.data import DataLoader, Dataset
+from torch.utils.data.distributed import DistributedSampler
+
+import megatron.core.parallel_state as ps
+from megatron.core.fusions.fused_linear_cross_entropy import linear_cross_entropy
+from megatron.core.models.gpt.gpt_layer_specs import (
+ get_gpt_decoder_block_spec,
+ get_gpt_mtp_block_spec,
+)
+from megatron.core.models.gpt.gpt_model import GPTModel
+from megatron.training.utils import get_device_arch_version
+from tests.unit_tests.a2a_overlap.utils import (
+ deterministic_mode,
+ get_test_config,
+ get_valid_fp8_flags,
+ get_valid_token_dispatcher_types,
+)
+from tests.unit_tests.test_utilities import Utils
+
+
+# 1. Define a standardized context to hold your distributed info
+@dataclass
+class DistContext:
+ rank: int
+ world_size: int
+ group: dist.ProcessGroup
+ is_chief: bool
+
+
+# 2. Create a module-scoped fixture
+# This runs ONE time per file, no matter how many test classes you have.
+@pytest.fixture(scope="module")
+def distributed_context():
+ # --- PRE-CHECK ---
+ if "WORLD_SIZE" not in os.environ or int(os.environ["WORLD_SIZE"]) < 2:
+ pytest.skip("Requires torchrun with multiple GPUs (WORLD_SIZE >= 2)")
+
+ # --- SETUP ---
+ is_external_init = dist.is_initialized()
+
+ if not is_external_init:
+ # Initialize only if not already done (e.g., by another test runner)
+ dist.init_process_group(
+ backend="nccl",
+ init_method="env://",
+ world_size=int(os.environ["WORLD_SIZE"]),
+ rank=int(os.environ["RANK"]),
+ )
+
+ # Set device immediately to avoid cross-device pollution
+ local_rank = int(os.environ.get("LOCAL_RANK", os.environ["RANK"]))
+ device = torch.device(f"cuda:{local_rank}")
+ torch.cuda.set_device(device)
+
+ # Gather context data
+ rank = dist.get_rank()
+ world_size = dist.get_world_size()
+ group = dist.group.WORLD
+
+ print(f"[INFO]: Initialized Rank: {rank} / {world_size}")
+
+ context = DistContext(rank=rank, world_size=world_size, group=group, is_chief=(rank == 0))
+
+ # Yield control to the tests
+ yield context
+
+ # --- TEARDOWN ---
+ # Only destroy if we were the ones who initialized it
+ if not is_external_init:
+ dist.destroy_process_group()
+
+
+class MockDataset(Dataset):
+ """
+ Mock dataset for torchtitan GPT training tests
+ Generates synthetic tokenized sequences on-the-fly
+ """
+
+ def __init__(
+ self,
+ num_samples=10000,
+ micro_batch_size=4,
+ sequence_length=2048,
+ vocab_size=128256,
+ seed=42,
+ ):
+ """
+ Initialize mock dataset
+
+ Args:
+ num_samples: Total number of samples
+ sequence_length: Length of each sequence
+ vocab_size: Size of vocabulary
+ seed: Random seed for reproducibility
+ """
+ self.num_samples = num_samples
+ self.micro_batch_size = micro_batch_size
+ self.sequence_length = sequence_length
+ self.vocab_size = vocab_size
+ self.seed = seed
+
+ # Set numpy seed for deterministic generation
+ np.random.seed(seed)
+
+ def __len__(self):
+ return self.num_samples
+
+ def __getitem__(self, idx):
+ """
+ Generate a single training sample
+
+ Returns:
+ dict with 'tokens' and 'labels'
+ """
+ # Use idx as seed for reproducible but varied samples
+ rng = np.random.RandomState(self.seed + idx)
+
+ # Generate random token sequence
+ tokens = rng.randint(0, self.vocab_size, size=self.sequence_length, dtype=np.int64)
+
+ # Labels are tokens shifted by 1 (next token prediction)
+ labels = rng.randint(0, self.vocab_size, size=self.sequence_length, dtype=np.int64)
+
+ return {
+ 'input_ids': torch.from_numpy(tokens.copy()),
+ 'labels': torch.from_numpy(labels.copy()),
+ "attention_mask": torch.ones(
+ (1, self.sequence_length, self.sequence_length), dtype=bool
+ ),
+ }
+
+
+def build_model(config):
+ max_seq_len = 300
+
+ # build layer spec
+ transformer_layer_spec = get_gpt_decoder_block_spec(config=config, use_transformer_engine=True)
+ mtp_block_spec = get_gpt_mtp_block_spec(config, transformer_layer_spec.layer_specs[-1], True)
+
+ # build model
+ gpt_model = GPTModel(
+ config=config,
+ transformer_layer_spec=transformer_layer_spec,
+ mtp_block_spec=mtp_block_spec,
+ vocab_size=100,
+ pre_process=True,
+ post_process=True,
+ max_sequence_length=max_seq_len,
+ )
+ return gpt_model
+
+
+# Define a reusable context manager
+@contextlib.contextmanager
+def init_model_parallel(tp=1, pp=1, ep=1):
+ try:
+ Utils.initialize_model_parallel(
+ tensor_model_parallel_size=tp,
+ pipeline_model_parallel_size=pp,
+ expert_model_parallel_size=ep,
+ )
+ yield
+ finally:
+ Utils.destroy_model_parallel()
+
+
+def init_gpt_dataloader(
+ dp_group, micro_batch_size=1, vocab_size=50257, sequence_length=128, batch_size=8
+):
+ dataset = MockDataset(
+ num_samples=1000,
+ micro_batch_size=micro_batch_size,
+ sequence_length=sequence_length,
+ vocab_size=vocab_size,
+ seed=42,
+ )
+ sampler = DistributedSampler(dataset, num_replicas=dp_group.size(), rank=dp_group.rank())
+ dataloader = DataLoader(dataset, batch_size=batch_size, sampler=sampler)
+ return dataloader
+
+
+# skip it for good
+@pytest.mark.skipif(
+ ("WORLD_SIZE" not in os.environ or int(os.environ["WORLD_SIZE"]) < 2) or True,
+ reason="Requires torchrun with multiple GPUs",
+)
+class TestFusedLinearCrossEntropyOnGptModel:
+ @pytest.mark.parametrize("fp8_flag", get_valid_fp8_flags())
+ @pytest.mark.parametrize("mtp_layers", [0, 1])
+ @pytest.mark.parametrize("dispatcher_type", get_valid_token_dispatcher_types())
+ @pytest.mark.parametrize("layer_num", [2])
+ def test_gpt_model(self, mtp_layers, dispatcher_type, fp8_flag, layer_num):
+ with ExitStack() as stack:
+ gpu_count = torch.cuda.device_count()
+ tp = min(2, gpu_count)
+ ep = gpu_count // tp
+ stack.enter_context(init_model_parallel(tp=tp, ep=ep))
+ stack.enter_context(deterministic_mode())
+
+ # create TransformerConfig
+ extra_kwargs = {
+ "moe_token_dispatcher_type": dispatcher_type,
+ "sequence_parallel": tp > 1,
+ "tensor_model_parallel_size": tp,
+ }
+ if dispatcher_type == "flex":
+ extra_kwargs["moe_enable_deepep"] = True
+ extra_kwargs["moe_router_dtype"] = "fp32"
+ if fp8_flag is not None:
+ extra_kwargs["fp8"] = fp8_flag[0]
+ extra_kwargs["fp8_recipe"] = fp8_flag[1]
+ if mtp_layers > 0:
+ extra_kwargs["mtp_num_layers"] = mtp_layers
+ extra_kwargs["mtp_loss_scaling_factor"] = 1.1
+
+ # build config
+ config = get_test_config(num_layers=layer_num, extra_kwargs=extra_kwargs)
+ config.expert_model_parallel_size = ep
+
+ # build model
+ gpt_model = build_model(config)
+ gpt_model.cuda()
+
+ dataloader = init_gpt_dataloader(
+ ps.get_data_parallel_group(),
+ vocab_size=gpt_model.vocab_size,
+ micro_batch_size=1,
+ sequence_length=gpt_model.max_sequence_length,
+ batch_size=4,
+ )
+ # for batch in dataloder:
+ for batch in dataloader:
+ batch["position_ids"] = torch.arange(
+ gpt_model.max_sequence_length, dtype=torch.int64
+ )
+ batch = {k: v.cuda() for k, v in batch.items()}
+ gpt_model.zero_grad()
+ output = gpt_model(**batch)
+ loss = output.sum()
+ loss.backward()
+
+
+@pytest.mark.skipif(
+ "WORLD_SIZE" in os.environ and os.environ["WORLD_SIZE"] != "1", reason="Requires single GPU"
+)
+@pytest.mark.skipif(get_device_arch_version() != 10, reason="Requires GPU architecture = 10")
+class TestFusedLinearCrossEntropyDataParallel:
+ def cleanup(self):
+ torch.cuda.empty_cache()
+ torch.cuda.reset_peak_memory_stats()
+ import gc
+
+ gc.collect()
+ torch.cuda.synchronize()
+
+ @staticmethod
+ def torch_linear_cross_entropy(
+ hidden: torch.Tensor,
+ weight: torch.Tensor,
+ labels: torch.Tensor,
+ reduction: str,
+ ignore_index: int,
+ ):
+ # NOTE: need to convert to fp32 to fp32 accumulation,
+ # thus assure accuracy
+ logits = hidden.to(torch.float32) @ weight.T.to(torch.float32)
+ logprobs = torch.nn.functional.cross_entropy(
+ logits.view(-1, logits.shape[-1]),
+ labels.view(-1),
+ reduction=reduction,
+ ignore_index=ignore_index,
+ )
+ return logprobs.to(torch.float32)
+
+ @staticmethod
+ def get_problems():
+ return [
+ (80, 125, 64),
+ (80, 152064, 64),
+ (1024, 152064, 4096),
+ (4096, 152063, 8192),
+ ((1, 4096), 152064, 8192),
+ ((2, 4096), 152064, 8192),
+ ]
+
+ @staticmethod
+ def get_ignore_index():
+ return [-100, 4]
+
+ def test_kernel_launch(self):
+ """
+ Check if the compiled kernel can be
+ launched with different problem sizes
+ """
+ self.cleanup()
+
+ num_tokens = [15, 26, 128, 513, 2048, 8192]
+ vocab_size = 152064
+ dim = 4096
+ dtype = torch.bfloat16
+ reduction = "mean"
+ ignore_index = -100
+
+ weight = torch.randn(vocab_size, dim, dtype=dtype, device="cuda").requires_grad_()
+ for num_token in num_tokens:
+ hidden = torch.randn(num_token, dim, dtype=dtype, device="cuda").requires_grad_()
+ labels = torch.randint(0, vocab_size, (num_token,), dtype=torch.long, device="cuda")
+
+ logprobs = linear_cross_entropy(
+ hidden, weight, labels, reduction=reduction, ignore_index=ignore_index
+ )
+ assert not torch.isnan(logprobs).any()
+
+ gLogprobs = torch.randn_like(logprobs)
+ (d_hidden, d_weight) = torch.autograd.grad(
+ (logprobs,), (hidden, weight), (gLogprobs,), retain_graph=False
+ )
+ assert not torch.isnan(d_hidden).any()
+ assert not torch.isnan(d_weight).any()
+
+ @pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float16])
+ @pytest.mark.parametrize("problem", get_problems())
+ @pytest.mark.parametrize("reduction", ["none", "mean", "sum"])
+ @pytest.mark.parametrize("ignore_index", get_ignore_index())
+ def test_correctness(self, dtype, problem, reduction, ignore_index):
+ num_tokens, vocabsize, dim = problem
+ hidden_shape = (num_tokens, dim) if isinstance(num_tokens, int) else (*num_tokens, dim)
+ labels_shape = (num_tokens,) if isinstance(num_tokens, int) else num_tokens
+
+ hidden = (
+ torch.empty(hidden_shape, dtype=dtype, device="cuda")
+ .uniform_(-0.1, 0.1)
+ .requires_grad_()
+ )
+ weight = (
+ torch.empty((vocabsize, dim), dtype=dtype, device="cuda")
+ .uniform_(-0.1, 0.1)
+ .requires_grad_()
+ )
+ labels = torch.randint(0, vocabsize, labels_shape, dtype=torch.long, device="cuda")
+ if ignore_index >= 0 and ignore_index < vocabsize:
+ pad_labels = torch.nn.functional.pad(labels, (0, 1), value=ignore_index)
+ labels = pad_labels[..., 1:].contiguous()
+
+ # forward
+ torch_logprobs = self.torch_linear_cross_entropy(
+ hidden, weight, labels, reduction=reduction, ignore_index=ignore_index
+ )
+
+ custom_logprobs = linear_cross_entropy(
+ hidden, weight, labels, reduction=reduction, ignore_index=ignore_index
+ )
+
+ torch.testing.assert_close(torch_logprobs, custom_logprobs)
+
+ # backward
+ g_logprobs = torch.empty_like(torch_logprobs).uniform_(-0.1, 0.1)
+
+ (d_torch_hidden, d_torch_weight) = torch.autograd.grad(
+ (torch_logprobs,), (hidden, weight), (g_logprobs,), retain_graph=False
+ )
+
+ (d_custom_hidden, d_custom_weight) = torch.autograd.grad(
+ (custom_logprobs,), (hidden, weight), (g_logprobs,), retain_graph=False
+ )
+
+ torch.testing.assert_close(d_torch_hidden, d_custom_hidden, atol=1e-3, rtol=1e-3)
+ torch.testing.assert_close(d_torch_weight, d_custom_weight, atol=1e-3, rtol=1e-3)
+
+ @pytest.mark.parametrize("problem", [((1, 4096), 129280, 7168)])
+ @pytest.mark.parametrize("dtype", [torch.bfloat16])
+ @pytest.mark.parametrize("reduction", ["mean"])
+ @pytest.mark.parametrize("ignore_index", [-100])
+ def test_performance(self, problem, dtype, reduction, ignore_index):
+ num_tokens, vocabsize, dim = problem
+ hidden_shape = (num_tokens, dim) if isinstance(num_tokens, int) else (*num_tokens, dim)
+ labels_shape = (num_tokens,) if isinstance(num_tokens, int) else num_tokens
+
+ start_event = torch.cuda.Event(enable_timing=True)
+ end_event = torch.cuda.Event(enable_timing=True)
+
+ torch_fwd_latency = list()
+ torch_bwd_latency = list()
+ custom_fwd_latency = list()
+ custom_bwd_latency = list()
+
+ iterations = 5
+ for i in range(iterations):
+ hidden = (
+ torch.empty(hidden_shape, dtype=dtype, device="cuda")
+ .uniform_(-0.1, 0.1)
+ .requires_grad_()
+ )
+ weight = (
+ torch.empty((vocabsize, dim), dtype=dtype, device="cuda")
+ .uniform_(-0.1, 0.1)
+ .requires_grad_()
+ )
+ labels = torch.randint(0, vocabsize, labels_shape, dtype=torch.long, device="cuda")
+ if ignore_index >= 0 and ignore_index < vocabsize:
+ pad_labels = torch.nn.functional.pad(labels, (0, 1), value=ignore_index)
+ labels = pad_labels[..., 1:].contiguous()
+
+ # -------- forward -------- #
+ start_event.record()
+ torch_logprobs = self.torch_linear_cross_entropy(
+ hidden, weight, labels, reduction=reduction, ignore_index=ignore_index
+ )
+ end_event.record()
+ torch.cuda.synchronize()
+ torch_fwd_latency.append(start_event.elapsed_time(end_event))
+
+ start_event.record()
+ custom_logprobs = linear_cross_entropy(
+ hidden, weight, labels, reduction=reduction, ignore_index=ignore_index
+ )
+ end_event.record()
+ torch.cuda.synchronize()
+ custom_fwd_latency.append(start_event.elapsed_time(end_event))
+
+ # -------- backward -------- #
+ g_logprobs = torch.empty_like(torch_logprobs).uniform_(-0.1, 0.1)
+
+ start_event.record()
+ (d_torch_hidden, d_torch_weight) = torch.autograd.grad(
+ (torch_logprobs,), (hidden, weight), (g_logprobs,), retain_graph=False
+ )
+ end_event.record()
+ torch.cuda.synchronize()
+ torch_bwd_latency.append(start_event.elapsed_time(end_event))
+
+ start_event.record()
+ (d_custom_hidden, d_custom_weight) = torch.autograd.grad(
+ (custom_logprobs,), (hidden, weight), (g_logprobs,), retain_graph=False
+ )
+ end_event.record()
+ torch.cuda.synchronize()
+ custom_bwd_latency.append(start_event.elapsed_time(end_event))
+
+ # --- remove first latency due to warmup --- #
+ torch_fwd_latency = torch_fwd_latency[1:]
+ torch_bwd_latency = torch_bwd_latency[1:]
+ custom_fwd_latency = custom_fwd_latency[1:]
+ custom_bwd_latency = custom_bwd_latency[1:]
+
+ print()
+ print(f"[INFO]: On problem {problem}, dtype {dtype}, reduction {reduction}:")
+ print(
+ f"[INFO]: Torch forward latency: {sum(torch_fwd_latency) / len(torch_fwd_latency):.2f} ms"
+ )
+ print(
+ f"[INFO]: Custom forward latency: {sum(custom_fwd_latency) / len(custom_fwd_latency):.2f} ms"
+ )
+ print(
+ f"[INFO]: Torch backward latency: {sum(torch_bwd_latency) / len(torch_bwd_latency):.2f} ms"
+ )
+ print(
+ f"[INFO]: Custom backward latency: {sum(custom_bwd_latency) / len(custom_bwd_latency):.2f} ms"
+ )
+
+ @pytest.mark.parametrize("problem", [((1, 4096), 129280, 7168)])
+ @pytest.mark.parametrize("dtype", [torch.bfloat16])
+ @pytest.mark.parametrize("reduction", ["mean"])
+ @pytest.mark.parametrize("ignore_index", [-100])
+ def test_storage(self, problem, dtype, reduction, ignore_index):
+ num_tokens, vocabsize, dim = problem
+ hidden_shape = (num_tokens, dim) if isinstance(num_tokens, int) else (*num_tokens, dim)
+ labels_shape = (num_tokens,) if isinstance(num_tokens, int) else num_tokens
+ print()
+ print(f"[INFO]: On problem {problem}, dtype {dtype}, reduction {reduction}:")
+
+ def torch_storage():
+ hidden = (
+ torch.empty(hidden_shape, dtype=dtype, device="cuda")
+ .uniform_(-0.1, 0.1)
+ .requires_grad_()
+ )
+ weight = (
+ torch.empty((vocabsize, dim), dtype=dtype, device="cuda")
+ .uniform_(-0.1, 0.1)
+ .requires_grad_()
+ )
+ labels = torch.randint(0, vocabsize, labels_shape, dtype=torch.long, device="cuda")
+ if ignore_index >= 0 and ignore_index < vocabsize:
+ pad_labels = torch.nn.functional.pad(labels, (0, 1), value=ignore_index)
+ labels = pad_labels[..., 1:].contiguous()
+
+ torch.cuda.reset_peak_memory_stats()
+ torch_logprobs = self.torch_linear_cross_entropy(
+ hidden, weight, labels, reduction=reduction, ignore_index=ignore_index
+ )
+ torch.cuda.synchronize()
+ torch_max_memory = torch.cuda.max_memory_allocated() / 1024 / 1024
+ print(f"[INFO]: Torch Forward pass peak memory: {torch_max_memory:.2f} MB")
+
+ torch.cuda.reset_peak_memory_stats()
+ g_logprobs = torch.empty_like(torch_logprobs).uniform_(-0.1, 0.1)
+ (d_torch_hidden, d_torch_weight) = torch.autograd.grad(
+ (torch_logprobs,), (hidden, weight), (g_logprobs,), retain_graph=False
+ )
+ torch.cuda.synchronize()
+ torch_backward_max_memory = torch.cuda.max_memory_allocated() / 1024 / 1024
+ print(f"[INFO]: Torch Backward pass peak memory: {torch_backward_max_memory:.2f} MB")
+
+ def custom_storage():
+ hidden = (
+ torch.empty(hidden_shape, dtype=dtype, device="cuda")
+ .uniform_(-0.1, 0.1)
+ .requires_grad_()
+ )
+ weight = (
+ torch.empty((vocabsize, dim), dtype=dtype, device="cuda")
+ .uniform_(-0.1, 0.1)
+ .requires_grad_()
+ )
+ labels = torch.randint(0, vocabsize, labels_shape, dtype=torch.long, device="cuda")
+ if ignore_index >= 0 and ignore_index < vocabsize:
+ pad_labels = torch.nn.functional.pad(labels, (0, 1), value=ignore_index)
+ labels = pad_labels[..., 1:].contiguous()
+
+ torch.cuda.reset_peak_memory_stats()
+ custom_logprobs = linear_cross_entropy(
+ hidden, weight, labels, reduction=reduction, ignore_index=ignore_index
+ )
+ torch.cuda.synchronize()
+ custom_max_memory = torch.cuda.max_memory_allocated() / 1024 / 1024
+ print(f"[INFO]: Custom Forward pass peak memory: {custom_max_memory:.2f} MB")
+
+ torch.cuda.reset_peak_memory_stats()
+ g_logprobs = torch.empty_like(custom_logprobs).uniform_(-0.1, 0.1)
+ (d_custom_hidden, d_custom_weight) = torch.autograd.grad(
+ (custom_logprobs,), (hidden, weight), (g_logprobs,), retain_graph=False
+ )
+ torch.cuda.synchronize()
+ custom_backward_max_memory = torch.cuda.max_memory_allocated() / 1024 / 1024
+ print(f"[INFO]: Custom Backward pass peak memory: {custom_backward_max_memory:.2f} MB")
+
+ self.cleanup()
+ torch_storage()
+ self.cleanup()
+ custom_storage()
+
+
+@pytest.mark.skipif(
+ ("WORLD_SIZE" not in os.environ or int(os.environ["WORLD_SIZE"]) < 2), # or True,
+ reason="Requires torchrun with multiple GPUs",
+)
+@pytest.mark.skipif(get_device_arch_version() != 10, reason="Requires GPU architecture = 10")
+@pytest.mark.usefixtures("distributed_context")
+class TestFusedLinearCrossEntropyTensorParallel:
+ @pytest.fixture(autouse=True)
+ def setup_attrs(self, distributed_context):
+ """
+ Setup attributes for the test class.
+ """
+ self.tp_group = distributed_context.group
+ self.tp_rank = distributed_context.rank
+ self.tp_world_size = distributed_context.world_size
+ self.is_chief = distributed_context.is_chief
+
+ def cleanup(self):
+ torch.cuda.empty_cache()
+ torch.cuda.reset_peak_memory_stats()
+ import gc
+
+ gc.collect()
+ torch.cuda.synchronize()
+
+ @staticmethod
+ def torch_linear_cross_entropy_single_gpu(
+ hidden: torch.Tensor,
+ weight: torch.Tensor,
+ labels: torch.Tensor,
+ reduction: typing.Optional[str] = "mean",
+ ):
+ logits = hidden.to(torch.float32) @ weight.T.to(torch.float32)
+ logprobs = torch.nn.functional.cross_entropy(
+ logits.view(-1, logits.shape[-1]), labels.view(-1), reduction=reduction
+ )
+ return logprobs.to(torch.float32)
+
+ class TorchLinearCrossEntropy(torch.autograd.Function):
+ @staticmethod
+ def forward(
+ ctx,
+ hidden: torch.Tensor,
+ weight: torch.Tensor,
+ labels: torch.Tensor,
+ tp_group: torch.distributed.ProcessGroup,
+ reduction: typing.Optional[str] = "mean",
+ ):
+ tp_rank = 0 if tp_group is None else torch.distributed.get_rank(tp_group)
+ tp_world_size = 1 if tp_group is None else torch.distributed.get_world_size(tp_group)
+
+ logits = hidden.to(torch.float32) @ weight.T.to(torch.float32)
+
+ whole_logits = torch.empty(
+ (logits.shape[0], logits.shape[-1] * tp_world_size),
+ dtype=logits.dtype,
+ device=logits.device,
+ )
+ whole_logits_ref = [
+ whole_logits[..., i * logits.shape[-1] : (i + 1) * logits.shape[-1]]
+ for i in range(tp_world_size)
+ ]
+ dist.all_gather(whole_logits_ref, logits, group=tp_group)
+
+ logprobs = torch.nn.functional.cross_entropy(
+ whole_logits.view(-1, whole_logits.shape[-1]), labels.view(-1), reduction=reduction
+ )
+
+ # If we don't preserve whole_logits,
+ # we need to re-compute it in the backward pass
+ ctx.save_for_backward(hidden, weight, labels)
+ ctx.tp_group = tp_group
+ ctx.reduction = reduction
+ ctx.tp_rank = tp_rank
+ ctx.tp_world_size = tp_world_size
+
+ return logprobs.to(torch.float32)
+
+ @staticmethod
+ def backward(ctx, g_logprobs: torch.Tensor):
+ hidden, weight, labels = ctx.saved_tensors
+ tp_group = ctx.tp_group
+ reduction = ctx.reduction
+ tp_rank = ctx.tp_rank
+ tp_world_size = ctx.tp_world_size
+
+ num_tokens, dim = hidden.shape
+
+ if reduction == "mean":
+ _g_logprobs = torch.broadcast_to(g_logprobs / num_tokens, (num_tokens,))
+ elif reduction == "sum":
+ _g_logprobs = torch.broadcast_to(g_logprobs, (num_tokens,))
+ else:
+ _g_logprobs = g_logprobs
+
+ # re-compute whole_logits
+ logits = hidden.to(torch.float32) @ weight.T.to(torch.float32)
+ whole_logits = torch.empty(
+ (logits.shape[0], logits.shape[-1] * tp_world_size),
+ dtype=logits.dtype,
+ device=logits.device,
+ )
+ whole_logits_ref = [
+ whole_logits[..., i * logits.shape[-1] : (i + 1) * logits.shape[-1]]
+ for i in range(tp_world_size)
+ ]
+ dist.all_gather(whole_logits_ref, logits, group=tp_group)
+
+ one_hot = torch.zeros_like(whole_logits)
+ one_hot.scatter_(1, labels.view(-1).unsqueeze(-1), 1)
+
+ pd = torch.nn.functional.softmax(whole_logits, dim=-1)
+ d_logits = (pd - one_hot) * _g_logprobs.unsqueeze(-1)
+ d_logits = d_logits.to(hidden.dtype)
+
+ local_size = weight.size(0)
+ local_d_logits = d_logits[:, tp_rank * local_size : (tp_rank + 1) * local_size]
+
+ local_d_hidden = local_d_logits @ weight
+ local_d_weight = local_d_logits.T @ hidden
+
+ dist.all_reduce(local_d_hidden, op=dist.ReduceOp.SUM, group=tp_group)
+
+ return local_d_hidden, local_d_weight, None, None, None
+
+ @pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float16])
+ @pytest.mark.parametrize("reduction", ["mean", "sum", "none"])
+ @pytest.mark.parametrize("problem", [(4096, 129280, 8192)])
+ def test_torch_tp_vs_single_gpu(self, dtype, reduction, problem):
+ num_tokens, vocabsize, dim = problem
+ vocabsize = vocabsize // self.tp_world_size
+
+ hidden = (
+ torch.empty((num_tokens, dim), dtype=dtype, device="cuda")
+ .uniform_(-0.1, 0.1)
+ .requires_grad_()
+ )
+ weight = (
+ torch.empty((vocabsize, dim), dtype=dtype, device="cuda")
+ .uniform_(-0.1, 0.1)
+ .requires_grad_()
+ )
+ labels = torch.randint(0, vocabsize, (num_tokens,), dtype=torch.long, device="cuda")
+
+ # ------------ forward pass ------------ #
+ dist.broadcast(hidden, src=0, group=self.tp_group)
+ dist.broadcast(labels, src=0, group=self.tp_group)
+
+ # single GPU
+ whole_weight = torch.empty(
+ (vocabsize * self.tp_world_size, dim), dtype=dtype, device="cuda"
+ )
+ whole_weight_view = [
+ whole_weight[i * vocabsize : (i + 1) * vocabsize, :] for i in range(self.tp_world_size)
+ ]
+ dist.all_gather(whole_weight_view, weight, group=self.tp_group)
+ whole_weight = whole_weight.clone().requires_grad_()
+ logprobs_single_gpu = self.torch_linear_cross_entropy_single_gpu(
+ hidden, whole_weight, labels, reduction=reduction
+ )
+
+ # TP
+ logprobs_tp = self.TorchLinearCrossEntropy.apply(
+ hidden, weight, labels, self.tp_group, reduction
+ )
+ torch.testing.assert_close(logprobs_single_gpu, logprobs_tp)
+
+ # ------------ backward pass ------------ #
+ g_logprobs = torch.empty_like(logprobs_single_gpu).uniform_(-0.1, 0.1)
+ dist.broadcast(g_logprobs, src=0, group=self.tp_group)
+
+ # single GPU
+ (d_hidden_single_gpu, d_weight_single_gpu) = torch.autograd.grad(
+ (logprobs_single_gpu,), (hidden, whole_weight), (g_logprobs,), retain_graph=False
+ )
+
+ # TP
+ (d_hidden_tp, d_weight_tp) = torch.autograd.grad(
+ (logprobs_tp,), (hidden, weight), (g_logprobs,), retain_graph=False
+ )
+ torch.testing.assert_close(d_hidden_single_gpu, d_hidden_tp, atol=1e-3, rtol=1e-3)
+ local_d_weight_single_gpu = d_weight_single_gpu[
+ self.tp_rank * weight.shape[0] : (self.tp_rank + 1) * weight.shape[0], :
+ ]
+ torch.testing.assert_close(local_d_weight_single_gpu, d_weight_tp, atol=1e-3, rtol=1e-3)
+
+ @staticmethod
+ def get_problems():
+ return [
+ (80, 125, 64),
+ (80, 152064, 64),
+ (1024, 152064, 4096),
+ (4096, 152063, 8192),
+ ((1, 4096), 152064, 8192),
+ ((2, 4096), 152064, 8192),
+ ]
+
+ @pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float16])
+ @pytest.mark.parametrize("reduction", ["mean", "sum", "none"])
+ @pytest.mark.parametrize("problem", get_problems())
+ def test_correctness(self, dtype, reduction, problem):
+ num_tokens, vocabsize, dim = problem
+ hidden_shape = (num_tokens, dim) if isinstance(num_tokens, int) else (*num_tokens, dim)
+ labels_shape = (num_tokens,) if isinstance(num_tokens, int) else num_tokens
+
+ hidden = (
+ torch.empty(hidden_shape, dtype=dtype, device="cuda")
+ .uniform_(-0.1, 0.1)
+ .requires_grad_()
+ )
+ weight = (
+ torch.empty((vocabsize, dim), dtype=dtype, device="cuda")
+ .uniform_(-0.1, 0.1)
+ .requires_grad_()
+ )
+ labels = torch.randint(0, vocabsize, labels_shape, dtype=torch.long, device="cuda")
+
+ # ------ forward pass ------ #
+ dist.broadcast(hidden, src=0, group=self.tp_group)
+ dist.broadcast(labels, src=0, group=self.tp_group)
+
+ torch_logprobs = self.TorchLinearCrossEntropy.apply(
+ hidden.view(-1, dim), weight, labels, self.tp_group, reduction
+ )
+
+ custom_logprobs = linear_cross_entropy(
+ hidden, weight, labels, tp_group=self.tp_group, reduction=reduction
+ )
+
+ torch.testing.assert_close(torch_logprobs, custom_logprobs)
+
+ # ------- backward pass ------- #
+ g_logprobs = torch.empty_like(torch_logprobs).uniform_(-0.1, 0.1)
+ dist.broadcast(g_logprobs, src=0, group=self.tp_group)
+
+ (d_hidden_torch, d_weight_torch) = torch.autograd.grad(
+ (torch_logprobs,), (hidden, weight), (g_logprobs,), retain_graph=False
+ )
+ (d_hidden_custom, d_weight_custom) = torch.autograd.grad(
+ (custom_logprobs,), (hidden, weight), (g_logprobs,), retain_graph=False
+ )
+ torch.testing.assert_close(d_hidden_torch, d_hidden_custom, atol=1e-3, rtol=1e-3)
+ torch.testing.assert_close(d_weight_torch, d_weight_custom, atol=1e-4, rtol=1e-4)
+
+ @pytest.mark.parametrize("problem", [((1, 4096), 129280, 7168)])
+ @pytest.mark.parametrize("dtype", [torch.bfloat16])
+ @pytest.mark.parametrize("reduction", ["mean"])
+ def test_performance(self, problem, dtype, reduction):
+ num_tokens, vocabsize, dim = problem
+ hidden_shape = (num_tokens, dim) if isinstance(num_tokens, int) else (*num_tokens, dim)
+ labels_shape = (num_tokens,) if isinstance(num_tokens, int) else num_tokens
+
+ start_event = torch.cuda.Event(enable_timing=True)
+ end_event = torch.cuda.Event(enable_timing=True)
+
+ torch_fwd_latency = list()
+ torch_bwd_latency = list()
+ custom_fwd_latency = list()
+ custom_bwd_latency = list()
+
+ iterations = 5
+ for i in range(iterations):
+ hidden = (
+ torch.empty(hidden_shape, dtype=dtype, device="cuda")
+ .uniform_(-0.1, 0.1)
+ .requires_grad_()
+ )
+ weight = (
+ torch.empty((vocabsize, dim), dtype=dtype, device="cuda")
+ .uniform_(-0.1, 0.1)
+ .requires_grad_()
+ )
+ labels = torch.randint(0, vocabsize, labels_shape, dtype=torch.long, device="cuda")
+
+ # ------ forward pass ------ #
+ dist.broadcast(hidden, src=0, group=self.tp_group)
+ dist.broadcast(labels, src=0, group=self.tp_group)
+
+ start_event.record()
+ torch_logprobs = self.TorchLinearCrossEntropy.apply(
+ hidden.view(-1, dim), weight, labels, self.tp_group, reduction
+ )
+ end_event.record()
+ torch.cuda.synchronize()
+ torch_fwd_latency.append(start_event.elapsed_time(end_event))
+
+ start_event.record()
+ custom_logprobs = linear_cross_entropy(
+ hidden, weight, labels, tp_group=self.tp_group, reduction=reduction
+ )
+ end_event.record()
+ torch.cuda.synchronize()
+ custom_fwd_latency.append(start_event.elapsed_time(end_event))
+
+ # ------- backward pass ------- #
+ g_logprobs = torch.empty_like(torch_logprobs).uniform_(-0.1, 0.1)
+ dist.broadcast(g_logprobs, src=0, group=self.tp_group)
+
+ start_event.record()
+ (d_hidden_torch, d_weight_torch) = torch.autograd.grad(
+ (torch_logprobs,), (hidden, weight), (g_logprobs,), retain_graph=False
+ )
+ end_event.record()
+ torch.cuda.synchronize()
+ torch_bwd_latency.append(start_event.elapsed_time(end_event))
+
+ start_event.record()
+ (d_hidden_custom, d_weight_custom) = torch.autograd.grad(
+ (custom_logprobs,), (hidden, weight), (g_logprobs,), retain_graph=False
+ )
+ end_event.record()
+ torch.cuda.synchronize()
+ custom_bwd_latency.append(start_event.elapsed_time(end_event))
+
+ # --- remove first latency due to warmup --- #
+ torch_fwd_latency = torch_fwd_latency[1:]
+ torch_bwd_latency = torch_bwd_latency[1:]
+ custom_fwd_latency = custom_fwd_latency[1:]
+ custom_bwd_latency = custom_bwd_latency[1:]
+
+ if self.is_chief:
+ print()
+ print(
+ f"[INFO]: On problem {problem}, dtype {dtype}, reduction {reduction}, TP size {self.tp_world_size}:"
+ )
+ print(
+ f"[INFO]: Torch forward latency: {sum(torch_fwd_latency) / len(torch_fwd_latency):.2f} ms"
+ )
+ print(
+ f"[INFO]: Custom forward latency: {sum(custom_fwd_latency) / len(custom_fwd_latency):.2f} ms"
+ )
+ print(
+ f"[INFO]: Torch backward latency: {sum(torch_bwd_latency) / len(torch_bwd_latency):.2f} ms"
+ )
+ print(
+ f"[INFO]: Custom backward latency: {sum(custom_bwd_latency) / len(custom_bwd_latency):.2f} ms"
+ )
+
+ @pytest.mark.parametrize("problem", [((1, 4096), 129280, 7168)])
+ @pytest.mark.parametrize("dtype", [torch.bfloat16])
+ @pytest.mark.parametrize("reduction", ["mean"])
+ def test_storage(self, problem, dtype, reduction):
+ num_tokens, vocabsize, dim = problem
+ hidden_shape = (num_tokens, dim) if isinstance(num_tokens, int) else (*num_tokens, dim)
+ labels_shape = (num_tokens,) if isinstance(num_tokens, int) else num_tokens
+
+ if self.is_chief:
+ print()
+ print(
+ f"[INFO]: On problem {problem}, dtype {dtype}, reduction {reduction}, TP size {self.tp_world_size}:"
+ )
+
+ def torch_storage():
+ hidden = (
+ torch.empty(hidden_shape, dtype=dtype, device="cuda")
+ .uniform_(-0.1, 0.1)
+ .requires_grad_()
+ )
+ weight = (
+ torch.empty((vocabsize, dim), dtype=dtype, device="cuda")
+ .uniform_(-0.1, 0.1)
+ .requires_grad_()
+ )
+ labels = torch.randint(0, vocabsize, labels_shape, dtype=torch.long, device="cuda")
+
+ dist.broadcast(hidden, src=0, group=self.tp_group)
+ dist.broadcast(labels, src=0, group=self.tp_group)
+
+ torch.cuda.reset_peak_memory_stats()
+ torch_logprobs = self.TorchLinearCrossEntropy.apply(
+ hidden.view(-1, dim), weight, labels, self.tp_group, reduction
+ )
+ torch.cuda.synchronize()
+ torch_max_memory = torch.cuda.max_memory_allocated() / 1024 / 1024
+ if self.is_chief:
+ print(
+ f"[INFO]: On GPU {self.tp_rank}, Torch Forward pass peak memory: {torch_max_memory:.2f} MB"
+ )
+
+ g_logprobs = torch.empty_like(torch_logprobs).uniform_(-0.1, 0.1)
+ dist.broadcast(g_logprobs, src=0, group=self.tp_group)
+
+ torch.cuda.reset_peak_memory_stats()
+ (d_hidden_torch, d_weight_torch) = torch.autograd.grad(
+ (torch_logprobs,), (hidden, weight), (g_logprobs,), retain_graph=False
+ )
+ torch.cuda.synchronize()
+ torch_max_memory = torch.cuda.max_memory_allocated() / 1024 / 1024
+ if self.is_chief:
+ print(
+ f"[INFO]: On GPU {self.tp_rank}, Torch Backward pass peak memory: {torch_max_memory:.2f} MB"
+ )
+
+ def custom_storage():
+ hidden = (
+ torch.empty(hidden_shape, dtype=dtype, device="cuda")
+ .uniform_(-0.1, 0.1)
+ .requires_grad_()
+ )
+ weight = (
+ torch.empty((vocabsize, dim), dtype=dtype, device="cuda")
+ .uniform_(-0.1, 0.1)
+ .requires_grad_()
+ )
+ labels = torch.randint(0, vocabsize, labels_shape, dtype=torch.long, device="cuda")
+
+ dist.broadcast(hidden, src=0, group=self.tp_group)
+ dist.broadcast(labels, src=0, group=self.tp_group)
+
+ torch.cuda.reset_peak_memory_stats()
+ custom_logprobs = linear_cross_entropy(
+ hidden, weight, labels, tp_group=self.tp_group, reduction=reduction
+ )
+ torch.cuda.synchronize()
+ custom_max_memory = torch.cuda.max_memory_allocated() / 1024 / 1024
+ if self.is_chief:
+ print(
+ f"[INFO]: On GPU {self.tp_rank}, Custom Forward pass peak memory: {custom_max_memory:.2f} MB"
+ )
+
+ g_logprobs = torch.empty_like(custom_logprobs).uniform_(-0.1, 0.1)
+ dist.broadcast(g_logprobs, src=0, group=self.tp_group)
+
+ torch.cuda.reset_peak_memory_stats()
+ (d_hidden_custom, d_weight_custom) = torch.autograd.grad(
+ (custom_logprobs,), (hidden, weight), (g_logprobs,), retain_graph=False
+ )
+ torch.cuda.synchronize()
+ custom_max_memory = torch.cuda.max_memory_allocated() / 1024 / 1024
+ if self.is_chief:
+ print(
+ f"[INFO]: On GPU {self.tp_rank}, Custom Backward pass peak memory: {custom_max_memory:.2f} MB"
+ )
+
+ self.cleanup()
+ torch_storage()
+ self.cleanup()
+ custom_storage()
+
+
+@pytest.mark.skipif(
+ "WORLD_SIZE" not in os.environ or int(os.environ["WORLD_SIZE"]) < 2,
+ reason="Requires torchrun with multiple GPUs",
+)
+@pytest.mark.skipif(get_device_arch_version() != 10, reason="Requires GPU architecture = 10")
+@pytest.mark.usefixtures("distributed_context")
+class TestFusedLinearCrossEntropySequenceParallel:
+ @pytest.fixture(autouse=True)
+ def setup_attrs(self, distributed_context):
+ """
+ Setup attributes for the test class.
+ """
+ self.tp_group = distributed_context.group
+ self.tp_rank = distributed_context.rank
+ self.tp_world_size = distributed_context.world_size
+ self.is_chief = distributed_context.is_chief
+
+ @staticmethod
+ def timed_barrier(timeout_s=10):
+ import time
+
+ work = torch.distributed.barrier(async_op=True)
+ t0 = time.time()
+ while not work.is_completed():
+ if time.time() - t0 > timeout_s:
+ exit(1)
+ time.sleep(0.05)
+ work.wait()
+
+ def cleanup(self):
+ torch.cuda.empty_cache()
+ torch.cuda.reset_peak_memory_stats()
+ import gc
+
+ gc.collect()
+ torch.cuda.synchronize()
+
+ @staticmethod
+ def torch_linear_cross_entropy_single_gpu(
+ hidden: torch.Tensor,
+ weight: torch.Tensor,
+ labels: torch.Tensor,
+ reduction: typing.Optional[str] = "mean",
+ ):
+ logits = hidden.to(torch.float32) @ weight.T.to(torch.float32)
+ logprobs = torch.nn.functional.cross_entropy(
+ logits.view(-1, logits.shape[-1]), labels.view(-1), reduction=reduction
+ )
+ return logprobs.to(torch.float32)
+
+ class TorchLinearCrossEntropy(torch.autograd.Function):
+ @staticmethod
+ def forward(
+ ctx,
+ hidden: torch.Tensor,
+ weight: torch.Tensor,
+ labels: torch.Tensor,
+ tp_group: torch.distributed.ProcessGroup,
+ reduction: typing.Optional[str] = "mean",
+ ):
+ tp_rank = 0 if tp_group is None else torch.distributed.get_rank(tp_group)
+ tp_world_size = 1 if tp_group is None else torch.distributed.get_world_size(tp_group)
+
+ whole_hidden = torch.empty(
+ (hidden.shape[0] * tp_world_size, hidden.shape[-1]),
+ dtype=hidden.dtype,
+ device=hidden.device,
+ )
+ dist.all_gather_into_tensor(whole_hidden, hidden, group=tp_group)
+
+ logits = whole_hidden.to(torch.float32) @ weight.T.to(torch.float32)
+
+ whole_logits = torch.empty(
+ (logits.shape[0], logits.shape[-1] * tp_world_size),
+ dtype=logits.dtype,
+ device=logits.device,
+ )
+ whole_logits_ref = [
+ whole_logits[..., i * logits.shape[-1] : (i + 1) * logits.shape[-1]]
+ for i in range(tp_world_size)
+ ]
+ dist.all_gather(whole_logits_ref, logits, group=tp_group)
+
+ logprobs = torch.nn.functional.cross_entropy(
+ whole_logits.view(-1, whole_logits.shape[-1]), labels.view(-1), reduction=reduction
+ )
+
+ # If we don't preserve whole_logits,
+ # we need to re-compute it in the backward pass
+ ctx.save_for_backward(whole_hidden, weight, labels)
+ ctx.tp_group = tp_group
+ ctx.reduction = reduction
+ ctx.tp_rank = tp_rank
+ ctx.tp_world_size = tp_world_size
+
+ return logprobs.to(torch.float32)
+
+ @staticmethod
+ def backward(ctx, g_logprobs: torch.Tensor):
+ whole_hidden, weight, labels = ctx.saved_tensors
+ tp_group = ctx.tp_group
+ reduction = ctx.reduction
+ tp_rank = ctx.tp_rank
+ tp_world_size = ctx.tp_world_size
+
+ num_tokens, dim = whole_hidden.shape
+
+ if reduction == "mean":
+ _g_logprobs = torch.broadcast_to(g_logprobs / num_tokens, (num_tokens,))
+ elif reduction == "sum":
+ _g_logprobs = torch.broadcast_to(g_logprobs, (num_tokens,))
+ else:
+ _g_logprobs = g_logprobs
+
+ # re-compute whole_logits
+ logits = whole_hidden.to(torch.float32) @ weight.T.to(torch.float32)
+ whole_logits = torch.empty(
+ (logits.shape[0], logits.shape[-1] * tp_world_size),
+ dtype=logits.dtype,
+ device=logits.device,
+ )
+ whole_logits_ref = [
+ whole_logits[..., i * logits.shape[-1] : (i + 1) * logits.shape[-1]]
+ for i in range(tp_world_size)
+ ]
+ dist.all_gather(whole_logits_ref, logits, group=tp_group)
+
+ one_hot = torch.zeros_like(whole_logits)
+ one_hot.scatter_(1, labels.view(-1).unsqueeze(-1), 1)
+
+ pd = torch.nn.functional.softmax(whole_logits, dim=-1)
+ d_logits = (pd - one_hot) * _g_logprobs.unsqueeze(-1)
+ d_logits = d_logits.to(whole_hidden.dtype)
+
+ local_size = weight.size(0)
+ local_d_logits = d_logits[:, tp_rank * local_size : (tp_rank + 1) * local_size]
+
+ d_hidden = local_d_logits @ weight
+ local_d_weight = local_d_logits.T @ whole_hidden
+
+ # dist.all_reduce(
+ # local_d_hidden,
+ # op=dist.ReduceOp.SUM,
+ # group=tp_group
+ # )
+
+ # split the local_d_hidden along the sequence length dimension
+ local_num_tokens = num_tokens // tp_world_size
+ # local_d_hidden = local_d_hidden[tp_rank * local_num_tokens : (tp_rank + 1) * local_num_tokens, :]
+
+ local_d_hidden = torch.empty(
+ (local_num_tokens, dim), dtype=weight.dtype, device=weight.device
+ )
+ dist.reduce_scatter_tensor(
+ local_d_hidden, d_hidden, op=dist.ReduceOp.SUM, group=tp_group
+ )
+ return local_d_hidden, local_d_weight, None, None, None
+
+ @pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float16])
+ @pytest.mark.parametrize("reduction", ["mean", "sum", "none"])
+ @pytest.mark.parametrize("problem", [(256, 129280, 8192)])
+ def test_torch_sp_vs_single_gpu(self, dtype, reduction, problem):
+ num_tokens, vocabsize, dim = problem
+ vocabsize = vocabsize // self.tp_world_size
+
+ hidden = (
+ torch.empty((num_tokens, dim), dtype=dtype, device="cuda")
+ .uniform_(-0.1, 0.1)
+ .requires_grad_()
+ )
+ weight = (
+ torch.empty((vocabsize, dim), dtype=dtype, device="cuda")
+ .uniform_(-0.1, 0.1)
+ .requires_grad_()
+ )
+ labels = torch.randint(
+ 0, vocabsize, (num_tokens * self.tp_world_size,), dtype=torch.long, device="cuda"
+ )
+
+ # ------------ forward pass ------------ #
+ dist.broadcast(labels, src=0, group=self.tp_group)
+
+ # single GPU
+ whole_hidden = torch.empty(
+ (num_tokens * self.tp_world_size, dim), dtype=dtype, device="cuda"
+ )
+ dist.all_gather_into_tensor(whole_hidden, hidden, group=self.tp_group)
+ whole_hidden = whole_hidden.clone().requires_grad_()
+
+ whole_weight = torch.empty(
+ (vocabsize * self.tp_world_size, dim), dtype=dtype, device="cuda"
+ )
+ whole_weight_view = [
+ whole_weight[i * vocabsize : (i + 1) * vocabsize, :] for i in range(self.tp_world_size)
+ ]
+ dist.all_gather(whole_weight_view, weight, group=self.tp_group)
+ whole_weight = whole_weight.clone().requires_grad_()
+ logprobs_single_gpu = self.torch_linear_cross_entropy_single_gpu(
+ whole_hidden, whole_weight, labels, reduction=reduction
+ )
+
+ # TP
+ logprobs_tp = self.TorchLinearCrossEntropy.apply(
+ hidden, weight, labels, self.tp_group, reduction
+ )
+ torch.testing.assert_close(logprobs_single_gpu, logprobs_tp)
+
+ # ------------ backward pass ------------ #
+ g_logprobs = torch.empty_like(logprobs_single_gpu).uniform_(-0.1, 0.1)
+ dist.broadcast(g_logprobs, src=0, group=self.tp_group)
+
+ # single GPU
+ (d_hidden_single_gpu, d_weight_single_gpu) = torch.autograd.grad(
+ (logprobs_single_gpu,), (whole_hidden, whole_weight), (g_logprobs,), retain_graph=False
+ )
+
+ # TP
+ (d_hidden_tp, d_weight_tp) = torch.autograd.grad(
+ (logprobs_tp,), (hidden, weight), (g_logprobs,), retain_graph=False
+ )
+
+ local_d_hidden_single_gpu = d_hidden_single_gpu[
+ self.tp_rank * hidden.shape[0] : (self.tp_rank + 1) * hidden.shape[0], :
+ ]
+ torch.testing.assert_close(local_d_hidden_single_gpu, d_hidden_tp, atol=1e-3, rtol=1e-3)
+ local_d_weight_single_gpu = d_weight_single_gpu[
+ self.tp_rank * weight.shape[0] : (self.tp_rank + 1) * weight.shape[0], :
+ ]
+ torch.testing.assert_close(local_d_weight_single_gpu, d_weight_tp, atol=1e-3, rtol=1e-3)
+
+ self.cleanup()
+
+ @staticmethod
+ def get_problems():
+ return [
+ (80, 125, 64),
+ (80, 152064, 64),
+ (1024, 152064, 4096),
+ (4096, 15206, 1024),
+ ((1, 4096), 15206, 1024),
+ ((4, 1024), 15206, 1024),
+ ]
+
+ @pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float16])
+ @pytest.mark.parametrize("reduction", ["mean", "sum", "none"])
+ @pytest.mark.parametrize("problem", get_problems())
+ def test_correctness(self, dtype, reduction, problem):
+ num_tokens, vocabsize, dim = problem
+ hidden_shape = (num_tokens, dim) if isinstance(num_tokens, int) else (*num_tokens, dim)
+ labels_shape = (
+ (num_tokens * self.tp_world_size,)
+ if isinstance(num_tokens, int)
+ else (num_tokens[0] * self.tp_world_size, *num_tokens[1:])
+ )
+
+ hidden = (
+ torch.empty(hidden_shape, dtype=dtype, device="cuda")
+ .uniform_(-0.1, 0.1)
+ .requires_grad_()
+ )
+ weight = (
+ torch.empty((vocabsize, dim), dtype=dtype, device="cuda")
+ .uniform_(-0.1, 0.1)
+ .requires_grad_()
+ )
+ labels = torch.randint(0, vocabsize, labels_shape, dtype=torch.long, device="cuda")
+
+ # ------ forward pass ------ #
+ dist.broadcast(labels, src=0, group=self.tp_group)
+
+ torch_logprobs = self.TorchLinearCrossEntropy.apply(
+ hidden.view(-1, dim), weight, labels, self.tp_group, reduction
+ )
+
+ custom_logprobs = linear_cross_entropy(
+ hidden,
+ weight,
+ labels,
+ tp_group=self.tp_group,
+ reduction=reduction,
+ sequence_parallel=True,
+ )
+
+ torch.testing.assert_close(torch_logprobs, custom_logprobs)
+
+ # ------- backward pass ------- #
+ g_logprobs = torch.empty_like(torch_logprobs).uniform_(-0.1, 0.1)
+ dist.broadcast(g_logprobs, src=0, group=self.tp_group)
+
+ (d_hidden_torch, d_weight_torch) = torch.autograd.grad(
+ (torch_logprobs,), (hidden, weight), (g_logprobs,), retain_graph=False
+ )
+ (d_hidden_custom, d_weight_custom) = torch.autograd.grad(
+ (custom_logprobs,), (hidden, weight), (g_logprobs,), retain_graph=False
+ )
+
+ # in case one GPU failed, and leading to hang
+ torch.testing.assert_close(d_hidden_torch, d_hidden_custom, atol=1e-3, rtol=1e-3)
+ torch.testing.assert_close(d_weight_torch, d_weight_custom, atol=1e-3, rtol=1e-3)
+ self.timed_barrier()
+
+ self.cleanup()
+
+ @pytest.mark.parametrize("problem", [((1, 1024), 129280, 7168)])
+ @pytest.mark.parametrize("dtype", [torch.bfloat16])
+ @pytest.mark.parametrize("reduction", ["mean"])
+ def test_performance(self, problem, dtype, reduction):
+ num_tokens, vocabsize, dim = problem
+ hidden_shape = (num_tokens, dim) if isinstance(num_tokens, int) else (*num_tokens, dim)
+ labels_shape = (
+ (num_tokens * self.tp_world_size,)
+ if isinstance(num_tokens, int)
+ else (num_tokens[0] * self.tp_world_size, *num_tokens[1:])
+ )
+
+ start_event = torch.cuda.Event(enable_timing=True)
+ end_event = torch.cuda.Event(enable_timing=True)
+
+ torch_fwd_latency = list()
+ torch_bwd_latency = list()
+ custom_fwd_latency = list()
+ custom_bwd_latency = list()
+
+ iterations = 5
+ for i in range(iterations):
+ hidden = (
+ torch.empty(hidden_shape, dtype=dtype, device="cuda")
+ .uniform_(-0.1, 0.1)
+ .requires_grad_()
+ )
+ weight = (
+ torch.empty((vocabsize, dim), dtype=dtype, device="cuda")
+ .uniform_(-0.1, 0.1)
+ .requires_grad_()
+ )
+ labels = torch.randint(0, vocabsize, labels_shape, dtype=torch.long, device="cuda")
+
+ # ------ forward pass ------ #
+ dist.broadcast(labels, src=0, group=self.tp_group)
+
+ start_event.record()
+ torch_logprobs = self.TorchLinearCrossEntropy.apply(
+ hidden.view(-1, dim), weight, labels, self.tp_group, reduction
+ )
+ end_event.record()
+ torch.cuda.synchronize()
+ torch_fwd_latency.append(start_event.elapsed_time(end_event))
+
+ start_event.record()
+ custom_logprobs = linear_cross_entropy(
+ hidden,
+ weight,
+ labels,
+ tp_group=self.tp_group,
+ reduction=reduction,
+ sequence_parallel=True,
+ )
+ end_event.record()
+ torch.cuda.synchronize()
+ custom_fwd_latency.append(start_event.elapsed_time(end_event))
+
+ # ------- backward pass ------- #
+ g_logprobs = torch.empty_like(torch_logprobs).uniform_(-0.1, 0.1)
+ dist.broadcast(g_logprobs, src=0, group=self.tp_group)
+
+ start_event.record()
+ (d_hidden_torch, d_weight_torch) = torch.autograd.grad(
+ (torch_logprobs,), (hidden, weight), (g_logprobs,), retain_graph=False
+ )
+ end_event.record()
+ torch.cuda.synchronize()
+ torch_bwd_latency.append(start_event.elapsed_time(end_event))
+
+ start_event.record()
+ (d_hidden_custom, d_weight_custom) = torch.autograd.grad(
+ (custom_logprobs,), (hidden, weight), (g_logprobs,), retain_graph=False
+ )
+ end_event.record()
+ torch.cuda.synchronize()
+ custom_bwd_latency.append(start_event.elapsed_time(end_event))
+
+ # --- remove first latency due to warmup --- #
+ torch_fwd_latency = torch_fwd_latency[1:]
+ torch_bwd_latency = torch_bwd_latency[1:]
+ custom_fwd_latency = custom_fwd_latency[1:]
+ custom_bwd_latency = custom_bwd_latency[1:]
+
+ if self.is_chief:
+ print()
+ print(
+ f"[INFO]: On problem {problem}, dtype {dtype}, reduction {reduction}, TP size {self.tp_world_size}, Sequence Parallel: True:"
+ )
+ print(
+ f"[INFO]: Torch forward latency: {sum(torch_fwd_latency) / len(torch_fwd_latency):.2f} ms"
+ )
+ print(
+ f"[INFO]: Custom forward latency: {sum(custom_fwd_latency) / len(custom_fwd_latency):.2f} ms"
+ )
+ print(
+ f"[INFO]: Torch backward latency: {sum(torch_bwd_latency) / len(torch_bwd_latency):.2f} ms"
+ )
+ print(
+ f"[INFO]: Custom backward latency: {sum(custom_bwd_latency) / len(custom_bwd_latency):.2f} ms"
+ )
+
+ @pytest.mark.parametrize("problem", [((1, 1024), 129280, 7168)])
+ @pytest.mark.parametrize("dtype", [torch.bfloat16])
+ @pytest.mark.parametrize("reduction", ["mean"])
+ def test_storage(self, problem, dtype, reduction):
+ num_tokens, vocabsize, dim = problem
+ hidden_shape = (num_tokens, dim) if isinstance(num_tokens, int) else (*num_tokens, dim)
+ labels_shape = (
+ (num_tokens * self.tp_world_size,)
+ if isinstance(num_tokens, int)
+ else (num_tokens[0] * self.tp_world_size, *num_tokens[1:])
+ )
+
+ if self.is_chief:
+ print()
+ print(
+ f"[INFO]: On problem {problem}, dtype {dtype}, reduction {reduction}, TP size {self.tp_world_size}, Sequence Parallel: True:"
+ )
+
+ def torch_storage():
+ hidden = (
+ torch.empty(hidden_shape, dtype=dtype, device="cuda")
+ .uniform_(-0.1, 0.1)
+ .requires_grad_()
+ )
+ weight = (
+ torch.empty((vocabsize, dim), dtype=dtype, device="cuda")
+ .uniform_(-0.1, 0.1)
+ .requires_grad_()
+ )
+ labels = torch.randint(0, vocabsize, labels_shape, dtype=torch.long, device="cuda")
+
+ dist.broadcast(hidden, src=0, group=self.tp_group)
+ dist.broadcast(labels, src=0, group=self.tp_group)
+
+ torch.cuda.reset_peak_memory_stats()
+ torch_logprobs = self.TorchLinearCrossEntropy.apply(
+ hidden.view(-1, dim), weight, labels, self.tp_group, reduction
+ )
+ torch.cuda.synchronize()
+ torch_max_memory = torch.cuda.max_memory_allocated() / 1024 / 1024
+ if self.is_chief:
+ print(
+ f"[INFO]: On GPU {self.tp_rank}, Torch Forward pass peak memory: {torch_max_memory:.2f} MB"
+ )
+
+ g_logprobs = torch.empty_like(torch_logprobs).uniform_(-0.1, 0.1)
+ dist.broadcast(g_logprobs, src=0, group=self.tp_group)
+
+ torch.cuda.reset_peak_memory_stats()
+ (d_hidden_torch, d_weight_torch) = torch.autograd.grad(
+ (torch_logprobs,), (hidden, weight), (g_logprobs,), retain_graph=False
+ )
+ torch.cuda.synchronize()
+ torch_max_memory = torch.cuda.max_memory_allocated() / 1024 / 1024
+ if self.is_chief:
+ print(
+ f"[INFO]: On GPU {self.tp_rank}, Torch Backward pass peak memory: {torch_max_memory:.2f} MB"
+ )
+
+ def custom_storage():
+ hidden = (
+ torch.empty(hidden_shape, dtype=dtype, device="cuda")
+ .uniform_(-0.1, 0.1)
+ .requires_grad_()
+ )
+ weight = (
+ torch.empty((vocabsize, dim), dtype=dtype, device="cuda")
+ .uniform_(-0.1, 0.1)
+ .requires_grad_()
+ )
+ labels = torch.randint(0, vocabsize, labels_shape, dtype=torch.long, device="cuda")
+
+ dist.broadcast(hidden, src=0, group=self.tp_group)
+ dist.broadcast(labels, src=0, group=self.tp_group)
+
+ torch.cuda.reset_peak_memory_stats()
+ custom_logprobs = linear_cross_entropy(
+ hidden,
+ weight,
+ labels,
+ tp_group=self.tp_group,
+ reduction=reduction,
+ sequence_parallel=True,
+ )
+ torch.cuda.synchronize()
+ custom_max_memory = torch.cuda.max_memory_allocated() / 1024 / 1024
+ if self.is_chief:
+ print(
+ f"[INFO]: On GPU {self.tp_rank}, Custom Forward pass peak memory: {custom_max_memory:.2f} MB"
+ )
+
+ g_logprobs = torch.empty_like(custom_logprobs).uniform_(-0.1, 0.1)
+ dist.broadcast(g_logprobs, src=0, group=self.tp_group)
+
+ torch.cuda.reset_peak_memory_stats()
+ (d_hidden_custom, d_weight_custom) = torch.autograd.grad(
+ (custom_logprobs,), (hidden, weight), (g_logprobs,), retain_graph=False
+ )
+ torch.cuda.synchronize()
+ custom_max_memory = torch.cuda.max_memory_allocated() / 1024 / 1024
+ if self.is_chief:
+ print(
+ f"[INFO]: On GPU {self.tp_rank}, Custom Backward pass peak memory: {custom_max_memory:.2f} MB"
+ )
+
+ self.cleanup()
+ torch_storage()
+ self.cleanup()
+ custom_storage()
diff --git a/tests/unit_tests/inference/engines/test_dynamic_engine.py b/tests/unit_tests/inference/engines/test_dynamic_engine.py
index ef6252094a7..21f6d94dd1a 100644
--- a/tests/unit_tests/inference/engines/test_dynamic_engine.py
+++ b/tests/unit_tests/inference/engines/test_dynamic_engine.py
@@ -45,6 +45,7 @@
from megatron.core.models.mamba.mamba_model import MambaModel
from megatron.core.tensor_parallel.random import model_parallel_cuda_manual_seed
from megatron.core.transformer.cuda_graphs import CudaGraphManager, _CudagraphGlobalRecord
+from megatron.core.transformer.enums import CudaGraphScope
from megatron.core.transformer.transformer_config import TransformerConfig
from megatron.core.utils import (
check_mamba_sequence_packing_support,
@@ -105,7 +106,9 @@ class DynamicEngineTestConfig:
return_log_probs: bool = False
materialize_only_last_token_logits: bool = True
skip_prompt_log_probs: bool = False
- cuda_graph_scope: str = "full_iteration"
+ cuda_graph_scope: List[CudaGraphScope] = field(
+ default_factory=lambda: [CudaGraphScope.full_iteration]
+ )
force_build_cuda_graphs: bool = False
transformer_impl: str = "local"
# If False, do not build cuda graphs in the tests, even if
@@ -528,7 +531,7 @@ def teardown_method(self, method):
)
@pytest.mark.parametrize("model_provider", ["gpt", "mamba"])
@pytest.mark.parametrize("num_cuda_graphs", [None, 1, 4])
- @pytest.mark.parametrize("cuda_graph_scope", ["full", "full_iteration"])
+ @pytest.mark.parametrize("cuda_graph_scope", [[], [CudaGraphScope.full_iteration]])
def test_simple(self, model_provider, num_cuda_graphs, cuda_graph_scope) -> None:
"""Simple test that runs without errors, and validates output."""
skip_if_mamba_sequence_packing_not_available(model_provider)
diff --git a/tests/unit_tests/optimizer/__init__.py b/tests/unit_tests/optimizer/__init__.py
new file mode 100644
index 00000000000..b5dff7b5663
--- /dev/null
+++ b/tests/unit_tests/optimizer/__init__.py
@@ -0,0 +1 @@
+# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
diff --git a/tests/unit_tests/optimizer/test_optimizer_config.py b/tests/unit_tests/optimizer/test_optimizer_config.py
new file mode 100644
index 00000000000..0ecb877ed27
--- /dev/null
+++ b/tests/unit_tests/optimizer/test_optimizer_config.py
@@ -0,0 +1,38 @@
+# Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+import torch
+
+from megatron.core.optimizer.optimizer_config import ParamKey, ParamPredicate
+
+
+def test_paramkey_matches():
+ len_1_predicate = ParamPredicate(name="param_len_1", fn=lambda param: len(param.shape) == 1)
+ endswith_bias = ParamKey(name="*.bias")
+ has_dotbias = ParamKey(name="*.bias*")
+ len_1_param = ParamKey(predicate=len_1_predicate)
+ has_bias_or_len1_param = ParamKey(name="*.bias", predicate=len_1_predicate)
+ has_attr = ParamKey(attr="is_embedding_or_output_parameter")
+
+ assert endswith_bias.matches(torch.nn.Parameter(torch.empty(10, 10)), "interesting.bias")
+ assert not endswith_bias.matches(
+ torch.nn.Parameter(torch.empty(10, 10)), "something.bias.other"
+ )
+ assert has_dotbias.matches(torch.nn.Parameter(torch.empty(10)), "random.biasstuff")
+ assert not has_dotbias.matches(torch.nn.Parameter(torch.empty(10, 10)), "random_bias_name")
+ assert len_1_param.matches(torch.nn.Parameter(torch.empty(10)), "interesting.bias")
+ assert not len_1_param.matches(torch.nn.Parameter(torch.empty(10, 10)), "interesting_bias")
+ assert has_bias_or_len1_param.matches(
+ torch.nn.Parameter(torch.empty(10, 10)), "interesting.bias"
+ )
+ assert has_bias_or_len1_param.matches(torch.nn.Parameter(torch.empty(10)), "interesting_bias")
+ assert not has_bias_or_len1_param.matches(
+ torch.nn.Parameter(torch.empty(10, 10)), "random_bias_name"
+ )
+ p_with_attr = torch.nn.Parameter(torch.empty(10, 10))
+ setattr(p_with_attr, "is_embedding_or_output_parameter", True)
+ assert has_attr.matches(p_with_attr, "interesting.bias")
+ assert not has_attr.matches(torch.nn.Parameter(torch.empty(10, 10)), "interesting.bias")
+
+ # We expect that if the return of the attribute is False, it should not match even if
+ # it has the attribute.
+ setattr(p_with_attr, "is_embedding_or_output_parameter", False)
+ assert not has_attr.matches(p_with_attr, "interesting.bias")
diff --git a/tests/unit_tests/pipeline_parallel/test_fine_grained_activation_offloading.py b/tests/unit_tests/pipeline_parallel/test_fine_grained_activation_offloading.py
new file mode 100644
index 00000000000..558c6934a0c
--- /dev/null
+++ b/tests/unit_tests/pipeline_parallel/test_fine_grained_activation_offloading.py
@@ -0,0 +1,573 @@
+# Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+
+import gc
+import os
+from contextlib import nullcontext
+from typing import Dict, List, Optional, Tuple
+
+import pytest
+import torch
+
+from megatron.core.models.gpt.gpt_layer_specs import get_gpt_layer_with_transformer_engine_spec
+from megatron.core.models.gpt.gpt_model import GPTModel
+from megatron.core.pipeline_parallel.fine_grained_activation_offload import (
+ FineGrainedActivationOffloadingInterface as off_interface,
+)
+from megatron.core.tensor_parallel.random import model_parallel_cuda_manual_seed
+from megatron.core.transformer.enums import AttnBackend
+from megatron.core.transformer.transformer_config import MLATransformerConfig, TransformerConfig
+from megatron.core.utils import is_te_min_version
+from tests.unit_tests.test_utilities import Utils
+
+# Tolerance for memory expectation check (GPU allocator jitter etc).
+EPSILON = 0.30
+EPSILON_A2A = 0.30
+DELTA = 20 # MiB
+
+
+def _reset_cuda_memory() -> None:
+ gc.collect()
+ if torch.cuda.is_available():
+ torch.cuda.empty_cache()
+ torch.cuda.synchronize()
+
+
+def _build_gpt_model(
+ *,
+ seed: int,
+ num_layers: int,
+ hidden_size: int,
+ num_attention_heads: int,
+ vocab_size: int,
+ seq_length: int,
+ num_experts: Optional[int],
+ fine_grained_activation_offloading: bool,
+ offload_modules: Optional[List[str]],
+ min_offloaded_tensor_size: int,
+ is_mla: bool,
+) -> GPTModel:
+ """Build a GPTModel that uses TE-based transformer layer spec."""
+ model_parallel_cuda_manual_seed(seed)
+ torch.manual_seed(seed)
+ ConfigClass = MLATransformerConfig if is_mla else TransformerConfig
+ transformer_config = ConfigClass(
+ num_layers=num_layers,
+ hidden_size=hidden_size,
+ num_attention_heads=num_attention_heads,
+ use_cpu_initialization=True,
+ attention_backend=AttnBackend.unfused,
+ bf16=True,
+ # Recompute
+ recompute_modules=["layernorm", "moe_act"] if num_experts is not None else ["layernorm"],
+ recompute_granularity="selective",
+ # MoE
+ num_moe_experts=num_experts,
+ moe_grouped_gemm=(num_experts is not None),
+ # Fine-grained activation offloading
+ fine_grained_activation_offloading=fine_grained_activation_offloading,
+ offload_modules=offload_modules,
+ min_offloaded_tensor_size=min_offloaded_tensor_size,
+ )
+ gpt_model = GPTModel(
+ config=transformer_config,
+ transformer_layer_spec=get_gpt_layer_with_transformer_engine_spec(
+ num_experts=num_experts,
+ moe_grouped_gemm=num_experts is not None,
+ moe_use_legacy_grouped_gemm=False,
+ multi_latent_attention=is_mla,
+ ),
+ vocab_size=vocab_size,
+ max_sequence_length=seq_length,
+ ).bfloat16()
+ return gpt_model
+
+
+def _make_gpt_inputs(
+ *, seq_length: int, micro_batch_size: int, device: torch.device
+) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
+ data = list(range(seq_length))
+ input_ids = torch.tensor(data, dtype=torch.int64).repeat((micro_batch_size, 1)).to(device)
+ position_ids = torch.tensor(data, dtype=torch.int64).repeat((micro_batch_size, 1)).to(device)
+ attention_mask = torch.ones((micro_batch_size, 1, seq_length, seq_length), dtype=bool).to(
+ device
+ )
+ return input_ids, position_ids, attention_mask
+
+
+def _run_one_iter_and_capture(
+ model: GPTModel,
+ *,
+ input_ids: torch.Tensor,
+ position_ids: torch.Tensor,
+ attention_mask: torch.Tensor,
+ enable_offload_reset: bool,
+) -> Tuple[torch.Tensor, Dict[str, torch.Tensor], int]:
+ """
+ Run a single forward+backward iteration.
+
+ Returns:
+ - logits (CPU float32)
+ - selected grads (CPU float32)
+ - peak_memory_allocated (bytes) during the iteration
+ """
+
+ if enable_offload_reset:
+ off_interface.reset()
+
+ # for p in model.parameters():
+ # if p.grad is not None:
+ # p.grad = None
+
+ torch.cuda.reset_peak_memory_stats()
+ logits = model(input_ids=input_ids, position_ids=position_ids, attention_mask=attention_mask)
+ loss = logits.float().sum()
+ loss.backward()
+ torch.cuda.synchronize()
+ peak_bytes = int(torch.cuda.max_memory_allocated())
+
+ # capture all gradients for correctness
+ grads: Dict[str, torch.Tensor] = {}
+ for name, p in model.named_parameters():
+ grads[name] = p.grad.detach().float().cpu() if p.grad is not None else None
+
+ return logits.detach().float().cpu(), grads, peak_bytes
+
+
+@pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA is required for offloading tests.")
+@pytest.mark.parametrize(
+ "is_moe, is_mla, offload_modules",
+ [
+ # Dense GPT modules
+ (False, True, ["attn_norm"]),
+ (True, False, ["qkv_linear"]),
+ (True, False, ["core_attn"]),
+ # # attn_proj depends on core_attn (validated in TransformerConfig.__post_init__)
+ (True, True, ["core_attn", "attn_proj"]),
+ (True, False, ["mlp_norm"]),
+ (True, False, ["expert_fc1"]),
+ (True, False, ["moe_act"]),
+ ],
+)
+def test_gpt_fine_grained_activation_offloading_correctness_and_memory(
+ is_moe: bool, is_mla: bool, offload_modules: List[str]
+):
+ """
+ Initialize a GPTModel and verify:
+ - forward output correctness under each offload_modules setting
+ - backward gradient correctness (subset)
+ - peak GPU memory is reduced roughly as expected (based on recorded offload bytes)
+ """
+ # setup distributed/model-parallel (same pattern as other UTs)
+ os.environ.pop("NVTE_FUSED_ATTN", None)
+ os.environ.pop("NVTE_FLASH_ATTN", None)
+ os.environ.pop("NVTE_UNFUSED_ATTN", None)
+ # os.environ["NVTE_FLASH_ATTN"] = "1"
+ Utils.initialize_model_parallel(tensor_model_parallel_size=1, pipeline_model_parallel_size=1)
+
+ seed = 123
+ # Choose shapes large enough to make memory deltas stable but still fast.
+ num_experts = 4 if is_moe else None
+ num_layers = 8
+ hidden_size = 2048 if num_experts is None else 1024
+ num_attention_heads = 16 if hidden_size >= 2048 else 8
+ vocab_size = 1024
+ seq_length = 1024
+ micro_batch_size = 2
+ device = torch.device("cuda")
+
+ input_ids, position_ids, attention_mask = _make_gpt_inputs(
+ seq_length=seq_length, micro_batch_size=micro_batch_size, device=device
+ )
+
+ from megatron.core.pipeline_parallel import fine_grained_activation_offload as off
+
+ off_interface.reset_instance()
+
+ try:
+ # 1) Baseline run (no offloading)
+ _reset_cuda_memory()
+ base_model = _build_gpt_model(
+ seed=seed,
+ num_layers=num_layers,
+ hidden_size=hidden_size,
+ num_attention_heads=num_attention_heads,
+ vocab_size=vocab_size,
+ seq_length=seq_length,
+ num_experts=num_experts,
+ fine_grained_activation_offloading=False,
+ offload_modules=None,
+ min_offloaded_tensor_size=1024 * 1024,
+ is_mla=is_mla,
+ ).cuda()
+ base_model.train()
+
+ # Warmup baseline once for allocator stability
+ _run_one_iter_and_capture(
+ base_model,
+ input_ids=input_ids,
+ position_ids=position_ids,
+ attention_mask=attention_mask,
+ enable_offload_reset=False,
+ )
+ _reset_cuda_memory()
+ base_logits, base_grads, base_peak = _run_one_iter_and_capture(
+ base_model,
+ input_ids=input_ids,
+ position_ids=position_ids,
+ attention_mask=attention_mask,
+ enable_offload_reset=False,
+ )
+ # Free baseline model GPU memory before offload path
+ del base_model
+ _reset_cuda_memory()
+
+ # 2) Offload run (warmup to record bytes + steady-state measurement)
+ off_model = _build_gpt_model(
+ seed=seed,
+ num_layers=num_layers,
+ hidden_size=hidden_size,
+ num_attention_heads=num_attention_heads,
+ vocab_size=vocab_size,
+ seq_length=seq_length,
+ num_experts=num_experts,
+ fine_grained_activation_offloading=True,
+ offload_modules=offload_modules,
+ min_offloaded_tensor_size=1024, # force offloading for UT determinism
+ is_mla=is_mla,
+ ).cuda()
+ off_model.train()
+
+ # Warmup 1 iter to populate cached chunks, then reset to finish warmup bookkeeping.
+ _run_one_iter_and_capture(
+ off_model,
+ input_ids=input_ids,
+ position_ids=position_ids,
+ attention_mask=attention_mask,
+ enable_offload_reset=True,
+ )
+ # Reset once more to trigger post_warmup_callback and apply steady-state offload decisions.
+ off_interface.reset()
+
+ from megatron.core.pipeline_parallel.fine_grained_activation_offload import (
+ PipelineOffloadManager,
+ )
+
+ mgr = PipelineOffloadManager.get_instance()
+ expected_offload_bytes = int(
+ sum(mgr.offload_summary_bytes.get(k, 0) for k in offload_modules)
+ )
+ expected_offload_mib = expected_offload_bytes / (1024**2)
+
+ _reset_cuda_memory()
+ off_logits, off_grads, off_peak = _run_one_iter_and_capture(
+ off_model,
+ input_ids=input_ids,
+ position_ids=position_ids,
+ attention_mask=attention_mask,
+ enable_offload_reset=True,
+ )
+ del off_model
+ _reset_cuda_memory()
+
+ # 3) Correctness checks (forward + selected grads)
+ assert torch.allclose(off_logits, base_logits, rtol=1e-3, atol=1e-3)
+ assert set(off_grads.keys()) == set(base_grads.keys())
+ for name, gb in base_grads.items():
+ go = off_grads[name]
+ if gb is None or go is None:
+ assert gb is None and go is None, f"Grad None mismatch for {name}"
+ continue
+ assert torch.allclose(go, gb, rtol=1e-3, atol=1e-3), f"Grad mismatch for {name}"
+
+ # 4) Memory checks (peak allocated over forward+backward)
+ saved_mib = (base_peak - off_peak) / (1024**2)
+ assert saved_mib > 0.0, (
+ f"Expected GPU peak memory reduction for offload_modules={offload_modules}, "
+ f"but got saved={saved_mib:.2f}MiB (base={base_peak/(1024**2):.2f}MiB, "
+ f"off={off_peak/(1024**2):.2f}MiB)"
+ )
+
+ # If expectation is large enough, enforce approximate match.
+ # For tiny expectations, allocator noise may dominate; we only require a positive reduction.
+ if expected_offload_mib >= 2.0:
+ rel_err = abs(saved_mib - expected_offload_mib) / max(expected_offload_mib, 1e-6)
+ abs_err = abs(saved_mib - expected_offload_mib)
+ assert rel_err <= EPSILON and abs_err <= DELTA, (
+ f"Memory saving mismatch for offload_modules={offload_modules}: "
+ f"saved={saved_mib:.2f}MiB expected~={expected_offload_mib:.2f}MiB "
+ f"(rel_err={rel_err:.2f}, abs_err={abs_err:.2f})"
+ )
+ print(
+ f"Rank {torch.distributed.get_rank()}: Saved {saved_mib:.2f}MiB, expected {expected_offload_mib:.2f}MiB"
+ )
+ finally:
+ Utils.destroy_model_parallel()
+
+
+@pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA is required for offloading tests.")
+@pytest.mark.skipif(
+ not is_te_min_version("1.9.0.dev0"),
+ reason="EP A2A overlap requires TE 1.9.0.dev0+ in this repo's tests.",
+)
+@pytest.mark.parametrize(
+ "dispatcher_backend, is_mla, offload_modules",
+ [
+ ("alltoall", True, ["attn_norm"]),
+ ("alltoall", True, ["core_attn"]),
+ ("alltoall", True, ["attn_norm", "core_attn", "attn_proj"]),
+ ("alltoall", True, ["mlp_norm"]),
+ ("alltoall", False, ["expert_fc1"]),
+ ("alltoall", False, ["moe_act"]),
+ ("alltoall", False, ["mlp_norm", "expert_fc1", "moe_act"]),
+ (
+ "alltoall",
+ True,
+ ["attn_norm", "core_attn", "attn_proj", "mlp_norm", "expert_fc1", "moe_act"],
+ ),
+ (
+ "alltoall",
+ False,
+ ["attn_norm", "core_attn", "attn_proj", "mlp_norm", "expert_fc1", "moe_act"],
+ ),
+ ],
+)
+def test_fine_grained_activation_offload_with_ep_a2a_overlap_compatibility(
+ dispatcher_backend: str, is_mla: bool, offload_modules: List[str]
+):
+ """
+ Compatibility test for:
+ - fine-grained activation offloading
+ - EP all-to-all overlap (overlap_moe_expert_parallel_comm)
+ - memory saving roughly matches expected offload bytes (when expectation is large enough)
+
+ The EP A2A overlap initialization pattern is aligned with
+ `tests/unit_tests/a2a_overlap/test_schedule_chunk_1f1b.py`.
+ """
+ from megatron.core.models.common.model_chunk_schedule_plan import (
+ TransformerModelChunkSchedulePlan,
+ )
+ from megatron.core.pipeline_parallel.utils import set_streams
+ from tests.unit_tests.a2a_overlap.utils import deterministic_mode
+
+ # EP overlap requires distributed initialization with EP groups.
+ ep_size = 4
+ if Utils.world_size % ep_size != 0:
+ pytest.skip(
+ f"Skipping: WORLD_SIZE={Utils.world_size} must be divisible by ep_size={ep_size}."
+ )
+
+ seed = 123
+ num_experts = 8 # must be divisible by ep_size
+ if num_experts % ep_size != 0:
+ pytest.skip(
+ f"Skipping: num_moe_experts={num_experts} must be divisible by ep_size={ep_size}."
+ )
+
+ # Small shapes to keep this compatibility test fast.
+ num_layers = 8
+ hidden_size = 1024
+ num_attention_heads = 16
+ vocab_size = 1024
+ seq_length = 1024
+ micro_batch_size = 2
+ device = torch.device("cuda")
+
+ from megatron.core.pipeline_parallel import fine_grained_activation_offload as off
+
+ def _make_schedule_inputs() -> Dict[str, torch.Tensor]:
+ data = list(range(seq_length))
+ input_ids = torch.tensor(data, dtype=torch.int64).repeat((micro_batch_size, 1)).to(device)
+ position_ids = (
+ torch.tensor(data, dtype=torch.int64).repeat((micro_batch_size, 1)).to(device)
+ )
+ attention_mask = torch.ones((micro_batch_size, 1, seq_length, seq_length), dtype=bool).to(
+ device
+ )
+ labels = input_ids.clone()
+ return {
+ "input_ids": input_ids,
+ "labels": labels,
+ "position_ids": position_ids,
+ "attention_mask": attention_mask,
+ }
+
+ def _capture_params(model: torch.nn.Module) -> Dict[str, torch.Tensor]:
+ params: Dict[str, torch.Tensor] = {}
+ for name, p in model.named_parameters():
+ params[name] = p.detach().clone()
+ return params
+
+ def _restore_params(model: torch.nn.Module, params: Dict[str, torch.Tensor]) -> None:
+ for name, p in model.named_parameters():
+ p.data.copy_(params[name])
+
+ def _build_overlap_moe_gpt(
+ *, enable_offload: bool, is_mla: bool, dispatcher_backend: str
+ ) -> GPTModel:
+ model_parallel_cuda_manual_seed(seed)
+ torch.manual_seed(seed)
+ ConfigClass = MLATransformerConfig if is_mla else TransformerConfig
+ transformer_config = ConfigClass(
+ num_layers=num_layers,
+ hidden_size=hidden_size,
+ num_attention_heads=num_attention_heads,
+ use_cpu_initialization=True,
+ attention_backend=AttnBackend.unfused,
+ # Recompute
+ recompute_modules=["layernorm", "moe_act"],
+ recompute_granularity="selective",
+ bf16=True,
+ # MoE + EP overlap
+ num_moe_experts=num_experts,
+ moe_grouped_gemm=True,
+ expert_model_parallel_size=ep_size,
+ moe_token_dispatcher_type="alltoall" if dispatcher_backend == "alltoall" else "flex",
+ moe_flex_dispatcher_backend=dispatcher_backend,
+ moe_router_dtype="fp32" if dispatcher_backend == "hybridep" else "fp64",
+ overlap_moe_expert_parallel_comm=True,
+ delay_wgrad_compute=True,
+ # Fine-grained activation offloading
+ fine_grained_activation_offloading=enable_offload,
+ offload_modules=offload_modules if enable_offload else None,
+ min_offloaded_tensor_size=1024, # force offloading to exercise the code path
+ )
+ return (
+ GPTModel(
+ config=transformer_config,
+ transformer_layer_spec=get_gpt_layer_with_transformer_engine_spec(
+ num_experts=num_experts,
+ moe_grouped_gemm=True,
+ moe_use_legacy_grouped_gemm=False,
+ multi_latent_attention=is_mla,
+ ),
+ vocab_size=vocab_size,
+ max_sequence_length=seq_length,
+ )
+ .bfloat16()
+ .cuda()
+ )
+
+ def _run_schedule_1f1b_two_microbatches(
+ model: GPTModel, *, enable_offload_reset: bool
+ ) -> Tuple[List[torch.Tensor], Dict[str, torch.Tensor], int]:
+ """
+ Run a minimal 1F1B schedule (2 microbatches) using ModelChunkSchedulePlan.run().
+ This is the execution path that exercises EP A2A overlap scheduling.
+ """
+ if enable_offload_reset:
+ off_interface.reset()
+
+ data0 = _make_schedule_inputs()
+ data1 = _make_schedule_inputs()
+ plan0 = model.build_schedule_plan(**data0)
+
+ torch.cuda.reset_peak_memory_stats()
+ out0 = TransformerModelChunkSchedulePlan.run(plan0, None)
+ plan1 = model.build_schedule_plan(**data1)
+ out1 = TransformerModelChunkSchedulePlan.run(plan1, plan0, b_grad=torch.ones_like(out0))
+ TransformerModelChunkSchedulePlan.run(None, plan1, b_grad=torch.ones_like(out1))
+ torch.cuda.synchronize()
+ peak_bytes = int(torch.cuda.max_memory_allocated())
+
+ # capture outputs and grads
+ outputs = [out0.detach().float().cpu(), out1.detach().float().cpu()]
+ grads: Dict[str, torch.Tensor] = {}
+ for name, p in model.named_parameters():
+ grads[name] = p.grad.detach().float().cpu() if p.grad is not None else None
+ return outputs, grads, peak_bytes
+
+ # setup distributed/model-parallel
+ os.environ.pop("NVTE_FUSED_ATTN", None)
+ os.environ.pop("NVTE_FLASH_ATTN", None)
+ os.environ.pop("NVTE_UNFUSED_ATTN", None)
+
+ Utils.initialize_model_parallel(
+ tensor_model_parallel_size=1,
+ pipeline_model_parallel_size=1,
+ expert_model_parallel_size=ep_size,
+ )
+ set_streams()
+
+ off_interface.reset_instance()
+
+ try:
+ with deterministic_mode():
+ # Baseline: EP overlap on, offload off.
+ _reset_cuda_memory()
+ base_model = _build_overlap_moe_gpt(
+ enable_offload=False, is_mla=is_mla, dispatcher_backend=dispatcher_backend
+ )
+ base_model.train()
+ base_params = _capture_params(base_model)
+ # Warmup once for allocator stability / graph caching
+ _run_schedule_1f1b_two_microbatches(base_model, enable_offload_reset=False)
+ _reset_cuda_memory()
+ base_outs, base_grads, base_peak = _run_schedule_1f1b_two_microbatches(
+ base_model, enable_offload_reset=False
+ )
+ del base_model
+ _reset_cuda_memory()
+
+ # Offload: EP overlap on, fine-grained offload on.
+ off_model = _build_overlap_moe_gpt(
+ enable_offload=True, is_mla=is_mla, dispatcher_backend=dispatcher_backend
+ )
+ _restore_params(off_model, base_params)
+ off_model.train()
+ # Warmup once to populate cached chunks, then reset to apply steady-state offload decisions.
+ off_interface.reset()
+ _run_schedule_1f1b_two_microbatches(off_model, enable_offload_reset=False)
+ off_interface.reset()
+ from megatron.core.pipeline_parallel.fine_grained_activation_offload import (
+ PipelineOffloadManager,
+ )
+
+ mgr = PipelineOffloadManager.get_instance()
+ expected_offload_bytes = int(
+ sum(mgr.offload_summary_bytes.get(k, 0) for k in offload_modules)
+ )
+ expected_offload_mib = expected_offload_bytes / (1024**2)
+
+ _reset_cuda_memory()
+ off_outs, off_grads, off_peak = _run_schedule_1f1b_two_microbatches(
+ off_model, enable_offload_reset=True
+ )
+ del off_model
+ _reset_cuda_memory()
+
+ # Correctness (forward outputs + all grads)
+ assert len(off_outs) == len(base_outs) == 2
+ for i in range(2):
+ assert torch.allclose(off_outs[i], base_outs[i], rtol=1e-3, atol=1e-3)
+ assert set(off_grads.keys()) == set(base_grads.keys())
+ for name, gb in base_grads.items():
+ go = off_grads[name]
+ if gb is None or go is None:
+ assert gb is None and go is None, f"Grad None mismatch for {name}"
+ continue
+ assert torch.allclose(
+ go, gb, rtol=1e-3, atol=1e-3
+ ), f"Rank {torch.distributed.get_rank()}: Grad mismatch for {name}"
+
+ # Memory checks (peak allocated during the scheduled 1F1B run)
+ saved_mib = (base_peak - off_peak) / (1024**2)
+ assert saved_mib > 0.0, (
+ f"Expected GPU peak memory reduction for offload_modules={offload_modules}, "
+ f"but got saved={saved_mib:.2f}MiB (base={base_peak/(1024**2):.2f}MiB, "
+ f"off={off_peak/(1024**2):.2f}MiB)"
+ )
+ # If expectation is large enough, enforce approximate match.
+ if expected_offload_mib >= 2.0:
+ rel_err = abs(saved_mib - expected_offload_mib) / max(expected_offload_mib, 1e-6)
+ abs_err = abs(saved_mib - expected_offload_mib)
+ print(
+ f"Rank {torch.distributed.get_rank()}: Saved {saved_mib:.2f}MiB, expected {expected_offload_mib:.2f}MiB"
+ )
+ if abs_err > DELTA:
+ assert rel_err <= EPSILON_A2A, (
+ f"Memory saving mismatch for offload_modules={offload_modules}: "
+ f"saved={saved_mib:.2f}MiB expected~={expected_offload_mib:.2f}MiB "
+ f"(rel_err={rel_err:.2f}, abs_err={abs_err:.2f})"
+ )
+ finally:
+ Utils.destroy_model_parallel()
diff --git a/tests/unit_tests/pipeline_parallel/test_multimodule_communicator.py b/tests/unit_tests/pipeline_parallel/test_multimodule_communicator.py
new file mode 100644
index 00000000000..73739859f42
--- /dev/null
+++ b/tests/unit_tests/pipeline_parallel/test_multimodule_communicator.py
@@ -0,0 +1,780 @@
+import logging
+import os
+import sys
+
+import pytest
+import torch
+import torch.distributed as dist
+from packaging import version
+
+from megatron.core import parallel_state
+from megatron.core.hyper_comm_grid import HyperCommGrid
+from megatron.core.model_parallel_config import ModelParallelConfig
+from megatron.core.pipeline_parallel.multimodule_communicator import MultiModulePipelineCommunicator
+from tests.unit_tests.pipeline_parallel.test_bridge_communicator import (
+ _avg_params,
+ _create_transformer_block,
+ _get_pg_collection_from_grid,
+ create_hypercomm_grid,
+ get_transformer_block_and_grid,
+)
+from tests.unit_tests.test_utilities import Utils
+
+
+class TestMultiModulePipelineCommunicator:
+
+ @classmethod
+ def setup_class(cls):
+ """Set up distributed environment for the entire test class."""
+ if not dist.is_initialized():
+ dist.init_process_group(backend="nccl")
+ if torch.cuda.is_available():
+ torch.cuda.set_device(int(os.environ["LOCAL_RANK"]))
+
+ world_size = dist.get_world_size()
+ if world_size != 8:
+ pytest.skip(
+ f"These tests require 8 GPUs, but only {world_size} are available.",
+ allow_module_level=True,
+ )
+
+ def teardown_class(cls):
+ Utils.destroy_model_parallel()
+
+ def test_multimodule_communicator_init(self):
+ """Test MultiModulePipelineCommunicator initialization."""
+
+ # Create process group grids for each module
+ image_encoder_grid = create_hypercomm_grid(offset=0, tp=1, cp=1, pp=1, dp=1)
+ audio_encoder_grid = create_hypercomm_grid(offset=1, tp=1, cp=1, pp=1, dp=1)
+ llm_grid = create_hypercomm_grid(offset=2, tp=2, cp=1, pp=2, dp=1)
+ generator_grid = create_hypercomm_grid(offset=6, tp=2, cp=1, pp=1, dp=1)
+
+ # Define module-grid mapping
+ module_to_grid_map = {
+ 'image_encoder': image_encoder_grid,
+ 'audio_encoder': audio_encoder_grid,
+ 'llm': llm_grid,
+ 'generator': generator_grid,
+ }
+ # Define module computation topology
+ topology = {
+ 'image_encoder': ['llm'],
+ 'audio_encoder': ['llm'],
+ 'llm': ['generator'],
+ 'generator': [],
+ }
+ config = ModelParallelConfig(bf16=True)
+ # Initialize communicator
+ mllm_comm = MultiModulePipelineCommunicator(module_to_grid_map, topology, config)
+ # Test attributes match expectations
+ assert mllm_comm.module_to_grid_map == module_to_grid_map
+ assert mllm_comm.topology == topology
+ assert mllm_comm.config == config
+ assert mllm_comm.current_rank == dist.get_rank()
+
+ def test_compute_total_pipeline_stages(self):
+ """Test compute_total_pipeline_stages for overall chain and until specific ranks."""
+
+ # Create process group grids for each module
+ image_encoder_grid = create_hypercomm_grid(offset=0, tp=1, cp=1, pp=1, dp=1)
+ audio_encoder_grid = create_hypercomm_grid(offset=1, tp=1, cp=1, pp=1, dp=1)
+ llm_grid = create_hypercomm_grid(offset=2, tp=2, cp=1, pp=2, dp=1)
+ generator_grid = create_hypercomm_grid(offset=6, tp=1, cp=1, pp=1, dp=2)
+
+ # Define module-grid mapping and topology
+ module_to_grid_map = {
+ 'image_encoder': image_encoder_grid,
+ 'audio_encoder': audio_encoder_grid,
+ 'llm': llm_grid,
+ 'generator': generator_grid,
+ }
+ topology = {
+ 'image_encoder': ['llm'],
+ 'audio_encoder': ['llm'],
+ 'llm': ['generator'],
+ 'generator': [],
+ }
+
+ # Overall total pipeline stages: max(1,1) + 2 + 1 = 4
+ total = MultiModulePipelineCommunicator.compute_total_pipeline_stages(
+ topology, module_to_grid_map
+ )
+ assert total == 4
+
+ llm_pp_rank = MultiModulePipelineCommunicator.compute_total_pipeline_stages(
+ topology, module_to_grid_map, rank=2, module_name='llm'
+ )
+ assert llm_pp_rank == 2
+
+ def test_send_forward_recv_forward(self):
+ """Test send_forward and recv_forward operations."""
+ if not dist.is_initialized():
+ pytest.skip("Distributed not initialized")
+
+ # Create process group grids for each module
+ image_encoder_grid = create_hypercomm_grid(offset=0, tp=1, cp=1, pp=1, dp=1)
+ audio_encoder_grid = create_hypercomm_grid(offset=1, tp=1, cp=1, pp=1, dp=1)
+ llm_grid = create_hypercomm_grid(offset=2, tp=2, cp=1, pp=2, dp=1)
+ generator_grid = create_hypercomm_grid(offset=6, tp=1, cp=1, pp=1, dp=2)
+
+ # Set up module-grid mapping and topology
+ module_to_grid_map = {
+ 'image_encoder': image_encoder_grid,
+ 'audio_encoder': audio_encoder_grid,
+ 'llm': llm_grid,
+ 'generator': generator_grid,
+ }
+ topology = {
+ 'image_encoder': ['llm'],
+ 'audio_encoder': ['llm'],
+ 'llm': ['generator'],
+ 'generator': [],
+ }
+ config = ModelParallelConfig(pipeline_dtype=torch.float)
+ mllm_comm = MultiModulePipelineCommunicator(module_to_grid_map, topology, config)
+
+ # Simulate forward communication for each module
+ if mllm_comm.is_current_rank_in_grid(image_encoder_grid):
+ # Image encoder sends output forward
+ output_dict = {'image_encoder': torch.randn(2, 8, 128).cuda()}
+ mllm_comm.send_forward(output_dict)
+ if mllm_comm.is_current_rank_in_grid(audio_encoder_grid):
+ # Audio encoder sends output forward
+ output_dict = {'audio_encoder': torch.randn(2, 16, 128).cuda()}
+ mllm_comm.send_forward(output_dict)
+ if mllm_comm.is_current_rank_in_grid(llm_grid):
+ output_dict = {'llm': torch.randn(2, 32, 128).cuda()}
+ if dist.get_rank() == 2 or dist.get_rank() == 3:
+ # LLM stage receives both image and audio outputs
+ input_dict = mllm_comm.recv_forward()
+ assert input_dict['image_encoder'].shape == (2, 8, 128)
+ assert input_dict['audio_encoder'].shape == (2, 16, 128)
+ mllm_comm.send_forward(output_dict)
+ else:
+ # LLM stage receives concatenated LLM outputs
+ input_dict = mllm_comm.recv_forward(tensor_shape=(2, 32, 128))
+ assert input_dict['llm'].shape == (2, 32, 128)
+ mllm_comm.send_forward(output_dict)
+ if mllm_comm.is_current_rank_in_grid(generator_grid):
+ # Generator module receives final LLM output
+ input_dict = mllm_comm.recv_forward()
+ assert input_dict['llm'].shape == (1, 32, 128)
+
+ def test_send_forward_recv_forward_with_different_pp_size(self):
+ """Test for the case when pp(image_encoder) != pp(audio_encoder)."""
+ if not dist.is_initialized():
+ pytest.skip("Distributed not initialized")
+
+ # Create process group grids for each module
+ image_encoder_grid = create_hypercomm_grid(offset=0, tp=1, cp=1, pp=2, dp=1)
+ audio_encoder_grid = create_hypercomm_grid(offset=2, tp=2, cp=1, pp=1, dp=1)
+ llm_grid = create_hypercomm_grid(offset=4, tp=1, cp=1, pp=4, dp=1)
+
+ # Set up module-grid mapping and topology
+ module_to_grid_map = {
+ 'image_encoder': image_encoder_grid,
+ 'audio_encoder': audio_encoder_grid,
+ 'llm': llm_grid,
+ }
+ topology = {'image_encoder': ['llm'], 'audio_encoder': ['llm'], 'llm': []}
+ config = ModelParallelConfig(pipeline_dtype=torch.float)
+ mllm_comm = MultiModulePipelineCommunicator(module_to_grid_map, topology, config)
+
+ # Simulate forward communication for each module
+ if mllm_comm.is_current_rank_in_grid(image_encoder_grid):
+ output_dict = {'image_encoder': torch.randn(2, 8, 128).cuda()}
+ if dist.get_rank() == 0:
+ # Image encoder sends output forward
+ mllm_comm.send_forward(output_dict)
+ else:
+ # Image stage receives image outputs
+ input_dict = mllm_comm.recv_forward(tensor_shape=(2, 8, 128))
+ assert input_dict['image_encoder'].shape == (2, 8, 128)
+ mllm_comm.send_forward(output_dict)
+ if mllm_comm.is_current_rank_in_grid(audio_encoder_grid):
+ # Audio encoder sends output forward
+ output_dict = {'audio_encoder': torch.randn(2, 16, 128).cuda()}
+ mllm_comm.send_forward(output_dict)
+ if mllm_comm.is_current_rank_in_grid(llm_grid):
+ output_dict = {'llm': torch.randn(2, 32, 128).cuda()}
+ if dist.get_rank() == 4:
+ # LLM stage receives both image and audio outputs
+ input_dict = mllm_comm.recv_forward()
+ assert input_dict['image_encoder'].shape == (2, 8, 128)
+ assert input_dict['audio_encoder'].shape == (2, 16, 128)
+ mllm_comm.send_forward(output_dict)
+ elif dist.get_rank() == 5 or dist.get_rank() == 6:
+ # LLM stage receives concatenated LLM outputs
+ input_dict = mllm_comm.recv_forward(tensor_shape=(2, 32, 128))
+ assert input_dict['llm'].shape == (2, 32, 128)
+ mllm_comm.send_forward(output_dict)
+ elif dist.get_rank() == 7:
+ # LLM stage receives concatenated LLM outputs
+ input_dict = mllm_comm.recv_forward(tensor_shape=(2, 32, 128))
+ assert input_dict['llm'].shape == (2, 32, 128)
+
+ def test_send_backward_recv_backward(self):
+ """Test send_backward and recv_backward operations."""
+ if not dist.is_initialized():
+ pytest.skip("Distributed not initialized")
+
+ # Create process group grids for each module
+ image_encoder_grid = create_hypercomm_grid(offset=0, tp=1, cp=1, pp=1, dp=1)
+ audio_encoder_grid = create_hypercomm_grid(offset=1, tp=1, cp=1, pp=1, dp=1)
+ llm_grid = create_hypercomm_grid(offset=2, tp=2, cp=1, pp=2, dp=1)
+ generator_grid = create_hypercomm_grid(offset=6, tp=1, cp=1, pp=1, dp=2)
+
+ # Set up module-grid mapping and topology
+ module_to_grid_map = {
+ 'image_encoder': image_encoder_grid,
+ 'audio_encoder': audio_encoder_grid,
+ 'llm': llm_grid,
+ 'generator': generator_grid,
+ }
+ topology = {
+ 'image_encoder': ['llm'],
+ 'audio_encoder': ['llm'],
+ 'llm': ['generator'],
+ 'generator': [],
+ }
+ config = ModelParallelConfig(pipeline_dtype=torch.float)
+ mllm_comm = MultiModulePipelineCommunicator(module_to_grid_map, topology, config)
+
+ # Simulate backward communication for each module
+ if mllm_comm.is_current_rank_in_grid(generator_grid):
+ # Generator sends gradient backward
+ grad_dict = {'llm': torch.randn(1, 32, 128).cuda()}
+ mllm_comm.send_backward(grad_dict)
+ if mllm_comm.is_current_rank_in_grid(llm_grid):
+ if dist.get_rank() == 4 or dist.get_rank() == 5:
+ # LLM receives expanded gradient and sends backward
+ received_grad = mllm_comm.recv_backward()
+ assert received_grad['llm'].shape == (2, 32, 128)
+ grad_dict = {'llm': torch.randn(2, 32, 128).cuda()}
+ mllm_comm.send_backward(grad_dict)
+ else:
+ # LLM receives gradient and sends backward to both image/audio encoders
+ received_grad = mllm_comm.recv_backward(tensor_shape=(2, 32, 128))
+ assert received_grad['llm'].shape == (2, 32, 128)
+ grad_dict = {
+ 'image_encoder': torch.randn(2, 8, 128).cuda(),
+ 'audio_encoder': torch.randn(2, 16, 128).cuda(),
+ }
+ mllm_comm.send_backward(grad_dict)
+ if mllm_comm.is_current_rank_in_grid(image_encoder_grid):
+ # Image encoder receives its gradient
+ received_grad = mllm_comm.recv_backward()
+ assert received_grad['image_encoder'].shape == (2, 8, 128)
+ if mllm_comm.is_current_rank_in_grid(audio_encoder_grid):
+ # Audio encoder receives its gradient
+ received_grad = mllm_comm.recv_backward()
+ assert received_grad['audio_encoder'].shape == (2, 16, 128)
+
+ @pytest.mark.skipif(
+ version.parse(torch.__version__) < version.parse('2.3.0'),
+ reason="Feature requires PyTorch 2.3 or later",
+ )
+ def test_send_forward_recv_backward_send_backward_recv_forward(self):
+ """Test send_forward_recv_backward and send_backward_recv_forward operations."""
+ if not dist.is_initialized():
+ pytest.skip("Distributed not initialized")
+
+ # Create process group grids for each module
+ image_encoder_grid = create_hypercomm_grid(offset=0, tp=1, cp=1, pp=1, dp=1)
+ audio_encoder_grid = create_hypercomm_grid(offset=1, tp=1, cp=1, pp=1, dp=1)
+ llm_grid = create_hypercomm_grid(offset=2, tp=2, cp=1, pp=2, dp=1)
+ generator_grid = create_hypercomm_grid(offset=6, tp=1, cp=1, pp=1, dp=2)
+
+ # Set up module-grid mapping and topology
+ module_to_grid_map = {
+ 'image_encoder': image_encoder_grid,
+ 'audio_encoder': audio_encoder_grid,
+ 'llm': llm_grid,
+ 'generator': generator_grid,
+ }
+ topology = {
+ 'image_encoder': ['llm'],
+ 'audio_encoder': ['llm'],
+ 'llm': ['generator'],
+ 'generator': [],
+ }
+ config = ModelParallelConfig(pipeline_dtype=torch.float)
+ mllm_comm = MultiModulePipelineCommunicator(module_to_grid_map, topology, config)
+
+ # Simulate bidirectional send/recv for forward and backward in pipeline
+
+ # Encoder stages send forward to the first stage of LLM, and receive backward from the first stage of LLM
+ if mllm_comm.is_current_rank_in_grid(image_encoder_grid):
+ output_dict = {'image_encoder': torch.randn(2, 8, 128).cuda()}
+ received_grad = mllm_comm.send_forward_recv_backward(output_dict)
+ assert received_grad['image_encoder'].shape == (2, 8, 128)
+ if mllm_comm.is_current_rank_in_grid(audio_encoder_grid):
+ output_dict = {'audio_encoder': torch.randn(2, 16, 128).cuda()}
+ received_grad = mllm_comm.send_forward_recv_backward(output_dict)
+ assert received_grad['audio_encoder'].shape == (2, 16, 128)
+ if mllm_comm.is_current_rank_in_grid(llm_grid):
+ if dist.get_rank() == 2 or dist.get_rank() == 3:
+ grad_dict = {
+ 'image_encoder': torch.randn(2, 8, 128).cuda(),
+ 'audio_encoder': torch.randn(2, 16, 128).cuda(),
+ }
+ input_dict = mllm_comm.send_backward_recv_forward(grad_dict)
+ assert input_dict['image_encoder'].shape == (2, 8, 128)
+ assert input_dict['audio_encoder'].shape == (2, 16, 128)
+
+ # First stage of LLM sends forward to the second stage of LLM, and receive backward from the second stage of LLM
+ if mllm_comm.is_current_rank_in_grid(llm_grid):
+ if dist.get_rank() == 2 or dist.get_rank() == 3:
+ output_dict = {'llm': torch.randn(2, 32, 128).cuda()}
+ received_grad = mllm_comm.send_forward_recv_backward(
+ output_dict, tensor_shape=(2, 32, 128)
+ )
+ assert received_grad['llm'].shape == (2, 32, 128)
+ if dist.get_rank() == 4 or dist.get_rank() == 5:
+ grad_dict = {'llm': torch.randn(2, 32, 128).cuda()}
+ input_dict = mllm_comm.send_backward_recv_forward(
+ grad_dict, tensor_shape=(2, 32, 128)
+ )
+ assert input_dict['llm'].shape == (2, 32, 128)
+
+ # Second stage of LLM sends forward to generator, and receive backward from generator
+ if mllm_comm.is_current_rank_in_grid(llm_grid):
+ if dist.get_rank() == 4 or dist.get_rank() == 5:
+ output_dict = {'llm': torch.randn(2, 32, 128).cuda()}
+ received_grad = mllm_comm.send_forward_recv_backward(output_dict)
+ assert received_grad['llm'].shape == (2, 32, 128)
+ if mllm_comm.is_current_rank_in_grid(generator_grid):
+ grad_dict = {'llm': torch.randn(1, 32, 128).cuda()}
+ input_dict = mllm_comm.send_backward_recv_forward(grad_dict)
+ assert input_dict['llm'].shape == (1, 32, 128)
+
+ @pytest.mark.skipif(
+ version.parse(torch.__version__) < version.parse('2.3.0'),
+ reason="Feature requires PyTorch 2.3 or later",
+ )
+ def test_send_forward_recv_forward_with_transformer_blocks(self):
+ """Test send_forward and recv_forward operations."""
+
+ # Set model/test dimensions for easier debugging and output comparison
+ hidden_size = 16
+ sequence_length = 2
+ micro_batch_size = 2
+
+ # For reproducibility, set a fixed seed
+ torch.manual_seed(12345)
+ dtype = torch.float32
+
+ # Create random input hidden states tensor
+ hidden_states = torch.randn(
+ (sequence_length, micro_batch_size, hidden_size), device="cuda"
+ ).to(dtype)
+ current_rank = dist.get_rank()
+
+ # ========== Initialize tensor model-parallel environment ==========
+ parallel_state_tp = 2
+ Utils.initialize_model_parallel(tensor_model_parallel_size=2)
+
+ # ========== Build reference 1D grid and transformer block for weight sharing ==========
+ ref_grid = create_hypercomm_grid(offset=0, tp=1, cp=1, pp=1, dp=8)
+ ref_pg_collection = _get_pg_collection_from_grid(ref_grid)
+ ref_block = _create_transformer_block(
+ dtype=dtype, hidden_size=hidden_size, pg_collection=ref_pg_collection
+ )
+ _avg_params(
+ ref_block, ref_grid.get_pg("dp")
+ ) # Ensure parameters are averaged across data parallel (DP)
+
+ # ========== Create different transformer blocks for each model stage ==========
+ # Image encoder
+ image_encoder_block, image_encoder_grid = get_transformer_block_and_grid(
+ ref_block,
+ tp_size=1,
+ cp_size=1,
+ pp_size=1,
+ dp_size=1,
+ grid_offset=0,
+ hidden_size=hidden_size,
+ dtype=dtype,
+ )
+ # Audio encoder
+ audio_encoder_block, audio_encoder_grid = get_transformer_block_and_grid(
+ ref_block,
+ tp_size=1,
+ cp_size=1,
+ pp_size=1,
+ dp_size=1,
+ grid_offset=1,
+ hidden_size=hidden_size,
+ dtype=dtype,
+ )
+ # LLM (Large Language Model) block with tensor & pipeline parallelism
+ llm_block, llm_grid = get_transformer_block_and_grid(
+ ref_block,
+ tp_size=2,
+ cp_size=1,
+ pp_size=2,
+ dp_size=1,
+ grid_offset=2,
+ hidden_size=hidden_size,
+ dtype=dtype,
+ )
+ # Generator block (final stage) with DP=2
+ generator_block, generator_grid = get_transformer_block_and_grid(
+ ref_block,
+ tp_size=1,
+ cp_size=1,
+ pp_size=1,
+ dp_size=2,
+ grid_offset=6,
+ hidden_size=hidden_size,
+ dtype=dtype,
+ )
+
+ # ========== Define module-to-grid correspondence and pipeline topology ==========
+ module_to_grid_map = {
+ 'image_encoder': image_encoder_grid,
+ 'audio_encoder': audio_encoder_grid,
+ 'llm': llm_grid,
+ 'generator': generator_grid,
+ }
+ topology = {
+ 'image_encoder': ['llm'], # image_encoder sends output to llm
+ 'audio_encoder': ['llm'], # audio_encoder sends output to llm
+ 'llm': ['generator'], # llm sends output to generator
+ 'generator': [], # generator is the final module
+ }
+ config = ModelParallelConfig(pipeline_dtype=torch.float)
+ # Define dimension mapping for sequence, batch, hidden
+ dim_mapping = {'s': 0, 'h': 2, 'b': 1}
+ seq_dim = dim_mapping['s']
+
+ # Communication handler for multi-module pipeline (send/recv abstraction)
+ mllm_comm = MultiModulePipelineCommunicator(
+ module_to_grid_map, topology, config, dim_mapping=dim_mapping
+ )
+
+ # ========== Run actual distributed pipeline blocks (per process, depending on role) ==========
+ if mllm_comm.is_current_rank_in_grid(image_encoder_grid):
+ # Image encoder rank: run forward and send output
+ image_encoder_output = image_encoder_block(
+ hidden_states=hidden_states, attention_mask=None
+ )
+ output_dict = {'image_encoder': image_encoder_output}
+ mllm_comm.send_forward(output_dict)
+ if mllm_comm.is_current_rank_in_grid(audio_encoder_grid):
+ # Audio encoder rank: run forward and send output
+ audio_encoder_output = audio_encoder_block(
+ hidden_states=hidden_states, attention_mask=None
+ )
+ output_dict = {'audio_encoder': audio_encoder_output}
+ mllm_comm.send_forward(output_dict)
+ if mllm_comm.is_current_rank_in_grid(llm_grid):
+ if dist.get_rank() == 2 or dist.get_rank() == 3:
+ # LLM stage 0 (receives both image and audio, concatenates along seq_dim)
+ input_dict = mllm_comm.recv_forward()
+ llm_output = llm_block(
+ hidden_states=torch.cat(
+ [input_dict['image_encoder'], input_dict['audio_encoder']], dim=seq_dim
+ ),
+ attention_mask=None,
+ )
+ output_dict = {'llm': llm_output}
+ mllm_comm.send_forward(output_dict)
+ else:
+ # LLM stage 1 (receives output of previous LLM stage)
+ input_dict = mllm_comm.recv_forward(
+ tensor_shape=(sequence_length * 2, micro_batch_size, hidden_size)
+ )
+ llm_output = llm_block(hidden_states=input_dict['llm'], attention_mask=None)
+ output_dict = {'llm': llm_output}
+ mllm_comm.send_forward(output_dict)
+
+ if mllm_comm.is_current_rank_in_grid(generator_grid):
+ # Generator block: only receives from llm and runs forward
+ input_dict = mllm_comm.recv_forward()
+ generator_output = generator_block(hidden_states=input_dict['llm'], attention_mask=None)
+
+ # ========== Build a reference (serial/global) pipeline for correctness checking ==========
+ global_image_encoder_block, _ = get_transformer_block_and_grid(
+ ref_block,
+ tp_size=parallel_state_tp,
+ use_global_parallel_state=True,
+ hidden_size=hidden_size,
+ dtype=dtype,
+ )
+ global_audio_encoder_block, _ = get_transformer_block_and_grid(
+ ref_block,
+ tp_size=parallel_state_tp,
+ use_global_parallel_state=True,
+ hidden_size=hidden_size,
+ dtype=dtype,
+ )
+ global_llm_block_pp_rank_0, _ = get_transformer_block_and_grid(
+ ref_block,
+ tp_size=parallel_state_tp,
+ use_global_parallel_state=True,
+ hidden_size=hidden_size,
+ dtype=dtype,
+ )
+ global_llm_block_pp_rank_1, _ = get_transformer_block_and_grid(
+ ref_block,
+ tp_size=parallel_state_tp,
+ use_global_parallel_state=True,
+ hidden_size=hidden_size,
+ dtype=dtype,
+ )
+ global_generator_block, _ = get_transformer_block_and_grid(
+ ref_block,
+ tp_size=parallel_state_tp,
+ use_global_parallel_state=True,
+ hidden_size=hidden_size,
+ dtype=dtype,
+ )
+
+ # Run each stage sequentially as a global pipeline (for truth)
+ global_image_encoder_output = global_image_encoder_block(
+ hidden_states=hidden_states, attention_mask=None
+ )
+ global_audio_encoder_output = global_audio_encoder_block(
+ hidden_states=hidden_states, attention_mask=None
+ )
+ # Compare output between global and distributed blocks for image/audio stage
+ if current_rank == 0:
+ torch.testing.assert_close(
+ global_image_encoder_output, image_encoder_output, rtol=1e-3, atol=1e-3
+ )
+ if current_rank == 1:
+ torch.testing.assert_close(
+ global_audio_encoder_output, audio_encoder_output, rtol=1e-3, atol=1e-3
+ )
+
+ # Feed outputs to LLM stages (emulate pipeline cut with concatenation)
+ global_llm_input = torch.cat(
+ [global_image_encoder_output, global_audio_encoder_output], dim=seq_dim
+ )
+ global_llm_pp_rank_0_output = global_llm_block_pp_rank_0(
+ hidden_states=global_llm_input, attention_mask=None
+ )
+ if current_rank == 2 or current_rank == 3:
+ torch.testing.assert_close(
+ global_llm_pp_rank_0_output, llm_output, rtol=1e-3, atol=1e-3
+ )
+ global_llm_pp_rank_1_output = global_llm_block_pp_rank_1(
+ hidden_states=global_llm_pp_rank_0_output, attention_mask=None
+ )
+ if current_rank == 4 or current_rank == 5:
+ torch.testing.assert_close(
+ global_llm_pp_rank_1_output, llm_output, rtol=1e-3, atol=1e-3
+ )
+
+ # Generator output and comparison to distributed output (for each DP chunk)
+ global_generator_block_output = global_generator_block(
+ hidden_states=global_llm_pp_rank_1_output, attention_mask=None
+ )
+ global_generator_block_chunks = torch.split(
+ global_generator_block_output, global_generator_block_output.shape[1] // 2, dim=1
+ )
+ if current_rank == 6:
+ torch.testing.assert_close(
+ global_generator_block_chunks[0], generator_output, rtol=1e-3, atol=1e-3
+ )
+ if current_rank == 7:
+ torch.testing.assert_close(
+ global_generator_block_chunks[1], generator_output, rtol=1e-3, atol=1e-3
+ )
+
+ @pytest.mark.skipif(
+ version.parse(torch.__version__) < version.parse('2.3.0'),
+ reason="Feature requires PyTorch 2.3 or later",
+ )
+ @pytest.mark.parametrize(
+ "grid1_tp, grid1_pp, grid1_dp, grid2_tp, grid2_pp, grid2_dp, parallel_state_tp",
+ [
+ (2, 1, 1, 2, 1, 1, 2), # TP2PP1DP1 to TP2PP1DP1
+ (2, 1, 1, 2, 2, 1, 2), # TP2PP1DP1 to TP2PP2DP1
+ (2, 2, 1, 2, 2, 1, 2), # TP2PP2DP1 to TP2PP2DP1
+ (4, 1, 1, 4, 1, 1, 4), # TP4DP1 to TP4DP1
+ (2, 1, 2, 4, 1, 1, 2), # TP2DP2 to TP4DP1
+ (4, 1, 1, 2, 1, 2, 2), # TP4DP1 to TP2DP2
+ (2, 1, 2, 1, 1, 4, 2), # TP2DP2 to TP1DP4
+ ],
+ )
+ def test_send_forward_recv_forward_with_transformer_blocks_and_different_parallelisms(
+ self, grid1_tp, grid1_pp, grid1_dp, grid2_tp, grid2_pp, grid2_dp, parallel_state_tp
+ ):
+ """Test bridge communicator with two transformer blocks having different process group configurations."""
+ # Model and input configuration
+ hidden_size = 16
+ sequence_length = 2
+ micro_batch_size = 8
+ torch.manual_seed(12345)
+ dtype = torch.float32
+
+ # Create random input tensor on CUDA
+ hidden_states = torch.randn(
+ (sequence_length, micro_batch_size, hidden_size), device="cuda"
+ ).to(dtype)
+ hidden_states_ref = hidden_states.clone()
+ current_rank = dist.get_rank()
+
+ # Initialize model parallel with desired TP
+ Utils.initialize_model_parallel(tensor_model_parallel_size=parallel_state_tp)
+
+ # Build a reference grid and block for parameter sharing & DP averaging
+ ref_grid = create_hypercomm_grid(offset=0, tp=1, cp=1, pp=1, dp=8)
+ ref_pg_collection = _get_pg_collection_from_grid(ref_grid)
+ ref_block = _create_transformer_block(
+ dtype=dtype, hidden_size=hidden_size, pg_collection=ref_pg_collection
+ )
+ _avg_params(
+ ref_block, ref_grid.get_pg("dp")
+ ) # Synchronize parameters across DP for reproducibility
+
+ # ====== Create two transformer block+grid pairs with different TP/DP settings ======
+ block_grid_1, grid_1 = get_transformer_block_and_grid(
+ ref_block,
+ tp_size=grid1_tp,
+ pp_size=grid1_pp,
+ dp_size=grid1_dp,
+ grid_offset=0,
+ hidden_size=hidden_size,
+ dtype=dtype,
+ )
+
+ block_grid_2, grid_2 = get_transformer_block_and_grid(
+ ref_block,
+ tp_size=grid2_tp,
+ pp_size=grid2_pp,
+ dp_size=grid2_dp,
+ grid_offset=grid_1.size,
+ hidden_size=hidden_size,
+ dtype=dtype,
+ )
+
+ dist.barrier() # Synchronize ranks before communication
+
+ # Module-grid map and pipeline communication topology
+ module_to_grid_map = {'image_encoder': grid_1, 'llm': grid_2}
+ topology = {
+ 'image_encoder': ['llm'], # image_encoder sends forward results to llm
+ 'llm': [], # llm is the last stage here
+ }
+ config = ModelParallelConfig(pipeline_dtype=torch.float)
+ mllm_comm = MultiModulePipelineCommunicator(
+ module_to_grid_map, topology, config, dim_mapping={'s': 0, 'h': 2, 'b': 1}
+ )
+
+ output_grid_2 = None
+ # If current rank is in the first grid, run first block and send output
+ if grid_1 is not None and mllm_comm.is_current_rank_in_grid(grid_1):
+ rank_module_info = mllm_comm.rank_module_map['image_encoder']
+ if rank_module_info.pp_rank == 0:
+ hidden_states = block_grid_1(hidden_states=hidden_states, attention_mask=None)
+ mllm_comm.send_forward({'image_encoder': hidden_states})
+ else:
+ input_dict = mllm_comm.recv_forward(
+ tensor_shape=(sequence_length, micro_batch_size, hidden_size)
+ )
+ hidden_states = input_dict['image_encoder']
+ hidden_states = block_grid_1(hidden_states=hidden_states, attention_mask=None)
+ mllm_comm.send_forward({'image_encoder': hidden_states})
+
+ # If current rank is in second grid, receive and run the second block
+ if grid_2 is not None and mllm_comm.is_current_rank_in_grid(grid_2):
+ rank_module_info = mllm_comm.rank_module_map['llm']
+ if rank_module_info.pp_rank == 0:
+ input_dict = mllm_comm.recv_forward()
+ hidden_states = input_dict['image_encoder']
+ hidden_states = block_grid_2(hidden_states=hidden_states, attention_mask=None)
+ if rank_module_info.pp_rank == rank_module_info.pp_size - 1:
+ output_grid_2 = hidden_states
+ else:
+ mllm_comm.send_forward({'llm': hidden_states})
+ elif rank_module_info.pp_rank < rank_module_info.pp_size - 1:
+ input_dict = mllm_comm.recv_forward(
+ tensor_shape=(
+ sequence_length,
+ (grid1_dp * micro_batch_size) // grid2_dp,
+ hidden_size,
+ )
+ )
+ hidden_states = input_dict['llm']
+ hidden_states = block_grid_2(hidden_states=hidden_states, attention_mask=None)
+ mllm_comm.send_forward({'llm': hidden_states})
+ else:
+ input_dict = mllm_comm.recv_forward(
+ tensor_shape=(
+ sequence_length,
+ (grid1_dp * micro_batch_size) // grid2_dp,
+ hidden_size,
+ )
+ )
+ hidden_states = input_dict['llm']
+ output_grid_2 = block_grid_2(hidden_states=hidden_states, attention_mask=None)
+
+ # Compute expected output shape based on change in DP size (chunk/expand batch dimension appropriately)
+ factor = max(grid1_dp, grid2_dp) // min(grid1_dp, grid2_dp)
+ expected_output_shape = (
+ sequence_length,
+ (
+ micro_batch_size * factor
+ if grid1_dp > grid2_dp
+ else micro_batch_size // factor
+ ),
+ hidden_size,
+ )
+ assert (
+ output_grid_2.shape == expected_output_shape
+ ), f"Output2 shape mismatch: {output_grid_2.shape}"
+
+ # ====== Reference: global (replicated) pipeline forward for correctness checking ======
+ global_block_1, _ = get_transformer_block_and_grid(
+ ref_block,
+ tp_size=parallel_state_tp,
+ use_global_parallel_state=True,
+ hidden_size=hidden_size,
+ dtype=dtype,
+ )
+ global_block_2, _ = get_transformer_block_and_grid(
+ ref_block,
+ tp_size=parallel_state_tp,
+ use_global_parallel_state=True,
+ hidden_size=hidden_size,
+ dtype=dtype,
+ )
+
+ for i in range(grid1_pp):
+ hidden_states_ref = global_block_1(hidden_states=hidden_states_ref, attention_mask=None)
+
+ for i in range(grid2_pp):
+ hidden_states_ref = global_block_2(hidden_states=hidden_states_ref, attention_mask=None)
+
+ # Output comparison under different DP compositions between grids
+ if (
+ grid_2 is not None
+ and mllm_comm.is_current_rank_in_grid(grid_2)
+ and rank_module_info.pp_rank == rank_module_info.pp_size - 1
+ ):
+ if grid1_dp == grid2_dp:
+ # DP size matches: all outputs directly compared
+ torch.testing.assert_close(hidden_states_ref, output_grid_2, rtol=1e-3, atol=1e-3)
+ elif grid1_dp < grid2_dp:
+ # If grid2 expands DP: each output_grid_2 chunk corresponds to a split of the reference output
+ grid2_dp_ranks = grid_2._gen_rank_enum([x for x in grid_2.dim_names if x != "dp"])
+ global_block_2_chunks = torch.split(
+ hidden_states_ref, hidden_states_ref.shape[1] // (grid2_dp // grid1_dp), dim=1
+ )
+ relevant_chunk = None
+ for i, dp_ranks in enumerate(grid2_dp_ranks):
+ if current_rank in dp_ranks:
+ relevant_chunk = global_block_2_chunks[i % len(global_block_2_chunks)]
+ torch.testing.assert_close(relevant_chunk, output_grid_2, rtol=1e-3, atol=1e-3)
+ else:
+ # If DP shrinks (grid1_dp > grid2_dp): just compare the relevant first chunk
+ output_grid_2_first_chunk = torch.chunk(output_grid_2, grid1_dp // grid2_dp, dim=1)[
+ 0
+ ]
+ torch.testing.assert_close(
+ hidden_states_ref, output_grid_2_first_chunk, rtol=1e-3, atol=1e-3
+ )
diff --git a/tests/unit_tests/pipeline_parallel/test_pipeline_layout.py b/tests/unit_tests/pipeline_parallel/test_pipeline_layout.py
index faf83837a23..cdf32d01fd3 100644
--- a/tests/unit_tests/pipeline_parallel/test_pipeline_layout.py
+++ b/tests/unit_tests/pipeline_parallel/test_pipeline_layout.py
@@ -1,4 +1,4 @@
-# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved.
+# Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
import os
from pathlib import Path
@@ -21,6 +21,7 @@
)
from megatron.core.tensor_parallel.random import model_parallel_cuda_manual_seed
from megatron.core.transformer.enums import ModelType
+from megatron.core.transformer.multi_token_prediction import mtp_on_this_rank
from megatron.core.transformer.transformer_config import TransformerConfig
from megatron.training.checkpointing import load_checkpoint, save_checkpoint
from megatron.training.global_vars import set_args
@@ -53,6 +54,8 @@ def initialize_gpt_model(
virtual_pipeline_model_parallel_size=virtual_pipeline_model_parallel_size,
hidden_dropout=0.0,
attention_dropout=0.0,
+ mtp_num_layers=1 if with_mtp else None,
+ mtp_loss_scaling_factor=1.0 if with_mtp else None,
)
default_config_kwargs.update(**config_kwargs)
transformer_config = TransformerConfig(**default_config_kwargs)
@@ -61,9 +64,6 @@ def initialize_gpt_model(
transformer_config.moe_ffn_hidden_size = 128
transformer_config.num_moe_experts = 4
transformer_config.add_bias_linear = False
- if with_mtp:
- transformer_config.mtp_num_layers = 1
- transformer_config.mtp_loss_scaling_factor = 1.0
model = []
for i in range(virtual_pipeline_model_parallel_size or 1):
if is_moe:
@@ -71,8 +71,11 @@ def initialize_gpt_model(
else:
layer_spec = layer_spec_fn()
- if is_moe and with_mtp and mpu.is_pipeline_last_stage(ignore_virtual=False, vp_stage=i):
- transformer_layer_spec_for_mtp = gpt_te_spec(transformer_config)
+ if with_mtp and mtp_on_this_rank(transformer_config, ignore_virtual=False, vp_stage=i):
+ if is_moe:
+ transformer_layer_spec_for_mtp = gpt_te_spec(transformer_config)
+ else:
+ transformer_layer_spec_for_mtp = layer_spec
mtp_block_spec = get_gpt_mtp_block_spec(
transformer_config,
transformer_layer_spec_for_mtp,
@@ -81,6 +84,10 @@ def initialize_gpt_model(
)
else:
mtp_block_spec = None
+
+ # print("========================")
+ # print("[DEBUG] mtp_block_spec is ", mtp_block_spec)
+ # exit()
pre_process = mpu.is_pipeline_first_stage(ignore_virtual=False, vp_stage=i)
post_process = mpu.is_pipeline_last_stage(ignore_virtual=False, vp_stage=i)
this_model = (
@@ -162,7 +169,7 @@ def create_args():
[],
["decoder"],
["decoder"],
- ["decoder"] * 2 + ["loss"],
+ ["decoder"] * 2 + ["mtp"] + ["loss"],
],
False,
True,
@@ -184,7 +191,19 @@ def create_args():
False,
),
((1, 2, None), [["embedding"] + ["decoder"] * 4, ["decoder"] * 4 + ["loss"]], True, False),
- ((1, 4, 2), "E|t*3|(t|)*5L", True, True),
+ ((1, 4, 2), "E|t*3|(t|)*5mL", True, True), # mtp in the last stage
+ (
+ (1, 4, 2),
+ "E|t*3|(t|)*4tm|L",
+ True,
+ True,
+ ), # mtp in the second last stage with a decoder layer
+ (
+ (1, 4, 2),
+ "E|t*3|(t|)*3tt|m|L",
+ True,
+ True,
+ ), # mtp in the second last stage with no other layers
],
)
def test_forward_vpp(create_args, tmp_path_dist_ckpt, tp_pp_vpp, pp_layout, is_moe, with_mtp):
diff --git a/tests/unit_tests/pipeline_parallel/test_schedules.py b/tests/unit_tests/pipeline_parallel/test_schedules.py
index b861aa2df49..86b9219fe0f 100644
--- a/tests/unit_tests/pipeline_parallel/test_schedules.py
+++ b/tests/unit_tests/pipeline_parallel/test_schedules.py
@@ -1,3 +1,5 @@
+# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved.
+
import os
import pytest
@@ -127,6 +129,52 @@ def test_get_pipeline_parallel_order(
for k, v in order_cnt.items():
assert -k in order_cnt and order_cnt[-k] == v
+ layers_per_chunk = 2
+ num_layers_per_chunk = [layers_per_chunk] * num_model_chunks
+ # disable wgrad compute
+ overlapped_order, chunk_id_list = schedule.get_overlap_moe_expert_parallel_comm_order(
+ order, num_layers_per_chunk, False
+ )
+ assert max(overlapped_order) == num_model_chunks * layers_per_chunk
+ assert len(overlapped_order) == len(order) * layers_per_chunk
+ assert len(chunk_id_list) == len(overlapped_order)
+ order_cnt = {}
+ accumulated_order = 0
+ for o in overlapped_order:
+ order_cnt[o] = order_cnt.get(o, 0) + 1
+ if o < 0:
+ assert -o in order_cnt and order_cnt[-o] >= order_cnt[o]
+ elif -o in order_cnt:
+ assert order_cnt[-o] < order_cnt[o]
+ accumulated_order += o
+ assert accumulated_order >= 0
+ assert accumulated_order == 0
+
+ # enable wgrad compute
+ overlapped_order, chunk_id_list = schedule.get_overlap_moe_expert_parallel_comm_order(
+ order, num_layers_per_chunk, True
+ )
+ assert max(overlapped_order) == num_model_chunks * layers_per_chunk
+ assert len(overlapped_order) == len(order) * layers_per_chunk * 3 // 2
+ assert len(chunk_id_list) == len(overlapped_order)
+ from math import ceil
+
+ order_cnt = {}
+ accumulated_order = 0
+ prev_o = 0
+ for o in overlapped_order:
+ if ceil(o) != o:
+ assert prev_o - 0.5 == o
+ else:
+ order_cnt[o] = order_cnt.get(o, 0) + 1
+ if o < 0:
+ assert -o in order_cnt and order_cnt[-o] >= order_cnt[o]
+ elif -o in order_cnt:
+ assert order_cnt[-o] < order_cnt[o]
+ accumulated_order += o
+ prev_o = o
+ assert accumulated_order < 0
+
Utils.destroy_model_parallel()
diff --git a/tests/unit_tests/post_training/test_modelopt_module_spec.py b/tests/unit_tests/post_training/test_modelopt_module_spec.py
index ec80fcb1a72..dac96785bc0 100644
--- a/tests/unit_tests/post_training/test_modelopt_module_spec.py
+++ b/tests/unit_tests/post_training/test_modelopt_module_spec.py
@@ -173,6 +173,7 @@ def setup_method(self, method):
moe_ffn_hidden_size=128,
moe_shared_expert_intermediate_size=128,
qk_layernorm=True,
+ qk_l2_norm=True,
use_cpu_initialization=True,
)
default_spec = get_gpt_decoder_block_spec(
diff --git a/tests/unit_tests/ssm/test_gated_delta_net.py b/tests/unit_tests/ssm/test_gated_delta_net.py
new file mode 100644
index 00000000000..81f8eed0574
--- /dev/null
+++ b/tests/unit_tests/ssm/test_gated_delta_net.py
@@ -0,0 +1,194 @@
+# Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+
+from functools import partial
+from unittest import mock
+
+import pytest
+import torch
+import torch.nn.functional as F
+
+from megatron.core import parallel_state
+from megatron.core.models.common.embeddings.rope_utils import (
+ get_pos_emb_on_this_cp_rank as get_tensor_on_this_cp_rank,
+)
+from megatron.core.models.gpt.experimental_attention_variant_module_specs import (
+ get_experimental_attention_variant_module_spec,
+ get_transformer_block_with_experimental_attention_variant_spec,
+)
+from megatron.core.models.gpt.gpt_model import GPTModel
+from megatron.core.process_groups_config import ProcessGroupCollection
+from megatron.core.ssm.gated_delta_net import GatedDeltaNet
+from megatron.core.tensor_parallel.random import model_parallel_cuda_manual_seed
+from megatron.core.transformer import TransformerConfig
+from megatron.training.arguments import parse_args
+from megatron.training.checkpointing import load_checkpoint, save_checkpoint
+from megatron.training.global_vars import set_args
+from megatron.training.training import get_model
+from megatron.training.utils import unwrap_model
+from tests.unit_tests.dist_checkpointing import (
+ TempNamedDir,
+ init_basic_mock_args,
+ init_checkpointing_mock_args,
+)
+from tests.unit_tests.test_utilities import Utils
+from tests.unit_tests.transformer.test_attention import _test_parallel_attention_correctness
+
+try:
+ import fla
+
+ HAVE_FLA = True
+except ImportError:
+ HAVE_FLA = False
+
+
+@pytest.mark.parametrize(
+ ("tp_size", "sp", "cp_size"),
+ [(1, False, 1), (2, False, 1), (2, True, 1), (1, False, 2), (2, False, 2), (2, True, 2)],
+)
+@pytest.mark.skipif(not HAVE_FLA, reason="FLA is not installed.")
+@pytest.mark.internal
+class TestGatedDeltaNet:
+
+ @pytest.fixture(scope='function', autouse=True)
+ def setup_method(self, tp_size, sp, cp_size):
+ # Initialize parallel and random seed
+ Utils.initialize_model_parallel(
+ tensor_model_parallel_size=tp_size,
+ pipeline_model_parallel_size=1,
+ context_parallel_size=cp_size,
+ )
+ model_parallel_cuda_manual_seed(123)
+ self.tp_size = tp_size
+ self.cp_size = cp_size
+ self.sp_size = tp_size if sp else 1
+
+ # Get TP and CP process groups from device mesh
+ tp_group = parallel_state.get_tensor_model_parallel_group()
+ cp_group = parallel_state.get_context_parallel_group()
+ pg_collection = ProcessGroupCollection(tp=tp_group, cp=cp_group)
+
+ # Initialize model
+ self.transformer_config = TransformerConfig(
+ hidden_size=256,
+ linear_conv_kernel_dim=2,
+ linear_key_head_dim=64,
+ linear_value_head_dim=64,
+ linear_num_key_heads=4,
+ linear_num_value_heads=8,
+ num_layers=1,
+ normalization="RMSNorm",
+ use_cpu_initialization=True,
+ layernorm_zero_centered_gamma=True,
+ num_attention_heads=8,
+ activation_func=F.silu,
+ bf16=True,
+ tensor_model_parallel_size=tp_size,
+ sequence_parallel=sp,
+ context_parallel_size=cp_size,
+ experimental_attention_variant="gated_delta_net",
+ linear_attention_freq=[1],
+ transformer_impl="transformer_engine",
+ )
+ gdn_submodules = get_experimental_attention_variant_module_spec(
+ config=self.transformer_config
+ ).submodules
+
+ self.gdn = GatedDeltaNet(
+ self.transformer_config,
+ submodules=gdn_submodules,
+ layer_number=1,
+ bias=False,
+ conv_bias=False,
+ conv_init=1.0,
+ use_qk_l2norm=True,
+ A_init_range=(1, 16),
+ pg_collection=pg_collection,
+ )
+ self.gdn = self.gdn.cuda().bfloat16()
+
+ def teardown_method(self):
+ Utils.destroy_model_parallel()
+
+ def test_gpu_forward(self):
+ gdn = self.gdn
+
+ micro_batch_size = 2
+ seq_length = 64
+ hidden_states = torch.ones(
+ (seq_length // self.sp_size // self.cp_size, micro_batch_size, gdn.config.hidden_size),
+ device=torch.cuda.current_device(),
+ dtype=torch.bfloat16,
+ )
+ attention_mask = None
+
+ output, bias = gdn(hidden_states, attention_mask)
+
+ assert output.dim() == 3, f"Output too many dimensions ({output.shape=})"
+ assert output.shape[0] == seq_length // self.sp_size // self.cp_size, (
+ f"Output shape {output.shape[0]=} mismatch with "
+ f" {seq_length=} // {self.sp_size=} // {self.cp_size=}."
+ )
+ assert (
+ output.shape[1] == micro_batch_size
+ ), f"Output shape {output.shape[1]=} mismatch with {micro_batch_size=}"
+ assert (
+ output.shape[2] == gdn.config.hidden_size
+ ), f"Output shape {output.shape[2]=} mismatch with {gdn.config.hidden_size=}"
+ assert (
+ output.dtype == hidden_states.dtype
+ ), f"Output dtype {output.dtype=} mismatch with {hidden_states.dtype=}"
+
+
+@pytest.mark.parametrize(
+ ("tp", "sp", "cp"),
+ [
+ (4, False, 1), # TP w/o SP
+ (4, True, 1), # TP w/ SP
+ (1, False, 2), # CP
+ (2, False, 2), # TP w/o SP + CP
+ (2, True, 2), # TP w/ SP + CP
+ ],
+)
+@pytest.mark.skipif(not HAVE_FLA, reason="FLA is not installed.")
+def test_parallel_gated_delta_net_correctness(tmp_path_dist_ckpt, tp, sp, cp):
+ transformer_config = TransformerConfig(
+ hidden_size=128,
+ linear_conv_kernel_dim=2,
+ linear_key_head_dim=32,
+ linear_value_head_dim=32,
+ linear_num_key_heads=4,
+ linear_num_value_heads=8,
+ num_layers=1,
+ normalization="RMSNorm",
+ use_cpu_initialization=True,
+ layernorm_zero_centered_gamma=True,
+ num_attention_heads=8,
+ activation_func=F.silu,
+ bf16=True,
+ experimental_attention_variant="gated_delta_net",
+ linear_attention_freq=[1],
+ transformer_impl="transformer_engine",
+ )
+
+ transformer_layer_spec = get_transformer_block_with_experimental_attention_variant_spec(
+ config=transformer_config, vp_stage=None, pp_rank=0
+ )
+
+ if cp:
+ atol, rtol = 5e-3, 5e-3
+ else:
+ atol, rtol = 5e-4, 5e-4
+
+ _test_parallel_attention_correctness(
+ transformer_config=transformer_config,
+ transformer_layer_spec=transformer_layer_spec,
+ tmp_path_dist_ckpt=tmp_path_dist_ckpt,
+ atol=atol,
+ rtol=rtol,
+ tp=tp,
+ sp=sp,
+ cp=cp,
+ seed=123,
+ sequence_length=256,
+ micro_batch_size=4,
+ )
diff --git a/tests/unit_tests/tensor_parallel/test_random.py b/tests/unit_tests/tensor_parallel/test_random.py
index 47b607b8795..a15ad83cb90 100644
--- a/tests/unit_tests/tensor_parallel/test_random.py
+++ b/tests/unit_tests/tensor_parallel/test_random.py
@@ -1,3 +1,5 @@
+# Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+
import pytest
import torch
@@ -5,6 +7,7 @@
CheckpointWithoutOutput,
CudaRNGStatesTracker,
checkpoint,
+ convert_cuda_rng_state,
get_cuda_rng_tracker,
model_parallel_cuda_manual_seed,
)
@@ -33,6 +36,148 @@ def test_cuda_rng_states_tracker():
assert torch.equal(rng_tracker.get_states()['state2'], rng_state)
+@pytest.mark.parametrize("use_cudagraphable_rng", [True, False])
+def test_double_fork_cuda_rng_states_tracker(use_cudagraphable_rng):
+ rng_tracker = CudaRNGStatesTracker(use_cudagraphable_rng=use_cudagraphable_rng)
+ rng_tracker.add("state1", 1234)
+ rng_tracker.add("state2", 5678)
+ randn_double_fork_1 = []
+ randn_double_fork_2 = []
+ with rng_tracker.fork("state1"):
+ randn_double_fork_1.append(torch.randn(10, device="cuda"))
+ with rng_tracker.fork("state2"):
+ randn_double_fork_2.append(torch.randn(10, device="cuda"))
+ with rng_tracker.fork("state1"):
+ randn_double_fork_1.append(torch.randn(10, device="cuda"))
+ randn_double_fork_2.append(torch.randn(10, device="cuda"))
+ randn_double_fork_1.append(torch.randn(10, device="cuda"))
+ if use_cudagraphable_rng:
+ double_fork_state1 = rng_tracker.get_states()["state1"].get_state()
+ double_fork_state2 = rng_tracker.get_states()["state2"].get_state()
+ else:
+ double_fork_state1 = rng_tracker.get_states()["state1"]
+ double_fork_state2 = rng_tracker.get_states()["state2"]
+
+ rng_tracker.reset()
+ rng_tracker.add("state1", 1234)
+ rng_tracker.add("state2", 5678)
+ randn_single_fork_1 = []
+ randn_single_fork_2 = []
+ with rng_tracker.fork("state1"):
+ randn_single_fork_1.append(torch.randn(10, device="cuda"))
+ randn_single_fork_1.append(torch.randn(10, device="cuda"))
+ randn_single_fork_1.append(torch.randn(10, device="cuda"))
+ with rng_tracker.fork("state2"):
+ randn_single_fork_2.append(torch.randn(10, device="cuda"))
+ randn_single_fork_2.append(torch.randn(10, device="cuda"))
+ if use_cudagraphable_rng:
+ single_fork_state1 = rng_tracker.get_states()["state1"].get_state()
+ single_fork_state2 = rng_tracker.get_states()["state2"].get_state()
+ else:
+ single_fork_state1 = rng_tracker.get_states()["state1"]
+ single_fork_state2 = rng_tracker.get_states()["state2"]
+
+ assert torch.equal(randn_double_fork_1[0], randn_single_fork_1[0])
+ assert torch.equal(randn_double_fork_1[1], randn_single_fork_1[1])
+ assert torch.equal(randn_double_fork_1[2], randn_single_fork_1[2])
+ assert torch.equal(randn_double_fork_2[0], randn_single_fork_2[0])
+ assert torch.equal(randn_double_fork_2[1], randn_single_fork_2[1])
+ assert torch.equal(double_fork_state1, single_fork_state1)
+ assert torch.equal(double_fork_state2, single_fork_state2)
+
+
+def test_convert_cuda_rng_state():
+ ## Get the default rng state
+ torch.cuda.manual_seed(999)
+ randn = torch.randn(10, device="cuda")
+ rng_state = torch.cuda.get_rng_state()
+
+ try:
+ from megatron.core.extensions.transformer_engine import TECudaRNGStatesTracker
+ except ImportError:
+ TECudaRNGStatesTracker = None
+
+ ## from non-graphable RNG to graphable RNG
+ # get state from non-graphable RNG
+ tracker = CudaRNGStatesTracker(use_cudagraphable_rng=False)
+ tracker.add("state1", 123)
+ for i in range(3):
+ with tracker.fork("state1"):
+ randn = torch.randn(10, device="cuda")
+ state = convert_cuda_rng_state(tracker.states_["state1"], to_graphable=True)
+ rand_tensors = []
+ for i in range(3):
+ with tracker.fork("state1"):
+ randn = torch.randn(10, device="cuda")
+ rand_tensors.append(randn)
+
+ # set state to local graph RNG
+ cudagraphable_tracker = CudaRNGStatesTracker(use_cudagraphable_rng=True)
+ cudagraphable_tracker.set_states({"state1": state.clone_state()})
+ for i in range(3):
+ with cudagraphable_tracker.fork("state1"):
+ randn = torch.randn(10, device="cuda")
+ assert torch.equal(randn, rand_tensors[i])
+
+ # set state to TE RNG
+ if TECudaRNGStatesTracker is not None:
+ te_tracker = TECudaRNGStatesTracker()
+ te_tracker.set_states({"state1": state})
+ for i in range(3):
+ with te_tracker.fork("state1"):
+ randn = torch.randn(10, device="cuda")
+ assert torch.equal(randn, rand_tensors[i])
+
+ ## from graphable RNG to non-graphable RNG
+ # get state from graphable RNG
+ cudagraphable_tracker = CudaRNGStatesTracker(use_cudagraphable_rng=True)
+ cudagraphable_tracker.add("state2", 123)
+ for i in range(3):
+ with cudagraphable_tracker.fork("state2"):
+ randn = torch.randn(10, device="cuda")
+ state = convert_cuda_rng_state(cudagraphable_tracker.states_["state2"], to_graphable=False)
+ rand_tensors = []
+ for i in range(3):
+ with cudagraphable_tracker.fork("state2"):
+ randn = torch.randn(10, device="cuda")
+ rand_tensors.append(randn)
+
+ # set state to non-graphable RNG
+ tracker = CudaRNGStatesTracker(use_cudagraphable_rng=False)
+ tracker.set_states({"state2": state})
+ for i in range(3):
+ with tracker.fork("state2"):
+ randn = torch.randn(10, device="cuda")
+ assert torch.equal(randn, rand_tensors[i])
+
+ ## from TE RNG to non-graphable RNG
+ if TECudaRNGStatesTracker is not None:
+ # get state from TE RNG
+ cudagraphable_tracker = TECudaRNGStatesTracker()
+ cudagraphable_tracker.add("state3", 123)
+ for i in range(3):
+ with cudagraphable_tracker.fork("state3"):
+ randn = torch.randn(10, device="cuda")
+ state = convert_cuda_rng_state(cudagraphable_tracker.states_["state3"], to_graphable=False)
+ rand_tensors = []
+ for i in range(3):
+ with cudagraphable_tracker.fork("state3"):
+ randn = torch.randn(10, device="cuda")
+ rand_tensors.append(randn)
+
+ # set state to non-graphable RNG
+ tracker = CudaRNGStatesTracker(use_cudagraphable_rng=False)
+ tracker.set_states({"state3": state})
+ for i in range(3):
+ with tracker.fork("state3"):
+ randn = torch.randn(10, device="cuda")
+ assert torch.equal(randn, rand_tensors[i])
+
+ ## After all tests, check if the default rng state is still the same.
+ rng_state_final = torch.cuda.get_rng_state()
+ assert torch.equal(rng_state, rng_state_final)
+
+
def test_model_parallel_cuda_manual_seed():
Utils.initialize_model_parallel(4, 2)
model_parallel_cuda_manual_seed(0, force_reset_rng=True)
diff --git a/tests/unit_tests/tensor_parallel/test_tp_attrs_without_init.py b/tests/unit_tests/tensor_parallel/test_tp_attrs_without_init.py
new file mode 100644
index 00000000000..f7a518e8e88
--- /dev/null
+++ b/tests/unit_tests/tensor_parallel/test_tp_attrs_without_init.py
@@ -0,0 +1,87 @@
+import pytest
+import torch
+
+from megatron.core.tensor_parallel.layers import (
+ ColumnParallelLinear,
+ RowParallelLinear,
+ VocabParallelEmbedding,
+)
+from megatron.core.transformer.transformer_config import TransformerConfig
+from tests.unit_tests.test_utilities import Utils
+
+
+class TestTPAttributesWithoutInitialization:
+
+ def teardown_method(self, method):
+ Utils.destroy_model_parallel()
+
+ @pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA not available")
+ @pytest.mark.parametrize("use_cpu_init", [True, False])
+ def test_vocab_parallel_embedding_tp_attrs_no_init(self, use_cpu_init):
+ Utils.initialize_model_parallel(tensor_model_parallel_size=2)
+ cfg = TransformerConfig(
+ num_layers=1,
+ hidden_size=8,
+ num_attention_heads=4,
+ use_cpu_initialization=use_cpu_init,
+ perform_initialization=False,
+ )
+
+ emb = VocabParallelEmbedding(
+ num_embeddings=16, embedding_dim=8, init_method=cfg.init_method, config=cfg
+ )
+ w = emb.weight
+ assert hasattr(w, "tensor_model_parallel") and w.tensor_model_parallel is True
+ assert hasattr(w, "partition_dim") and w.partition_dim == 0
+ assert hasattr(w, "partition_stride") and w.partition_stride == 1
+
+ @pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA not available")
+ @pytest.mark.parametrize("use_cpu_init", [True, False])
+ def test_column_parallel_linear_tp_attrs_no_init(self, use_cpu_init):
+ Utils.initialize_model_parallel(tensor_model_parallel_size=2)
+ cfg = TransformerConfig(
+ num_layers=1,
+ hidden_size=8,
+ num_attention_heads=4,
+ use_cpu_initialization=use_cpu_init,
+ perform_initialization=False,
+ )
+
+ layer = ColumnParallelLinear(
+ input_size=8,
+ output_size=8,
+ init_method=cfg.init_method,
+ bias=True,
+ config=cfg,
+ skip_bias_add=False,
+ )
+ w = layer.weight
+ assert hasattr(w, "tensor_model_parallel") and w.tensor_model_parallel is True
+ assert hasattr(w, "partition_dim") and w.partition_dim == 0
+ assert hasattr(w, "partition_stride") and w.partition_stride == 1
+
+ @pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA not available")
+ @pytest.mark.parametrize("use_cpu_init", [True, False])
+ def test_row_parallel_linear_tp_attrs_no_init(self, use_cpu_init):
+ Utils.initialize_model_parallel(tensor_model_parallel_size=2)
+ cfg = TransformerConfig(
+ num_layers=1,
+ hidden_size=8,
+ num_attention_heads=4,
+ use_cpu_initialization=use_cpu_init,
+ perform_initialization=False,
+ )
+
+ layer = RowParallelLinear(
+ input_size=8,
+ output_size=8,
+ init_method=cfg.init_method,
+ bias=True,
+ input_is_parallel=True,
+ config=cfg,
+ skip_bias_add=False,
+ )
+ w = layer.weight
+ assert hasattr(w, "tensor_model_parallel") and w.tensor_model_parallel is True
+ assert hasattr(w, "partition_dim") and w.partition_dim == 1
+ assert hasattr(w, "partition_stride") and w.partition_stride == 1
diff --git a/tests/unit_tests/test_fp8_param.py b/tests/unit_tests/test_fp8_param.py
index 0b8d41769ec..59824478bdc 100644
--- a/tests/unit_tests/test_fp8_param.py
+++ b/tests/unit_tests/test_fp8_param.py
@@ -1,4 +1,4 @@
-# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved.
+# Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
import contextlib
import gc
diff --git a/tests/unit_tests/test_inference.py b/tests/unit_tests/test_inference.py
index 518aa7f4126..5b2bc07e6e1 100644
--- a/tests/unit_tests/test_inference.py
+++ b/tests/unit_tests/test_inference.py
@@ -1,3 +1,5 @@
+# Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+
import argparse
import unittest.mock
diff --git a/tests/unit_tests/test_layer_wise_optimizer.py b/tests/unit_tests/test_layer_wise_optimizer.py
new file mode 100644
index 00000000000..05ce26bcfa0
--- /dev/null
+++ b/tests/unit_tests/test_layer_wise_optimizer.py
@@ -0,0 +1,440 @@
+# Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+
+import os
+
+import pytest
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+from packaging.version import Version
+
+from megatron.core import parallel_state
+from megatron.core.distributed import DistributedDataParallel, DistributedDataParallelConfig
+from megatron.core.optimizer import OptimizerConfig, get_megatron_optimizer
+from megatron.core.optimizer.layer_wise_optimizer import LayerWiseDistributedOptimizer
+from megatron.core.optimizer.optimizer import Float16OptimizerWithFloat16Params, FP32Optimizer
+from megatron.core.process_groups_config import ProcessGroupCollection
+from megatron.core.transformer import TransformerConfig
+from megatron.core.utils import get_pg_size
+from tests.unit_tests.test_utilities import Utils
+
+# Skip all tests in this file for LTS versions
+pytestmark = pytest.mark.skipif(
+ Version(os.getenv('NVIDIA_PYTORCH_VERSION', "24.01")) <= Version("25.05"),
+ reason="Skip layer-wise optimizer for LTS test",
+)
+
+
+class SimpleModel(nn.Module):
+ """Simple model for testing LayerWiseDistributedOptimizer.
+
+ Model with 5 layers to ensure more than 8 parameters (10 total: 5 weights + 5 biases).
+ """
+
+ def __init__(self, input_size=80, hidden_size=48, output_size=10):
+ super().__init__()
+ self.fc1 = nn.Linear(input_size, hidden_size)
+ self.fc2 = nn.Linear(hidden_size, 32)
+ self.fc3 = nn.Linear(32, 24)
+ self.fc4 = nn.Linear(24, 16)
+ self.fc5 = nn.Linear(16, output_size)
+
+ def forward(self, x):
+ x = F.relu(self.fc1(x))
+ x = F.relu(self.fc2(x))
+ x = F.relu(self.fc3(x))
+ x = F.relu(self.fc4(x))
+ x = self.fc5(x)
+ return x
+
+
+class TinyModel(nn.Module):
+ """Tiny model with only 1 layer (2 parameters: weight and bias)."""
+
+ def __init__(self):
+ super().__init__()
+ self.fc1 = nn.Linear(10, 5)
+
+ def forward(self, x):
+ return self.fc1(x)
+
+
+@pytest.mark.skipif(
+ int(os.getenv('WORLD_SIZE', '1')) == 1, reason="Multi-rank test requires WORLD_SIZE > 1"
+)
+class TestLayerWiseOptimizer:
+ """Test class for LayerWiseDistributedOptimizer with common setup code."""
+
+ @pytest.fixture(autouse=True)
+ def setup_and_teardown(self):
+ """Setup and teardown for each test."""
+ world = int(os.getenv('WORLD_SIZE', '1'))
+ rank = int(os.getenv('RANK', '0'))
+ Utils.initialize_model_parallel()
+ yield
+ Utils.destroy_model_parallel()
+
+ def create_model_and_optimizer(
+ self,
+ model_class=SimpleModel,
+ clip_grad=1.0,
+ model_kwargs=None,
+ use_layer_wise=True,
+ copy_from=None,
+ ):
+ """Create model, DDP wrapper, and optimizer.
+
+ Args:
+ model_class: Model class to instantiate
+ clip_grad: Optional gradient clipping value
+ model_kwargs: Optional kwargs for model initialization
+ use_layer_wise: If True, wrap optimizer in LayerWiseDistributedOptimizer;
+ if False, use get_megatron_optimizer instead (for reference)
+
+ Returns:
+ tuple: (model, optimizer, pg_collection)
+ """
+ if model_kwargs is None:
+ model_kwargs = {}
+
+ model = model_class(**model_kwargs).bfloat16().cuda()
+ model.requires_grad_(True)
+
+ ddp_config = DistributedDataParallelConfig(use_distributed_optimizer=False)
+ model = DistributedDataParallel(
+ TransformerConfig(num_attention_heads=1, num_layers=1), ddp_config, model
+ )
+ if copy_from:
+ model.module.load_state_dict(copy_from.module.state_dict())
+ else:
+ model.broadcast_params()
+
+ optimizer_config = OptimizerConfig(
+ optimizer='adam',
+ lr=0.01,
+ weight_decay=0.01,
+ bf16=not use_layer_wise,
+ use_distributed_optimizer=False,
+ clip_grad=clip_grad,
+ )
+
+ pg_collection = ProcessGroupCollection.use_mpu_process_groups()
+ pg_collection.dp_cp = parallel_state.get_data_parallel_group(with_context_parallel=True)
+ pg_collection.expt_dp = parallel_state.get_expert_data_parallel_group()
+
+ optimizer = get_megatron_optimizer(optimizer_config, [model])
+ if use_layer_wise:
+ optimizer_config.bf16 = True
+ optimizer = LayerWiseDistributedOptimizer(
+ optimizer.chained_optimizers, optimizer_config, pg_collection
+ )
+ return model, optimizer, pg_collection
+
+ def create_reference_model(self, model):
+ """Create a reference model by cloning the current model."""
+ reference_model = type(model.module)().bfloat16().cuda()
+ reference_model.load_state_dict(model.module.state_dict())
+ return reference_model
+
+ def test_basic(self):
+ """Test basic LayerWiseDistributedOptimizer initialization and step with bf16."""
+ model, optimizer, pg_collection = self.create_model_and_optimizer()
+
+ # Verify basic properties
+ assert optimizer is not None, "Optimizer should not be None"
+ assert hasattr(optimizer, 'chained_optimizers'), "Should be a ChainedOptimizer"
+
+ reference_model = self.create_reference_model(model)
+
+ input_tensor = torch.randn(16, 80, dtype=torch.bfloat16, device='cuda')
+ output = model(input_tensor)
+ loss = output.sum()
+ loss.backward()
+
+ update_successful, grad_norm, num_zeros = optimizer.step()
+
+ assert update_successful, "Optimizer step should be successful"
+
+ # Verify parameters were updated
+ params_updated = 0
+ for param, ref_param in zip(model.parameters(), reference_model.parameters()):
+ if not torch.equal(param.data, ref_param.data):
+ params_updated += 1
+
+ assert params_updated > 0, "At least some parameters should be updated"
+
+ # Verify all ranks have the same updated parameters (test allgather)
+ dp_size = get_pg_size(pg_collection.dp_cp)
+
+ if dp_size > 1:
+ for name, param in model.named_parameters():
+ # Gather parameters from all ranks
+ param_list = [torch.zeros_like(param.data) for _ in range(dp_size)]
+ torch.distributed.all_gather(param_list, param.data, group=pg_collection.dp_cp)
+
+ # Verify all ranks have the same parameter values
+ for i in range(1, dp_size):
+ try:
+ torch.testing.assert_close(param_list[0], param_list[i])
+ except AssertionError as e:
+ # Append additional context without overwriting the default message
+ raise AssertionError(
+ f"Parameter {name} differs between rank 0 and rank {i}. {str(e)}"
+ ) from None
+
+ def test_get_grad_norm(self):
+ """Test LayerWiseDistributedOptimizer gradient norm computation."""
+ model, optimizer, pg_collection = self.create_model_and_optimizer()
+ reference_model, reference_optimizer, _ = self.create_model_and_optimizer(
+ use_layer_wise=False
+ )
+
+ # Set same gradients on both models
+ # note that model is different at this point but we're only testing grad norm here
+ for param, ref_param in zip(model.parameters(), reference_model.parameters()):
+ grad_value = torch.randn_like(param)
+ torch.distributed.broadcast(grad_value, src=0, group=pg_collection.dp_cp)
+ param.main_grad = grad_value.float().detach()
+ ref_param.main_grad = grad_value.float().detach()
+
+ # Test get_grad_norm on both optimizers
+ optimizer.prepare_grads()
+ grad_norm = optimizer.get_grad_norm()
+
+ reference_optimizer.prepare_grads()
+ reference_grad_norm = reference_optimizer.get_grad_norm()
+
+ assert grad_norm is not None, "Grad norm should not be None"
+ assert grad_norm >= 0, "Grad norm should be non-negative"
+
+ # Compare with reference optimizer grad norm
+ torch.testing.assert_close(grad_norm, reference_grad_norm, rtol=1e-5, atol=1e-5)
+
+ def test_state_dict(self):
+ """Test LayerWiseDistributedOptimizer state dict save and load."""
+ model, optimizer, pg_collection = self.create_model_and_optimizer()
+
+ for param in model.parameters():
+ param.grad = torch.randn_like(param)
+ optimizer.step()
+
+ # Test state_dict
+ state_dict = optimizer.state_dict()
+
+ # Test load_state_dict
+ # TODO(deyuf): fix this. not going through get() will cause missing keys like wd_mult
+ # optimizer.load_state_dict(state_dict)
+
+ def test_sharded_state_dict(self):
+ """Test LayerWiseDistributedOptimizer sharded_state_dict method."""
+ model, optimizer, pg_collection = self.create_model_and_optimizer()
+
+ for param in model.parameters():
+ param.grad = torch.randn_like(param)
+ optimizer.step()
+
+ # Get model sharded state dict
+ model_sharded_state_dict = model.sharded_state_dict()
+
+ # Test sharded_state_dict
+ sharded_state_dict = optimizer.sharded_state_dict(model_sharded_state_dict)
+
+ # Verify the sharded_state_dict is not None and has expected structure
+ assert sharded_state_dict is not None, "Sharded state dict should not be None"
+ assert (
+ 'optimizer' in sharded_state_dict
+ ), "Sharded state dict should contain 'optimizer' key"
+
+ # Verify that replica_id is set correctly (should be 0 for DP dimension)
+ from megatron.core.dist_checkpointing import ShardedTensor
+ from megatron.core.dist_checkpointing.dict_utils import nested_values
+
+ for sh_base in nested_values(sharded_state_dict):
+ if isinstance(sh_base, ShardedTensor):
+ assert (
+ len(sh_base.replica_id) == 3
+ ), f'Expected replica_id format (PP, TP, DP), got: {sh_base.replica_id}'
+ assert (
+ sh_base.replica_id[2] == 0
+ ), f'Expected DP replica_id to be 0 for layer-wise optimizer, got: {sh_base.replica_id[2]}'
+
+ def test_multiple_optimizers(self):
+ """Test LayerWiseDistributedOptimizer with multiple chained optimizers.
+
+ This test properly tests allgather functionality with multiple ranks.
+ """
+ model = SimpleModel().bfloat16().cuda()
+ model.requires_grad_(True)
+
+ ddp_config = DistributedDataParallelConfig(use_distributed_optimizer=False)
+ model = DistributedDataParallel(
+ TransformerConfig(num_attention_heads=1, num_layers=1), ddp_config, model
+ )
+
+ optimizer_config = OptimizerConfig(
+ optimizer='adam', lr=0.01, bf16=True, use_distributed_optimizer=False
+ )
+
+ # Split parameters into two groups for testing multiple optimizers
+ params = list(model.parameters())
+ mid_point = len(params) // 2
+ param_groups_1 = [{'params': params[:mid_point]}]
+ param_groups_2 = [{'params': params[mid_point:]}]
+
+ # Create two separate base optimizers
+ base_optimizer_1 = torch.optim.Adam(param_groups_1, lr=optimizer_config.lr)
+ base_optimizer_2 = torch.optim.Adam(param_groups_2, lr=optimizer_config.lr)
+
+ wrapped_optimizer_1 = FP32Optimizer(base_optimizer_1, optimizer_config, None)
+ wrapped_optimizer_2 = FP32Optimizer(base_optimizer_2, optimizer_config, None)
+
+ pg_collection = ProcessGroupCollection.use_mpu_process_groups()
+ pg_collection.dp_cp = parallel_state.get_data_parallel_group(with_context_parallel=True)
+ pg_collection.expt_dp = parallel_state.get_expert_data_parallel_group()
+
+ optimizer = LayerWiseDistributedOptimizer(
+ [wrapped_optimizer_1, wrapped_optimizer_2], optimizer_config, pg_collection
+ )
+
+ assert len(optimizer.chained_optimizers) == 2, "Should have two chained optimizers"
+
+ # Set gradients and test optimizer step - this will trigger allgather
+ for param in model.parameters():
+ param.grad = torch.randn_like(param)
+
+ update_successful, grad_norm, num_zeros = optimizer.step()
+
+ assert update_successful, "Optimizer step should be successful"
+
+ def test_bf16_wrapping(self):
+ """Test LayerWiseDistributedOptimizer automatically wraps optimizer with bf16."""
+ model, optimizer, pg_collection = self.create_model_and_optimizer()
+
+ # Verify bf16 wrapping happened
+ assert isinstance(
+ optimizer.chained_optimizers[0], Float16OptimizerWithFloat16Params
+ ), "Optimizer should be wrapped in Float16OptimizerWithFloat16Params"
+
+ for param in model.parameters():
+ param.grad = torch.randn_like(param)
+
+ update_successful, grad_norm, num_zeros = optimizer.step()
+
+ assert update_successful, "Optimizer step should be successful"
+
+ def test_bf16_error(self):
+ """Test LayerWiseDistributedOptimizer raises error when receiving pre-wrapped Float16 optimizer."""
+ model = SimpleModel().bfloat16().cuda()
+ model.requires_grad_(True)
+
+ ddp_config = DistributedDataParallelConfig(use_distributed_optimizer=False)
+ model = DistributedDataParallel(
+ TransformerConfig(num_attention_heads=1, num_layers=1), ddp_config, model
+ )
+
+ optimizer_config = OptimizerConfig(
+ optimizer='adam', lr=0.01, bf16=True, use_distributed_optimizer=False
+ )
+
+ # Create base optimizer and manually wrap in Float16 optimizer
+ param_groups = [{'params': list(model.parameters())}]
+ base_optimizer = torch.optim.Adam(param_groups, lr=optimizer_config.lr)
+ wrapped_optimizer = Float16OptimizerWithFloat16Params(
+ base_optimizer, optimizer_config, None, None
+ )
+
+ pg_collection = ProcessGroupCollection.use_mpu_process_groups()
+ pg_collection.dp_cp = parallel_state.get_data_parallel_group(with_context_parallel=True)
+ pg_collection.expt_dp = parallel_state.get_expert_data_parallel_group()
+
+ # Should raise TypeError when receiving already-wrapped Float16 optimizer
+ with pytest.raises(
+ TypeError, match='LayerWiseDistributedOptimizer received Float16 optimizer already'
+ ):
+ LayerWiseDistributedOptimizer([wrapped_optimizer], optimizer_config, pg_collection)
+
+ def _run_parameter_update_test(self, model_class=SimpleModel):
+ """Helper method to test parameter updates with a given model class.
+
+ Args:
+ model_class: Model class to use for testing
+ """
+ model, optimizer, pg_collection = self.create_model_and_optimizer(model_class=model_class)
+
+ # Create reference model and optimizer using the same function
+ reference_model, reference_optimizer, _ = self.create_model_and_optimizer(
+ model_class=model_class, use_layer_wise=False, copy_from=model
+ )
+
+ # Set same gradients on both models
+ for param, ref_param in zip(model.parameters(), reference_model.parameters()):
+ assert torch.equal(param.data, ref_param.data)
+ torch.testing.assert_close(param.data, ref_param.data, rtol=1e-5, atol=1e-5)
+ grad_value = torch.randn_like(param)
+ torch.distributed.broadcast(grad_value, src=0, group=pg_collection.dp_cp)
+ param.main_grad = grad_value.clone().detach()
+ ref_param.main_grad = grad_value.clone().detach()
+
+ optimizer.step()
+
+ # Verify at least some parameters were updated
+ params_updated = 0
+ for param, ref_param in zip(model.parameters(), reference_model.parameters()):
+ if not torch.equal(param.data, ref_param.data):
+ params_updated += 1
+
+ assert params_updated > 0, "At least some parameters should be updated"
+
+ reference_optimizer.step()
+
+ # Verify updated values match reference optimizer
+ for param, ref_param in zip(model.parameters(), reference_model.parameters()):
+ torch.testing.assert_close(param.data, ref_param.data, rtol=1e-5, atol=1e-5)
+
+ def test_parameter_updates(self):
+ """Test LayerWiseDistributedOptimizer actually updates model parameters."""
+ self._run_parameter_update_test()
+
+ def test_parameter_updates_insufficient_parameters(self):
+ """Test LayerWiseDistributedOptimizer when there are insufficient parameters for all ranks.
+
+ Uses a tiny model with only 1 layer (2 parameters: weight and bias).
+ This will be insufficient when world size > 2.
+ """
+ self._run_parameter_update_test(model_class=TinyModel)
+
+ def test_broadcast_vs_allgather(self):
+ """Test LayerWiseDistributedOptimizer allgather code agains broadcast code."""
+ model, optimizer, pg_collection = self.create_model_and_optimizer(model_class=SimpleModel)
+
+ # Create reference model and optimizer using the same function
+ reference_model, reference_optimizer, _ = self.create_model_and_optimizer(
+ model_class=SimpleModel, copy_from=model
+ )
+
+ # Set same gradients on both models
+ for param, ref_param in zip(model.parameters(), reference_model.parameters()):
+ assert torch.equal(param.data, ref_param.data)
+ torch.testing.assert_close(param.data, ref_param.data, rtol=0, atol=0)
+ grad_value = torch.randn_like(param)
+ torch.distributed.broadcast(grad_value, src=0, group=pg_collection.dp_cp)
+ param.main_grad = grad_value.clone().detach()
+ ref_param.main_grad = grad_value.clone().detach()
+
+ optimizer.step()
+
+ # Verify at least some parameters were updated
+ params_updated = 0
+ for param, ref_param in zip(model.parameters(), reference_model.parameters()):
+ if not torch.equal(param.data, ref_param.data):
+ params_updated += 1
+
+ assert params_updated > 0, "At least some parameters should be updated"
+
+ # step() internal call allgather_params. replace reference object with bcast
+ reference_optimizer.allgather_params = reference_optimizer.broadcast_params
+ reference_optimizer.step()
+
+ # Verify updated values match reference optimizer
+ for param, ref_param in zip(model.parameters(), reference_model.parameters()):
+ torch.testing.assert_close(param.data, ref_param.data, rtol=0, atol=0)
diff --git a/tests/unit_tests/test_muon_optimizer.py b/tests/unit_tests/test_muon_optimizer.py
new file mode 100644
index 00000000000..cc99f7a16e6
--- /dev/null
+++ b/tests/unit_tests/test_muon_optimizer.py
@@ -0,0 +1,670 @@
+# Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+
+import os
+
+import pytest
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+from packaging.version import Version
+
+from megatron.core import parallel_state
+from megatron.core.distributed import DistributedDataParallel, DistributedDataParallelConfig
+from megatron.core.optimizer import OptimizerConfig
+from megatron.core.optimizer.muon import TensorParallelMuon, get_megatron_muon_optimizer
+from megatron.core.process_groups_config import ProcessGroupCollection
+from megatron.core.transformer import TransformerConfig
+from tests.unit_tests.test_utilities import Utils
+
+# Skip all tests in this file for LTS versions
+pytestmark = pytest.mark.skipif(
+ Version(os.getenv('NVIDIA_PYTORCH_VERSION', "24.01")) <= Version("25.05"),
+ reason="Skip muon optimizer for LTS test",
+)
+
+
+class Net(nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.fc1 = nn.Linear(80, 48)
+ self.fc2 = nn.Linear(48, 32)
+ self.fc3 = nn.Linear(32, 24)
+ self.fc4 = nn.Linear(24, 16)
+ self.fc5 = nn.Linear(16, 10)
+
+ def forward(self, x):
+ x = F.relu(self.fc1(x))
+ x = F.relu(self.fc2(x))
+ x = F.relu(self.fc3(x))
+ x = F.relu(self.fc4(x))
+ x = self.fc5(x)
+ return x
+
+
+def test_muon_optimizer_smoke():
+ """Smoke test for TensorParallelMuon optimizer."""
+ # Create a simple linear model for testing
+ model = torch.nn.Linear(100, 50, bias=False, dtype=torch.float32, device='cuda')
+ model.requires_grad_(True)
+ model.weight.data.fill_(1.0)
+
+ # Create TensorParallelMuon optimizer
+ optimizer = TensorParallelMuon(
+ params=[model.weight],
+ lr=0.01,
+ momentum_beta=0.95,
+ use_nesterov=True,
+ weight_decay=0.01,
+ use_decoupled_weight_decay=True,
+ split_qkv=False,
+ fp32_matmul_prec="medium",
+ num_ns_steps=5,
+ scale_mode="spectral",
+ extra_scale_factor=1.0,
+ pg_collection=None,
+ mode="duplicated",
+ )
+
+ # Test basic properties
+ assert optimizer is not None, "Optimizer should not be None"
+ assert hasattr(optimizer, 'param_groups'), "Optimizer should have param_groups"
+ assert len(optimizer.param_groups) > 0, "Optimizer should have at least one parameter group"
+
+ # Test forward and backward pass
+ input_tensor = torch.randn(32, 100, dtype=torch.float32, device='cuda')
+ output = model(input_tensor)
+ loss = output.sum()
+ loss.backward()
+
+ # Store original weight
+ original_weight = model.weight.data.clone()
+
+ # Test optimizer step
+ optimizer.step()
+
+ # Verify weight was updated
+ assert not torch.equal(
+ model.weight.data, original_weight
+ ), "Weight should be updated after optimizer step"
+
+ # Test zero_grad
+ optimizer.zero_grad()
+ assert model.weight.grad is None or torch.all(
+ model.weight.grad == 0
+ ), "Gradients should be zeroed"
+
+ # Test state_dict and load_state_dict
+ state_dict = optimizer.state_dict()
+ assert 'state' in state_dict, "State dict should contain state"
+ assert 'param_groups' in state_dict, "State dict should contain param_groups"
+
+ # Load state dict should not raise error
+ optimizer.load_state_dict(state_dict)
+
+
+@pytest.mark.skipif(
+ int(os.getenv('WORLD_SIZE', '1')) == 1, reason="Multi-rank test requires WORLD_SIZE > 1"
+)
+class TestMuonOptimizerMultiRank:
+ """Test class for Muon optimizer with multi-rank setup."""
+
+ @pytest.fixture(autouse=True)
+ def setup_and_teardown(self):
+ """Setup and teardown for each test."""
+ Utils.initialize_model_parallel()
+ yield
+ Utils.destroy_model_parallel()
+
+ def create_ddp_model(self, model):
+ """Wrap model in DDP.
+
+ Args:
+ model: Model to wrap
+
+ Returns:
+ DDP-wrapped model
+ """
+ ddp_config = DistributedDataParallelConfig(use_distributed_optimizer=False)
+ return DistributedDataParallel(
+ TransformerConfig(num_attention_heads=1, num_layers=1), ddp_config, model
+ )
+
+ def test_get_megatron_muon_optimizer_smoke(self):
+ """Smoke test for get_megatron_muon_optimizer function."""
+ model = Net().bfloat16().cuda()
+ model.requires_grad_(True)
+ model = self.create_ddp_model(model)
+
+ # Ensure all parameters require gradients
+ for param in model.parameters():
+ assert param.requires_grad, "All parameters should require gradients"
+
+ # Create optimizer config for Muon
+ optimizer_config = OptimizerConfig(
+ optimizer='muon', # This will be changed internally to 'adam' for non-linear params
+ lr=0.01,
+ weight_decay=0.01,
+ bf16=True,
+ use_distributed_optimizer=False, # Muon doesn't support distributed optimizer
+ muon_momentum=0.95,
+ muon_use_nesterov=True,
+ muon_fp32_matmul_prec="medium",
+ muon_num_ns_steps=5,
+ muon_scale_mode="spectral",
+ muon_tp_mode="duplicated",
+ )
+
+ # Test creating the optimizer
+ optimizer = get_megatron_muon_optimizer(
+ config=optimizer_config,
+ model_chunks=[model],
+ use_gloo_process_groups=True,
+ layer_wise_distributed_optimizer=False,
+ )
+
+ # Test basic properties
+ assert optimizer is not None, "Optimizer should not be None"
+ assert hasattr(optimizer, 'param_groups'), "Optimizer should have param_groups"
+ assert hasattr(optimizer, 'chained_optimizers'), "Should be a ChainedOptimizer"
+ assert len(optimizer.chained_optimizers) >= 1, "Should have at least one chained optimizer"
+
+ # Test forward and backward pass
+ input_tensor = torch.randn(16, 80, dtype=torch.bfloat16, device='cuda')
+ output = model(input_tensor)
+ loss = output.sum()
+ loss.backward()
+
+ # Store original parameters
+ original_params = {}
+ for name, param in model.named_parameters():
+ original_params[name] = param.data.clone()
+
+ # Test optimizer step
+ optimizer.step()
+
+ # Verify at least some parameters were updated
+ params_updated = 0
+ for name, param in model.named_parameters():
+ if not torch.equal(param.data, original_params[name]):
+ params_updated += 1
+
+ assert params_updated > 0, "At least some parameters should be updated after optimizer step"
+
+ # Test zero_grad
+ optimizer.zero_grad()
+ for param in model.parameters():
+ assert param.grad is None or torch.all(
+ param.grad == 0
+ ), f"Gradients should be zeroed for all parameters"
+
+ # Test state_dict and load_state_dict
+ state_dict = optimizer.state_dict()
+ assert isinstance(state_dict, list), "State dict should be a list"
+
+ # Load state dict should not raise error
+ optimizer.load_state_dict(state_dict)
+
+ def test_get_megatron_muon_optimizer_validation(self):
+ """Test validation logic for get_megatron_muon_optimizer."""
+ model = torch.nn.Linear(100, 50, bias=False, dtype=torch.bfloat16, device='cuda')
+ model.requires_grad_(True)
+ model = self.create_ddp_model(model)
+
+ # Test 1: Distributed optimizer should raise exception
+ optimizer_config_dist = OptimizerConfig(
+ optimizer='muon',
+ lr=0.01,
+ bf16=True,
+ use_distributed_optimizer=True, # This should cause an exception
+ )
+
+ with pytest.raises(Exception, match='muon with dist optimizer is not supported'):
+ get_megatron_muon_optimizer(config=optimizer_config_dist, model_chunks=[model])
+
+ # Test 2: FP16 should raise exception
+ optimizer_config_fp16 = OptimizerConfig(
+ optimizer='muon',
+ lr=0.01,
+ fp16=True, # This should cause an exception
+ use_distributed_optimizer=False,
+ )
+
+ with pytest.raises(Exception, match='muon with fp16 is not supported'):
+ get_megatron_muon_optimizer(config=optimizer_config_fp16, model_chunks=[model])
+
+ # Test 3: Invalid num_ns_steps should raise exception
+ optimizer_config_invalid_ns = OptimizerConfig(
+ optimizer='muon',
+ lr=0.01,
+ bf16=True,
+ use_distributed_optimizer=False,
+ muon_num_ns_steps=0, # This should cause an exception
+ )
+
+ with pytest.raises(ValueError, match='num_ns_steps must be at least 1'):
+ get_megatron_muon_optimizer(config=optimizer_config_invalid_ns, model_chunks=[model])
+
+ def test_get_megatron_muon_optimizer_layer_wise(self):
+ """Test get_megatron_muon_optimizer with layer-wise distributed optimizer."""
+ model = Net().bfloat16().cuda()
+ model.requires_grad_(True)
+ model = self.create_ddp_model(model)
+
+ optimizer_config = OptimizerConfig(
+ optimizer='muon',
+ lr=0.01,
+ weight_decay=0.01,
+ bf16=True,
+ use_distributed_optimizer=False,
+ muon_momentum=0.95,
+ muon_use_nesterov=True,
+ muon_fp32_matmul_prec="medium",
+ muon_num_ns_steps=5,
+ muon_scale_mode="spectral",
+ muon_tp_mode="duplicated",
+ )
+
+ # Test with layer_wise_distributed_optimizer=True
+ optimizer = get_megatron_muon_optimizer(
+ config=optimizer_config,
+ model_chunks=[model],
+ use_gloo_process_groups=True,
+ layer_wise_distributed_optimizer=True,
+ )
+
+ # Verify it's a LayerWiseDistributedOptimizer
+ from megatron.core.optimizer.layer_wise_optimizer import LayerWiseDistributedOptimizer
+
+ assert isinstance(
+ optimizer, LayerWiseDistributedOptimizer
+ ), "Should return LayerWiseDistributedOptimizer"
+
+ # Test forward and backward pass
+ input_tensor = torch.randn(16, 80, dtype=torch.bfloat16, device='cuda')
+ output = model(input_tensor)
+ loss = output.sum()
+ loss.backward()
+
+ # Test optimizer step
+ update_successful, grad_norm, num_zeros = optimizer.step()
+
+ assert update_successful, "Optimizer step should be successful"
+ assert grad_norm is not None or grad_norm is None, "Grad norm should be returned"
+
+
+@pytest.mark.parametrize("mode", ["duplicated", "blockwise", "distributed"])
+def test_muon_optimizer_different_modes_single_rank(mode):
+ """Test TensorParallelMuon optimizer with different modes on single rank.
+
+ When TP size is 1, all modes should produce the same result.
+ """
+ # Set random seed for reproducibility
+ torch.manual_seed(42)
+ torch.cuda.manual_seed(42)
+
+ model = torch.nn.Linear(100, 50, bias=False, dtype=torch.float32, device='cuda')
+ model.requires_grad_(True)
+ model.weight.data.normal_(0, 0.02)
+
+ optimizer = TensorParallelMuon(
+ params=[model.weight],
+ lr=0.01,
+ momentum_beta=0.95,
+ weight_decay=0.0, # Disable weight decay for deterministic comparison
+ num_ns_steps=5,
+ pg_collection=None,
+ mode=mode,
+ )
+
+ # Use fixed input for deterministic results
+ torch.manual_seed(42)
+ input_tensor = torch.randn(32, 100, dtype=torch.float32, device='cuda')
+
+ output = model(input_tensor)
+ loss = output.sum()
+ loss.backward()
+
+ original_weight = model.weight.data.clone()
+ optimizer.step()
+
+ # Verify weight was updated
+ assert not torch.equal(
+ model.weight.data, original_weight
+ ), f"Weight should be updated with mode={mode}"
+
+
+@pytest.mark.skipif(
+ int(os.getenv('WORLD_SIZE', '1')) == 1, reason="Multi-rank test requires WORLD_SIZE > 1"
+)
+class TestMuonOptimizerMultiRankTP:
+ """Test class for Muon optimizer with multi-rank and tensor parallel setup."""
+
+ @pytest.fixture(autouse=True)
+ def setup_and_teardown(self):
+ """Setup and teardown for each test with tensor parallel."""
+ world = int(os.getenv('WORLD_SIZE', '1'))
+ Utils.initialize_model_parallel(tensor_model_parallel_size=min(world, 2))
+ yield
+ Utils.destroy_model_parallel()
+
+ def create_tp_model_and_optimizer(self, mode):
+ """Create model with TP and optimizer.
+
+ Args:
+ mode: Muon optimizer mode
+
+ Returns:
+ tuple: (model, optimizer, pg_collection)
+ """
+ rank = int(os.getenv('RANK', '0'))
+ pg_collection = ProcessGroupCollection.use_mpu_process_groups()
+
+ # Create model with partition_dim for TP
+ torch.manual_seed(42 + rank)
+ model = torch.nn.Linear(100, 50, bias=False, dtype=torch.float32, device='cuda')
+ model.requires_grad_(True)
+ model.weight.data.normal_(0, 0.02)
+ model.weight.partition_dim = 0 # Set partition dimension for TP
+
+ optimizer = TensorParallelMuon(
+ params=[model.weight],
+ lr=0.01,
+ momentum_beta=0.95,
+ weight_decay=0.0,
+ num_ns_steps=5,
+ pg_collection=pg_collection,
+ mode=mode,
+ )
+
+ return model, optimizer
+
+ @pytest.mark.parametrize("mode", ["duplicated", "distributed"])
+ def test_muon_optimizer_modes_multirank_same_result(self, mode):
+ """Test that duplicated and distributed modes produce same results with TP > 1."""
+ model, optimizer = self.create_tp_model_and_optimizer(mode)
+
+ # Use fixed input for deterministic results
+ torch.manual_seed(42)
+ input_tensor = torch.randn(32, 100, dtype=torch.float32, device='cuda')
+
+ output = model(input_tensor)
+ loss = output.sum()
+ loss.backward()
+
+ original_weight = model.weight.data.clone()
+ optimizer.step()
+
+ # Verify weight was updated
+ assert not torch.equal(
+ model.weight.data, original_weight
+ ), f"Weight should be updated with mode={mode}"
+
+ def test_muon_optimizer_blockwise_mode_different_result(self):
+ """Test that blockwise mode produces different results than duplicated/distributed with TP > 1."""
+ model, optimizer = self.create_tp_model_and_optimizer("blockwise")
+
+ # Use fixed input for deterministic results
+ torch.manual_seed(42)
+ input_tensor = torch.randn(32, 100, dtype=torch.float32, device='cuda')
+
+ output = model(input_tensor)
+ loss = output.sum()
+ loss.backward()
+
+ original_weight = model.weight.data.clone()
+ optimizer.step()
+
+ # Verify weight was updated
+ assert not torch.equal(
+ model.weight.data, original_weight
+ ), "Weight should be updated with mode=blockwise"
+
+
+@pytest.mark.parametrize(
+ "coefficient_type_and_steps", [("simple", 3), ("quintic", 5), ("polar_express", 8)]
+)
+def test_muon_optimizer_coefficient_types(coefficient_type_and_steps):
+ """Test TensorParallelMuon optimizer with different coefficient types."""
+ model = torch.nn.Linear(80, 40, bias=False, dtype=torch.float32, device='cuda')
+ model.requires_grad_(True)
+ model.weight.data.fill_(1.0)
+
+ optimizer = TensorParallelMuon(
+ params=[model.weight],
+ lr=0.01,
+ coefficient_type=coefficient_type_and_steps[0],
+ num_ns_steps=coefficient_type_and_steps[1],
+ pg_collection=None,
+ mode="duplicated",
+ )
+
+ input_tensor = torch.randn(16, 80, dtype=torch.float32, device='cuda')
+ output = model(input_tensor)
+ loss = output.sum()
+ loss.backward()
+
+ original_weight = model.weight.data.clone()
+ optimizer.step()
+
+ assert not torch.equal(
+ model.weight.data, original_weight
+ ), f"Weight should be updated with coefficient_type={coefficient_type_and_steps[0]} and num_ns_steps={coefficient_type_and_steps[1]}"
+
+
+@pytest.mark.parametrize("scale_mode", ["spectral", "unit_rms_norm", "shape_scaling"])
+def test_muon_optimizer_scale_modes(scale_mode):
+ """Test TensorParallelMuon optimizer with different scale modes."""
+ model = torch.nn.Linear(60, 30, bias=False, dtype=torch.float32, device='cuda')
+ model.requires_grad_(True)
+ model.weight.data.fill_(1.0)
+
+ optimizer = TensorParallelMuon(
+ params=[model.weight],
+ lr=0.01,
+ scale_mode=scale_mode,
+ num_ns_steps=5,
+ pg_collection=None,
+ mode="duplicated",
+ )
+
+ input_tensor = torch.randn(16, 60, dtype=torch.float32, device='cuda')
+ output = model(input_tensor)
+ loss = output.sum()
+ loss.backward()
+
+ original_weight = model.weight.data.clone()
+ optimizer.step()
+
+ assert not torch.equal(
+ model.weight.data, original_weight
+ ), f"Weight should be updated with scale_mode={scale_mode}"
+
+
+@pytest.mark.parametrize("use_nesterov", [True, False])
+def test_muon_optimizer_nesterov(use_nesterov):
+ """Test TensorParallelMuon optimizer with and without Nesterov momentum."""
+ model = torch.nn.Linear(50, 25, bias=False, dtype=torch.float32, device='cuda')
+ model.requires_grad_(True)
+ model.weight.data.fill_(1.0)
+
+ optimizer = TensorParallelMuon(
+ params=[model.weight],
+ lr=0.01,
+ momentum_beta=0.9,
+ use_nesterov=use_nesterov,
+ num_ns_steps=5,
+ pg_collection=None,
+ mode="duplicated",
+ )
+
+ input_tensor = torch.randn(16, 50, dtype=torch.float32, device='cuda')
+ output = model(input_tensor)
+ loss = output.sum()
+ loss.backward()
+
+ original_weight = model.weight.data.clone()
+ optimizer.step()
+
+ assert not torch.equal(
+ model.weight.data, original_weight
+ ), f"Weight should be updated with use_nesterov={use_nesterov}"
+
+
+def test_muon_optimizer_multiple_steps():
+ """Test TensorParallelMuon optimizer across multiple optimization steps."""
+ model = torch.nn.Linear(100, 50, bias=False, dtype=torch.float32, device='cuda')
+ model.requires_grad_(True)
+ model.weight.data.fill_(1.0)
+
+ optimizer = TensorParallelMuon(
+ params=[model.weight],
+ lr=0.01,
+ momentum_beta=0.95,
+ weight_decay=0.01,
+ num_ns_steps=5,
+ pg_collection=None,
+ mode="duplicated",
+ )
+
+ weights_history = [model.weight.data.clone()]
+
+ for i in range(3):
+ input_tensor = torch.randn(32, 100, dtype=torch.float32, device='cuda')
+ output = model(input_tensor)
+ loss = output.sum()
+ loss.backward()
+
+ optimizer.step()
+ optimizer.zero_grad()
+ weights_history.append(model.weight.data.clone())
+
+ # Verify weights changed at each step
+ for i in range(len(weights_history) - 1):
+ assert not torch.equal(
+ weights_history[i], weights_history[i + 1]
+ ), f"Weight should change at step {i}"
+
+
+def test_muon_optimizer_qkv_split():
+ """Test TensorParallelMuon optimizer with QKV splitting."""
+ # Create a model with QKV-like parameter
+ qkv_size = 3 * 64 * 16 # Combined Q, K, V dimensions, 16 heads x 64 per head
+ hidden_size = 1024
+ model = torch.nn.Linear(hidden_size, qkv_size, bias=False, dtype=torch.float32, device='cuda')
+ model.requires_grad_(True)
+ model.weight.data.fill_(1.0)
+
+ # Mark parameter as QKV
+ model.weight.is_qkv = True
+
+ # QKV split shapes: [Q_size, K_size, V_size]
+ qkv_split_shapes = (64, 64, 64)
+
+ # Test with split_qkv=True
+ optimizer_split = TensorParallelMuon(
+ params=[model.weight],
+ lr=0.01,
+ split_qkv=True,
+ is_qkv_fn=lambda p: getattr(p, 'is_qkv', False),
+ qkv_split_shapes=qkv_split_shapes,
+ num_ns_steps=5,
+ pg_collection=None,
+ mode="duplicated",
+ )
+
+ input_tensor = torch.randn(16, hidden_size, dtype=torch.float32, device='cuda')
+ output = model(input_tensor)
+ loss = output.sum()
+ loss.backward()
+
+ original_weight = model.weight.data.clone()
+ optimizer_split.step()
+ weight_with_split = model.weight.data.clone()
+
+ assert not torch.equal(
+ weight_with_split, original_weight
+ ), "QKV weight should be updated with split_qkv=True"
+
+ # Reset model and test with split_qkv=False
+ model.weight.data.fill_(1.0)
+ optimizer_no_split = TensorParallelMuon(
+ params=[model.weight],
+ lr=0.01,
+ split_qkv=False,
+ num_ns_steps=5,
+ pg_collection=None,
+ mode="duplicated",
+ )
+
+ output = model(input_tensor)
+ loss = output.sum()
+ loss.backward()
+
+ optimizer_no_split.step()
+ weight_without_split = model.weight.data.clone()
+
+ assert not torch.equal(
+ weight_without_split, original_weight
+ ), "QKV weight should be updated with split_qkv=False"
+
+ # Ensure the two results are different
+ assert not torch.equal(
+ weight_with_split, weight_without_split
+ ), "Weights should be different between split_qkv=True and split_qkv=False"
+
+
+def test_muon_optimizer_extra_scale_factor():
+ """Test TensorParallelMuon optimizer with different extra_scale_factor values."""
+ model = torch.nn.Linear(80, 40, bias=False, dtype=torch.float32, device='cuda')
+ model.requires_grad_(True)
+ model.weight.data.fill_(1.0)
+
+ optimizer = TensorParallelMuon(
+ params=[model.weight],
+ lr=0.01,
+ extra_scale_factor=2.0,
+ num_ns_steps=5,
+ pg_collection=None,
+ mode="duplicated",
+ )
+
+ input_tensor = torch.randn(16, 80, dtype=torch.float32, device='cuda')
+ output = model(input_tensor)
+ loss = output.sum()
+ loss.backward()
+
+ original_weight = model.weight.data.clone()
+ optimizer.step()
+
+ assert not torch.equal(
+ model.weight.data, original_weight
+ ), "Weight should be updated with extra_scale_factor"
+
+
+@pytest.mark.parametrize("num_ns_steps", [5, 15, 25])
+def test_muon_optimizer_num_ns_steps(num_ns_steps):
+ """Test TensorParallelMuon optimizer with different numbers of Newton-Schulz steps."""
+ model = torch.nn.Linear(60, 30, bias=False, dtype=torch.float32, device='cuda')
+ model.requires_grad_(True)
+ model.weight.data.fill_(1.0)
+
+ optimizer = TensorParallelMuon(
+ params=[model.weight],
+ lr=0.01,
+ coefficient_type="quintic",
+ num_ns_steps=num_ns_steps,
+ pg_collection=None,
+ mode="duplicated",
+ )
+
+ input_tensor = torch.randn(16, 60, dtype=torch.float32, device='cuda')
+ output = model(input_tensor)
+ loss = output.sum()
+ loss.backward()
+
+ original_weight = model.weight.data.clone()
+ optimizer.step()
+
+ assert not torch.equal(
+ model.weight.data, original_weight
+ ), f"Weight should be updated with num_ns_steps={num_ns_steps}"
diff --git a/tests/unit_tests/test_optimizer.py b/tests/unit_tests/test_optimizer.py
index cba7d60bbd1..1f5bbc3f14c 100644
--- a/tests/unit_tests/test_optimizer.py
+++ b/tests/unit_tests/test_optimizer.py
@@ -1,6 +1,7 @@
# Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
import os
+from unittest.mock import patch
import pytest
import torch
@@ -12,7 +13,17 @@
from transformer_engine.pytorch.fp8 import fp8_autocast
from megatron.core.distributed import DistributedDataParallel, DistributedDataParallelConfig
-from megatron.core.optimizer import ChainedOptimizer, OptimizerConfig, get_megatron_optimizer
+from megatron.core.optimizer import (
+ ChainedOptimizer,
+ OptimizerConfig,
+ ParamKey,
+ ParamPredicate,
+ _get_param_groups,
+ check_config_overrides_consistency,
+ get_megatron_optimizer,
+ get_standard_config_overrides,
+)
+from megatron.core.optimizer_param_scheduler import ParamGroupOverride
from megatron.core.process_groups_config import ProcessGroupCollection
from megatron.core.transformer import TransformerConfig
from megatron.core.utils import is_te_min_version, is_torch_min_version
@@ -24,7 +35,7 @@
from transformer_engine.pytorch.fp8 import check_fp8_block_scaling_support
fp8_block_scaling_available, reason_for_no_fp8_block_scaling = check_fp8_block_scaling_support()
- from transformer_engine.common.recipe import Float8BlockScaling, Format
+ from transformer_engine.common.recipe import DelayedScaling, Float8BlockScaling, Format
except:
fp8_block_scaling_available = False
reason_for_no_fp8_block_scaling = "FP8 block scaled GEMM requires Hopper and CUDA >= 12.9."
@@ -35,7 +46,7 @@
class Net(nn.Module):
- def __init__(self):
+ def __init__(self, add_layernorm=False):
super().__init__()
self.conv1 = nn.Conv2d(3, 6, 5)
self.pool = nn.MaxPool2d(2, 2)
@@ -43,6 +54,10 @@ def __init__(self):
self.fc1 = nn.Linear(16 * 5 * 5, 120)
self.fc2 = nn.Linear(120, 84)
self.fc3 = nn.Linear(84, 10)
+ if add_layernorm:
+ self.q_layernorm = nn.LayerNorm(10, bias=False)
+ self.k_layernorm = nn.LayerNorm(10, bias=False)
+ self.layernorm = nn.LayerNorm(10, bias=False)
def forward(self, x):
x = self.pool(F.relu(self.conv1(x)))
@@ -54,6 +69,223 @@ def forward(self, x):
return x
+@patch('torch.distributed.get_world_size', return_value=1)
+@patch(
+ 'torch.distributed.all_gather_object', lambda output_list, obj: output_list.__setitem__(0, obj)
+)
+def test_get_param_groups_no_overrides(mock_get_world_size):
+ net = Net()
+ # NOTE: to get no overrides, supply an empty dictionary rather than None.
+ param_groups = _get_param_groups([net], OptimizerConfig(optimizer='adam', lr=0.01), {})
+ assert len(param_groups) == 1
+ pg0 = param_groups[0]
+ assert pg0.keys() == {
+ 'params',
+ 'is_expert_parallel',
+ 'default_config',
+ 'wd_mult',
+ 'lr_mult',
+ 'is_decoupled_lr',
+ 'max_lr',
+ 'min_lr',
+ }
+ assert pg0['params'] == list(net.parameters())
+ assert pg0['is_expert_parallel'] == False
+ assert pg0['default_config'] == True
+ assert pg0['wd_mult'] == 1.0
+ assert pg0['lr_mult'] == 1.0
+ assert pg0['is_decoupled_lr'] == False
+ assert pg0['max_lr'] == 0.01 # from the optimizer config default for lr
+ assert pg0['min_lr'] is None # from the optimizer config default.
+
+
+@patch('torch.distributed.get_world_size', return_value=1)
+@patch(
+ 'torch.distributed.all_gather_object', lambda output_list, obj: output_list.__setitem__(0, obj)
+)
+def test_get_param_groups_default_overrides(mock_get_world_size):
+ """Test that the default overrides are applied to the parameter groups."""
+ net = Net()
+ # NOTE: to get legacy default overrides, supply None.
+ opt_config = OptimizerConfig(optimizer='adam', lr=0.01)
+ check_config_overrides_consistency(opt_config, None)
+ param_groups = _get_param_groups([net], opt_config, None)
+ assert len(param_groups) == 2
+ pg0, pg1 = param_groups
+ wd_mults = {pg0['wd_mult'], pg1['wd_mult']}
+ assert wd_mults == {1.0, 0.0}
+
+
+@patch('torch.distributed.get_world_size', return_value=1)
+@patch(
+ 'torch.distributed.all_gather_object', lambda output_list, obj: output_list.__setitem__(0, obj)
+)
+def test_get_param_groups_with_overrides(mock_get_world_size):
+ net = Net()
+ config_overrides = {
+ ParamKey(
+ name="*.bias",
+ predicate=ParamPredicate(name="param_len_1", fn=lambda param: len(param.shape) == 1),
+ ): ParamGroupOverride(wd_mult=0.0)
+ }
+ opt_config = OptimizerConfig(optimizer='adam', lr=0.01)
+ check_config_overrides_consistency(opt_config, config_overrides)
+ param_groups = _get_param_groups([net], opt_config, config_overrides)
+ assert len(param_groups) == 2
+ p_set = set(net.parameters())
+
+ assert p_set == set(param_groups[0]['params']) | set(param_groups[1]['params'])
+ assert len(p_set) == len(param_groups[0]['params']) + len(param_groups[1]['params'])
+ assert param_groups[0]['wd_mult'] == 0.0 or param_groups[1]['wd_mult'] == 0.0
+ assert param_groups[0]['wd_mult'] == 1.0 or param_groups[1]['wd_mult'] == 1.0
+ assert len(param_groups[0]['params']) > 0 and len(param_groups[1]['params']) > 0
+
+
+@patch('torch.distributed.get_world_size', return_value=1)
+@patch(
+ 'torch.distributed.all_gather_object', lambda output_list, obj: output_list.__setitem__(0, obj)
+)
+def test_get_param_groups_multiple_matches(mock_get_world_size):
+ net = Net()
+
+ param_groups = _get_param_groups(
+ [net],
+ OptimizerConfig(optimizer='adam', lr=0.01),
+ {
+ ParamKey(name="*.bias"): ParamGroupOverride(min_lr=1e-4, wd_mult=0.0),
+ ParamKey(
+ predicate=ParamPredicate(name="param_len_1", fn=lambda param: len(param.shape) == 1)
+ ): ParamGroupOverride(wd_mult=0.0, min_lr=1e-4),
+ },
+ )
+ config_overrides = {
+ ParamKey(
+ name="*.bias",
+ predicate=ParamPredicate(name="param_len_1", fn=lambda param: len(param.shape) == 1),
+ ): ParamGroupOverride(min_lr=1e-4, wd_mult=0.0)
+ }
+ opt_config = OptimizerConfig(optimizer='adam', lr=0.01)
+ check_config_overrides_consistency(opt_config, config_overrides)
+ param_groups2 = _get_param_groups([net], opt_config, config_overrides)
+ assert len(param_groups) == 2
+ assert param_groups == param_groups2
+
+
+@patch('torch.distributed.get_world_size', return_value=1)
+@patch(
+ 'torch.distributed.all_gather_object', lambda output_list, obj: output_list.__setitem__(0, obj)
+)
+def test_get_param_groups_overlapping_matches(mock_get_world_size):
+ """In this test, we see if we can have two matches that create three param groups."""
+ net = Net()
+ # We expect that all convolution parameters will have wd_mult=0.0
+ # However the conv1 related parameters will additionally have a different LR schedule.
+ # this should create three param groups (no match, conv1 (both wd_mult=0.0 and LR schedule), conv2 (only wd_mult=0.0))
+ config_overrides = {
+ ParamKey(name="*conv*"): ParamGroupOverride(wd_mult=0.0),
+ ParamKey(name="*conv1*"): ParamGroupOverride(min_lr=10, max_lr=20),
+ }
+ opt_config = OptimizerConfig(optimizer='adam', lr=0.01)
+ check_config_overrides_consistency(opt_config, config_overrides)
+ param_groups = _get_param_groups([net], opt_config, config_overrides)
+ assert len(param_groups) == 3
+ p_set = set(net.parameters())
+ assert p_set == set(param_groups[0]['params']) | set(param_groups[1]['params']) | set(
+ param_groups[2]['params']
+ )
+ assert len(p_set) == len(param_groups[0]['params']) + len(param_groups[1]['params']) + len(
+ param_groups[2]['params']
+ )
+ assert (
+ param_groups[0]['wd_mult'] == 1.0
+ ), "We expect the first param group to be the None one, which should have wd_mult=1.0"
+ assert (
+ param_groups[1]['wd_mult'] == 0.0
+ ), "We expect the second param group to be the conv1 one, which should have wd_mult=0.0"
+ assert (
+ param_groups[2]['wd_mult'] == 0.0
+ ), "We expect the third param group to be the conv2 one, which should have wd_mult=0.0"
+ assert param_groups[1]['min_lr'] == 10
+ assert param_groups[1]['max_lr'] == 20
+ assert param_groups[2]['min_lr'] is None
+ assert param_groups[2]['max_lr'] == 0.01
+
+
+@patch('torch.distributed.get_world_size', return_value=1)
+@patch(
+ 'torch.distributed.all_gather_object', lambda output_list, obj: output_list.__setitem__(0, obj)
+)
+def test_get_param_groups_with_standard_config_overrides(apply_wd_to_qk_layernorm: bool):
+ """In this test, we see if the standard config overrides are applied correctly."""
+
+ # Initialize the model with layernorm
+ net = Net()
+
+ config = OptimizerConfig(optimizer='adam', lr=0.01)
+ config_overrides = get_standard_config_overrides(config=config)
+ param_groups = _get_param_groups([net], config, config_overrides)
+
+ assert len(param_groups) == 2
+ p_set = set(net.parameters())
+
+ assert p_set == set(param_groups[0]['params']) | set(param_groups[1]['params'])
+ assert len(p_set) == len(param_groups[0]['params']) + len(param_groups[1]['params'])
+ assert param_groups[0]['wd_mult'] == 0.0 or param_groups[1]['wd_mult'] == 0.0
+ assert param_groups[0]['wd_mult'] == 1.0 or param_groups[1]['wd_mult'] == 1.0
+ assert len(param_groups[0]['params']) > 0 and len(param_groups[1]['params']) > 0
+
+ # Both param groups should have 5 parameters.
+ # Param group A (wd_mult=1.0): conv1.weight, conv2.weight, fc1.weight, fc2.weight, fc3.weight
+ # Param group B (wd_mult=0.0): conv1.bias, conv2.bias, fc1.bias, fc2.bias, fc3.bias
+ assert len(param_groups[0]['params']) == 5, (
+ f"Expected 5 parameters in the first param group, "
+ f"but got {len(param_groups[0]['params'])}"
+ )
+ assert len(param_groups[1]['params']) == 5, (
+ f"Expected 5 parameters in the second param group, "
+ f"but got {len(param_groups[1]['params'])}"
+ )
+
+
+@patch('torch.distributed.get_world_size', return_value=1)
+@patch(
+ 'torch.distributed.all_gather_object', lambda output_list, obj: output_list.__setitem__(0, obj)
+)
+def test_get_param_groups_appling_wd_to_qk_layernorm(apply_wd_to_qk_layernorm: bool):
+ """In this test, we see if the `apply_wd_to_qk_layernorm` config is applied correctly."""
+
+ # Initialize the model with layernorm
+ net = Net(add_layernorm=True)
+
+ config = OptimizerConfig(
+ optimizer='adam', lr=0.01, apply_wd_to_qk_layernorm=apply_wd_to_qk_layernorm
+ )
+ config_overrides = get_standard_config_overrides(config=config)
+ param_groups = _get_param_groups([net], config, config_overrides)
+
+ assert len(param_groups) == 2
+ p_set = set(net.parameters())
+
+ assert p_set == set(param_groups[0]['params']) | set(param_groups[1]['params'])
+ assert len(p_set) == len(param_groups[0]['params']) + len(param_groups[1]['params'])
+ assert param_groups[0]['wd_mult'] == 1.0
+ assert param_groups[1]['wd_mult'] == 0.0
+
+ # There are two param groups, having 7, and 6 parameters respectively.
+ # Param group A (wd_mult=1.0): conv1.weight, conv2.weight, fc1.weight, fc2.weight, fc3.weight,
+ # q_layernorm.weight, k_layernorm.weight
+ # Param group B (wd_mult=0.0): conv1.bias, conv2.bias, fc1.bias, fc2.bias, fc3.bias,
+ # layernorm.weight
+ assert len(param_groups[0]['params']) == 7, (
+ f"Expected 5 parameters in the first param group, "
+ f"but got {len(param_groups[0]['params'])}"
+ )
+ assert len(param_groups[1]['params']) == 6, (
+ f"Expected 6 parameters in the second param group, "
+ f"but got {len(param_groups[1]['params'])}"
+ )
+
+
def test_chained_optimizer():
net = Net()
optimizer_1 = Adam(list(net.parameters())[:2], lr=0.01)
@@ -246,24 +478,13 @@ def run_model(model, input, optim, fp8_recipe, fp8_recipe_settings):
test_model, input, test_optim, fp8_recipe, fp8_recipe_settings
)
- rtol = 1e-3 # relative tolerance
- atol = 1e-5 # absolute tolerance
+ rtol, atol = 1.6e-2, 1e-5
# Compare grad norms - allow small difference due to precision
- rel_diff = abs(test_grad_norm - baseline_grad_norm) / (
- abs(baseline_grad_norm) + 1e-7 # avoid div by 0
- )
- abs_diff = abs(test_grad_norm - baseline_grad_norm)
- assert (
- rel_diff <= rtol or abs_diff <= atol
- ), f"Grad norm mismatch: baseline={baseline_grad_norm}, test={test_grad_norm}, rel_diff={rel_diff}, abs_diff={abs_diff}"
+ torch.testing.assert_close(test_grad_norm, baseline_grad_norm, atol=atol, rtol=rtol)
# Compare losses - allow small difference due to precision
- loss_rel_diff = abs(test_loss - baseline_loss) / (abs(baseline_loss) + 1e-7)
- loss_abs_diff = abs(test_loss - baseline_loss)
- assert (
- loss_rel_diff <= rtol or loss_abs_diff <= atol
- ), f"Loss mismatch: baseline={baseline_loss}, test={test_loss}, rel_diff={loss_rel_diff}, abs_diff={loss_abs_diff}"
+ torch.testing.assert_close(test_loss, baseline_loss, atol=atol, rtol=rtol)
# Save and reload state dict for the test model
state_dict = test_optim.state_dict()
diff --git a/tests/unit_tests/test_optimizer_state_offloading.py b/tests/unit_tests/test_optimizer_state_offloading.py
new file mode 100644
index 00000000000..baaab355182
--- /dev/null
+++ b/tests/unit_tests/test_optimizer_state_offloading.py
@@ -0,0 +1,337 @@
+# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved.
+
+"""Unit tests for OptimizerStateOffloader."""
+
+import pytest
+import torch
+import torch.nn as nn
+
+from megatron.core.distributed import DistributedDataParallel, DistributedDataParallelConfig
+from megatron.core.optimizer import OptimizerConfig, get_megatron_optimizer
+from megatron.core.transformer import TransformerConfig
+from tests.unit_tests.test_utilities import Utils
+
+try:
+ from transformer_engine.pytorch.optimizers import FusedAdam # noqa: F401
+
+ TE_FUSED_ADAM_AVAILABLE = True
+except ImportError:
+ TE_FUSED_ADAM_AVAILABLE = False
+
+
+class SimpleModel(nn.Module):
+ """Simple model for testing."""
+
+ def __init__(self, hidden_size=256):
+ super().__init__()
+ self.fc1 = nn.Linear(hidden_size, hidden_size)
+ self.fc2 = nn.Linear(hidden_size, hidden_size)
+
+ def forward(self, x):
+ return self.fc2(torch.relu(self.fc1(x)))
+
+
+def create_model_and_optimizer(hidden_size=256, offload_optimizer_states=True, **optimizer_kwargs):
+ """Helper to create model and optimizer for tests."""
+ model = SimpleModel(hidden_size=hidden_size).bfloat16().cuda()
+ ddp_config = DistributedDataParallelConfig(use_distributed_optimizer=True)
+ model = DistributedDataParallel(
+ TransformerConfig(num_attention_heads=1, num_layers=1), ddp_config, model
+ )
+
+ default_config = dict(
+ optimizer='adam',
+ bf16=True,
+ lr=0.001,
+ use_distributed_optimizer=True,
+ offload_optimizer_states=offload_optimizer_states,
+ )
+ default_config.update(optimizer_kwargs)
+
+ optimizer_config = OptimizerConfig(**default_config)
+ optim = get_megatron_optimizer(optimizer_config, [model])
+ return model, optim
+
+
+def run_forward_backward_step(model, optim, hidden_size=256):
+ """Run a single forward-backward-step cycle."""
+ input_tensor = torch.randn(8, hidden_size, dtype=torch.bfloat16, device='cuda')
+ output = model(input_tensor)
+ output.sum().backward()
+ optim.step()
+ optim.zero_grad()
+
+
+# =============================================================================
+# Test 1: Basic OptimizerStateOffloader Initialization
+# =============================================================================
+@pytest.mark.skipif(not TE_FUSED_ADAM_AVAILABLE, reason="Requires TE FusedAdam")
+def test_offloader_initialization():
+ """Test that OptimizerStateOffloader initializes correctly."""
+ Utils.initialize_model_parallel()
+ model, optim = create_model_and_optimizer()
+ dist_optim = optim.chained_optimizers[0]
+
+ # Offloader is created in __init__ when offload_optimizer_states=True
+ assert dist_optim._state_offloader is not None
+ offloader = dist_optim._state_offloader
+
+ # Verify offloader properties
+ assert offloader.adam_optimizer is not None
+ assert offloader._d2h_stream is not None
+ assert offloader._h2d_stream is not None
+ assert offloader._offloaded is False
+
+ # Before first step, optimizer states are not initialized yet
+ assert offloader._optimizer_states_initialized is False
+
+ # Run one step to initialize optimizer states
+ run_forward_backward_step(model, optim)
+
+ # After first step, optimizer states should be marked as initialized
+ assert offloader._optimizer_states_initialized is True
+ Utils.destroy_model_parallel()
+
+
+# =============================================================================
+# Test 2: Early Master Weight Offloading Before First Step
+# =============================================================================
+@pytest.mark.skipif(not TE_FUSED_ADAM_AVAILABLE, reason="Requires TE FusedAdam")
+def test_early_master_weight_offloading():
+ """Test that master weights can be offloaded before the first optimizer step."""
+ Utils.initialize_model_parallel()
+ model, optim = create_model_and_optimizer()
+ dist_optim = optim.chained_optimizers[0]
+
+ # Offloader is created in __init__
+ assert dist_optim._state_offloader is not None
+ offloader = dist_optim._state_offloader
+
+ # Before first step, optimizer states are not initialized
+ assert offloader._optimizer_states_initialized is False
+
+ # Capture original master weights before offload
+ original_master_weights = []
+ for group in dist_optim.shard_fp32_from_float16_groups:
+ group_weights = [tensor.clone() for tensor in group]
+ original_master_weights.append(group_weights)
+
+ # Offload before first step - should only offload master weights
+ offloader.offload()
+ offloader.release_gpu_memory()
+ torch.cuda.synchronize()
+
+ # Verify master weights were offloaded (storage resized to 0)
+ for group in dist_optim.shard_fp32_from_float16_groups:
+ for tensor in group:
+ assert tensor.untyped_storage().size() == 0, "Master weight should be offloaded"
+
+ # Reload master weights
+ offloader.reload()
+ offloader.sync_before_step()
+
+ # Verify master weights match after reload
+ for group_idx, group in enumerate(dist_optim.shard_fp32_from_float16_groups):
+ for param_idx, tensor in enumerate(group):
+ original = original_master_weights[group_idx][param_idx]
+ torch.testing.assert_close(
+ tensor,
+ original,
+ msg=f"Master weight [{group_idx}][{param_idx}] mismatch after offload/reload",
+ )
+
+ # Now run a step and verify optimizer states can be offloaded after
+ run_forward_backward_step(model, optim)
+ assert offloader._optimizer_states_initialized is True
+
+ Utils.destroy_model_parallel()
+
+
+# =============================================================================
+# Test 3: Offload and Reload Correctness
+# =============================================================================
+@pytest.mark.skipif(not TE_FUSED_ADAM_AVAILABLE, reason="Requires TE FusedAdam")
+@pytest.mark.parametrize("offload_optimizer_states", [True, False])
+@pytest.mark.parametrize("offload_master_weights", [True, False])
+def test_offload_reload_correctness(offload_optimizer_states, offload_master_weights):
+ """Test that offload/reload preserves optimizer state values."""
+ if not offload_optimizer_states and not offload_master_weights:
+ pytest.skip("At least one offload type required")
+
+ Utils.initialize_model_parallel()
+ model, optim = create_model_and_optimizer()
+ dist_optim = optim.chained_optimizers[0]
+
+ # Run steps to build up optimizer state
+ for _ in range(3):
+ run_forward_backward_step(model, optim)
+
+ offloader = dist_optim._state_offloader
+
+ # Capture original states before offload
+ original_states = {}
+ for param, state in offloader.adam_optimizer.state.items():
+ original_states[param] = {
+ k: v.clone() for k, v in state.items() if isinstance(v, torch.Tensor)
+ }
+
+ # Offload
+ offloader.offload(
+ offload_optimizer_states=offload_optimizer_states,
+ offload_master_weights=offload_master_weights,
+ )
+
+ # Release GPU memory
+ offloader.release_gpu_memory()
+ torch.cuda.synchronize()
+
+ # Reload
+ offloader.reload()
+ offloader.sync_before_step()
+
+ # Verify states match after reload
+ for param, state in offloader.adam_optimizer.state.items():
+ if param in original_states:
+ for key, original_tensor in original_states[param].items():
+ if key in state and isinstance(state[key], torch.Tensor):
+ reloaded_tensor = state[key]
+ assert reloaded_tensor.device.type == 'cuda', f"State {key} should be on GPU"
+ torch.testing.assert_close(
+ reloaded_tensor,
+ original_tensor,
+ msg=f"State {key} mismatch after offload/reload",
+ )
+ Utils.destroy_model_parallel()
+
+
+# =============================================================================
+# Test 4: GPU Memory Release Verification
+# =============================================================================
+@pytest.mark.skipif(not TE_FUSED_ADAM_AVAILABLE, reason="Requires TE FusedAdam")
+def test_gpu_memory_release():
+ """Test that GPU memory is actually freed after release_gpu_memory()."""
+ Utils.initialize_model_parallel()
+ # Use larger model for measurable memory impact
+ model, optim = create_model_and_optimizer(hidden_size=1024)
+ dist_optim = optim.chained_optimizers[0]
+
+ # Initialize optimizer states
+ run_forward_backward_step(model, optim, hidden_size=1024)
+
+ offloader = dist_optim._state_offloader
+
+ # Measure memory before offload
+ torch.cuda.synchronize()
+ torch.cuda.empty_cache()
+ memory_before = torch.cuda.memory_allocated()
+
+ # Offload and release
+ offloader.offload()
+ offloader.release_gpu_memory()
+
+ # Wait for async operations
+ torch.cuda.synchronize()
+ torch.cuda.empty_cache()
+ memory_after = torch.cuda.memory_allocated()
+
+ # Memory should decrease
+ memory_freed = memory_before - memory_after
+ assert memory_freed > 0, f"Expected memory to be freed, but got {memory_freed} bytes difference"
+ Utils.destroy_model_parallel()
+
+
+# =============================================================================
+# Test 5: Multiple Offload/Reload Cycles
+# =============================================================================
+@pytest.mark.skipif(not TE_FUSED_ADAM_AVAILABLE, reason="Requires TE FusedAdam")
+def test_multiple_offload_reload_cycles():
+ """Test that multiple offload/reload cycles work correctly."""
+ Utils.initialize_model_parallel()
+ model, optim = create_model_and_optimizer()
+ dist_optim = optim.chained_optimizers[0]
+
+ # Initialize
+ run_forward_backward_step(model, optim)
+
+ offloader = dist_optim._state_offloader
+
+ # Run multiple cycles
+ for cycle in range(5):
+ # Offload
+ offloader.offload()
+ offloader.release_gpu_memory()
+
+ # Reload
+ offloader.reload()
+ offloader.sync_before_step()
+
+ # Run optimizer step
+ run_forward_backward_step(model, optim)
+
+ # Verify model can still produce valid outputs
+ input_tensor = torch.randn(8, 256, dtype=torch.bfloat16, device='cuda')
+ output = model(input_tensor)
+ assert not output.isnan().any(), "Model output contains NaN after multiple cycles"
+ Utils.destroy_model_parallel()
+
+
+# =============================================================================
+# Test 6: Training Correctness with Offloading
+# =============================================================================
+@pytest.mark.skipif(not TE_FUSED_ADAM_AVAILABLE, reason="Requires TE FusedAdam")
+def test_training_correctness_with_offloading():
+ """Test that training with offloading produces same results as without."""
+ Utils.initialize_model_parallel()
+ torch.manual_seed(42)
+
+ # Model 1: with offloading
+ model1, optim1 = create_model_and_optimizer(offload_optimizer_states=True, lr=0.01)
+
+ # Model 2: without offloading (reference)
+ torch.manual_seed(42)
+ model2, optim2 = create_model_and_optimizer(offload_optimizer_states=False, lr=0.01)
+
+ # Train both models
+ n_steps = 10
+ torch.manual_seed(123)
+ dist_optim1 = optim1.chained_optimizers[0]
+
+ # Offloader is created in __init__ when offload_optimizer_states=True
+ assert dist_optim1._state_offloader is not None
+ offloader = dist_optim1._state_offloader
+
+ for step in range(n_steps):
+ input_tensor = torch.randn(8, 256, dtype=torch.bfloat16, device='cuda')
+
+ # Model 1 with offloading
+ # Offload states (master weights can be offloaded from the start,
+ # optimizer states will be skipped until after first step)
+ offloader.offload()
+ offloader.release_gpu_memory()
+
+ output1 = model1(input_tensor)
+ loss1 = output1.sum()
+ loss1.backward()
+
+ offloader.reload()
+ offloader.sync_before_step()
+ optim1.step()
+ optim1.zero_grad()
+
+ # Model 2 without offloading
+ output2 = model2(input_tensor)
+ loss2 = output2.sum()
+ loss2.backward()
+ optim2.step()
+ optim2.zero_grad()
+
+ # Compare final model weights
+ for (n1, p1), (n2, p2) in zip(model1.named_parameters(), model2.named_parameters()):
+ torch.testing.assert_close(
+ p1.data,
+ p2.data,
+ atol=1e-5,
+ rtol=1e-4,
+ msg=f"Parameter {n1} mismatch between offloaded and non-offloaded training",
+ )
+ Utils.destroy_model_parallel()
diff --git a/tests/unit_tests/test_parallel_state.py b/tests/unit_tests/test_parallel_state.py
index 7218ed5b6e1..0c722ee0257 100644
--- a/tests/unit_tests/test_parallel_state.py
+++ b/tests/unit_tests/test_parallel_state.py
@@ -1,5 +1,7 @@
# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved.
+from math import log2
+
import pytest
import torch
@@ -499,3 +501,32 @@ def golden_rank_result_from_past_code(
assert expert_dp_group == expert_rank_generator.get_ranks(
"dp"
), f"{expert_dp_group} != {expert_rank_generator.get_ranks('dp')}."
+
+
+@pytest.mark.parametrize(
+ "world_size, tp_size, cp_size, dp_size",
+ [(8, 1, 2, 4), (8, 1, 1, 8)], # 8 GPUs, 1 TP, 2 CP, 4 DP # 8 GPUs, 1 TP, 1 CP, 8 DP
+)
+def test_hybrid_dp_cp_groups(world_size, tp_size, cp_size, dp_size):
+ """
+ Test that hybrid DPxCP groups are created correctly.
+ """
+ Utils.destroy_model_parallel()
+
+ # Skip if world size doesn't match
+ actual_world_size = torch.cuda.device_count()
+ if actual_world_size != world_size:
+ pytest.skip(f"Test requires world_size={world_size}, but got {actual_world_size}")
+ Utils.initialize_model_parallel(
+ tensor_model_parallel_size=tp_size,
+ context_parallel_size=cp_size,
+ hybrid_context_parallel=True,
+ )
+
+ dp_cp_size = ps.get_data_parallel_world_size(with_context_parallel=True)
+ group_sizes = [2**i for i in range(int(log2(dp_cp_size)))][1:]
+ for group_size in group_sizes:
+ group = ps.get_hybrid_data_context_parallel_groups(group_size=group_size)
+ assert group.size() == group_size
+
+ Utils.destroy_model_parallel()
diff --git a/tests/unit_tests/test_utilities.py b/tests/unit_tests/test_utilities.py
index f16f88f7865..39c78efb2b9 100644
--- a/tests/unit_tests/test_utilities.py
+++ b/tests/unit_tests/test_utilities.py
@@ -1,3 +1,4 @@
+# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
import os
from datetime import timedelta
@@ -27,8 +28,8 @@ def __init__(
class Utils:
- world_size = int(os.environ['WORLD_SIZE'])
- rank = int(os.environ['LOCAL_RANK'])
+ world_size = int(os.environ.get('WORLD_SIZE', '1'))
+ rank = int(os.environ.get('LOCAL_RANK', '0'))
inited = False
store = None
diff --git a/tests/unit_tests/transformer/moe/test_aux_loss.py b/tests/unit_tests/transformer/moe/test_aux_loss.py
index b1f78582383..f5726777383 100644
--- a/tests/unit_tests/transformer/moe/test_aux_loss.py
+++ b/tests/unit_tests/transformer/moe/test_aux_loss.py
@@ -576,3 +576,192 @@ def test_force_balanced_aux_loss(self, tp_size, ep_size, cp_size):
reduce_from_tensor_model_parallel_region(aux_loss, router.tp_cp_group)
assert aux_loss.item() == 1, f"{aux_loss_type}: {aux_loss.item()}"
clear_aux_losses_tracker()
+
+
+class TestPaddingMaskAuxLoss:
+ """Test padding mask support in various aux loss types."""
+
+ def setup_model_parallel(self, tp_size=1, ep_size=1, cp_size=1, sequence_parallel=False):
+ """Initialize model parallel with given configuration.
+
+ Args:
+ tp_size: Tensor parallel size.
+ ep_size: Expert parallel size.
+ cp_size: Context parallel size.
+ """
+ Utils.initialize_model_parallel(
+ tensor_model_parallel_size=tp_size,
+ pipeline_model_parallel_size=1,
+ context_parallel_size=cp_size,
+ expert_model_parallel_size=ep_size,
+ )
+ _set_random_seed(seed_=123, data_parallel_random_init=False)
+
+ # Store parallel configuration
+ self.tp_size = tp_size
+ self.ep_size = ep_size
+ self.cp_size = cp_size
+
+ # Default configuration
+ self.default_transformer_config = TransformerConfig(
+ num_layers=1,
+ hidden_size=12,
+ num_attention_heads=8,
+ num_moe_experts=32,
+ use_cpu_initialization=True,
+ moe_router_load_balancing_type="aux_loss",
+ moe_router_topk=8,
+ moe_aux_loss_coeff=1.0,
+ bf16=True,
+ params_dtype=torch.bfloat16,
+ add_bias_linear=False,
+ tensor_model_parallel_size=tp_size,
+ expert_model_parallel_size=ep_size,
+ context_parallel_size=cp_size,
+ sequence_parallel=sequence_parallel and tp_size > 1,
+ )
+
+ def new_router(self, **kwargs):
+ """Create a new router with updated configuration."""
+ pg_collection = get_default_pg_collection()
+ new_transformer_config = dataclasses.replace(self.default_transformer_config, **kwargs)
+ router = TopKRouter(config=new_transformer_config, pg_collection=pg_collection)
+ router.set_layer_number(0)
+ return router
+
+ @pytest.mark.internal
+ @pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA not available")
+ @pytest.mark.parametrize("sequence_parallel", [True, False])
+ @pytest.mark.parametrize("aux_loss_type", ["aux_loss", "seq_aux_loss", "global_aux_loss"])
+ @pytest.mark.parametrize(
+ "tp_size,ep_size,cp_size", [(8, 1, 1), (4, 2, 1), (1, 1, 8), (2, 1, 4), (2, 2, 2)]
+ )
+ def test_padding_mask_removes_padding_tokens(
+ self, aux_loss_type, tp_size, ep_size, cp_size, sequence_parallel
+ ):
+ """Test that padding tokens are correctly excluded from aux loss calculation."""
+ # Initialize model parallel with given configuration
+ self.setup_model_parallel(
+ tp_size=tp_size, ep_size=ep_size, cp_size=cp_size, sequence_parallel=sequence_parallel
+ )
+
+ try:
+ clear_aux_losses_tracker()
+
+ router = self.new_router(
+ moe_router_load_balancing_type=aux_loss_type,
+ moe_aux_loss_coeff=1.0,
+ moe_router_dtype="fp64",
+ ).cuda()
+
+ seq_len = 32
+ batch_size = 2
+ hidden_size = router.config.hidden_size
+
+ # Create input with padding
+ hidden_states_full = torch.randn(
+ (seq_len, batch_size, hidden_size), dtype=torch.bfloat16, device='cuda'
+ )
+
+ # Create padding mask: first half valid (False), second half padding (True)
+ # Convention: True = padding (exclude), False = valid (include)
+ padding_mask = torch.zeros((seq_len, batch_size), dtype=torch.bool, device='cuda')
+ padding_mask[seq_len // 2 :, :] = True
+
+ # Test with padding mask
+ router.weight.grad = None
+ scores_with_mask, routing_map_with_mask = router(
+ hidden_states_full, padding_mask=padding_mask
+ )
+ scores_with_mask.backward(torch.zeros_like(scores_with_mask))
+
+ loss_name = {
+ "aux_loss": "load_balancing_loss",
+ "seq_aux_loss": "seq_load_balancing_loss",
+ "global_aux_loss": "global_load_balancing_loss",
+ }[aux_loss_type]
+
+ tracker = get_moe_layer_wise_logging_tracker()
+ aux_loss_with_mask = tracker[loss_name]["values"][0].clone()
+ grad_with_mask = router.weight.grad.clone()
+
+ # Test without padding (with only half of the tokens)
+ clear_aux_losses_tracker()
+ router.weight.grad = None
+ hidden_states_valid = hidden_states_full[: seq_len // 2, :, :]
+ scores_without_mask, routing_map_without_mask = router(hidden_states_valid)
+ scores_without_mask.backward(torch.zeros_like(scores_without_mask))
+
+ aux_loss_without_mask = tracker[loss_name]["values"][0].clone()
+ grad_without_mask = router.weight.grad.clone()
+
+ # The aux loss with mask should be close to the aux loss without mask
+ assert torch.equal(aux_loss_with_mask, aux_loss_without_mask)
+ assert torch.equal(grad_with_mask, grad_without_mask)
+
+ clear_aux_losses_tracker()
+ finally:
+ # Always cleanup model parallel
+ Utils.destroy_model_parallel()
+
+ @pytest.mark.internal
+ @pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA not available")
+ @pytest.mark.parametrize(
+ "tp_size,ep_size,cp_size", [(8, 1, 1), (4, 2, 1), (1, 1, 8), (2, 1, 4), (2, 2, 2)]
+ )
+ def test_padding_mask_with_z_loss(self, tp_size, ep_size, cp_size):
+ """Test that padding mask works correctly with z_loss."""
+ # Initialize model parallel with given configuration
+ self.setup_model_parallel(tp_size=tp_size, ep_size=ep_size, cp_size=cp_size)
+
+ try:
+ clear_aux_losses_tracker()
+
+ router = self.new_router(
+ moe_router_load_balancing_type="aux_loss",
+ moe_aux_loss_coeff=0.0,
+ moe_z_loss_coeff=1.0,
+ moe_router_dtype="fp32",
+ ).cuda()
+
+ seq_len = 32
+ batch_size = 2
+ hidden_size = router.config.hidden_size
+
+ # Create input
+ hidden_states_full = torch.randn(
+ (seq_len, batch_size, hidden_size), dtype=torch.bfloat16, device='cuda'
+ )
+
+ # Create padding mask: first half valid (False), second half padding (True)
+ # Convention: True = padding (exclude), False = valid (include)
+ padding_mask = torch.zeros((seq_len, batch_size), dtype=torch.bool, device='cuda')
+ padding_mask[seq_len // 2 :, :] = True
+
+ # Test with padding mask
+ router.weight.grad = None
+ scores_with_mask, _ = router(hidden_states_full, padding_mask=padding_mask)
+ scores_with_mask.sum().backward()
+
+ tracker = get_moe_layer_wise_logging_tracker()
+ z_loss_with_mask = tracker["z_loss"]["values"][0].clone()
+ grad_with_mask = router.weight.grad.clone()
+
+ # Test without padding (with only half of the tokens)
+ clear_aux_losses_tracker()
+ router.weight.grad = None
+ hidden_states_valid = hidden_states_full[: seq_len // 2, :, :]
+ scores_without_mask, _ = router(hidden_states_valid)
+ scores_without_mask.sum().backward()
+
+ z_loss_without_mask = tracker["z_loss"]["values"][0].clone()
+ grad_without_mask = router.weight.grad.clone()
+
+ # The z_loss with mask should be close to the z_loss without mask
+ assert torch.equal(z_loss_with_mask, z_loss_without_mask)
+ assert torch.equal(grad_with_mask, grad_without_mask)
+
+ clear_aux_losses_tracker()
+ finally:
+ # Always cleanup model parallel
+ Utils.destroy_model_parallel()
diff --git a/tests/unit_tests/transformer/moe/test_moe_layer.py b/tests/unit_tests/transformer/moe/test_moe_layer.py
index 59385f757b3..2a2c995257e 100644
--- a/tests/unit_tests/transformer/moe/test_moe_layer.py
+++ b/tests/unit_tests/transformer/moe/test_moe_layer.py
@@ -192,3 +192,87 @@ def test_interleave_transformer_block(self, moe_layer_freq):
def teardown_method(self, method):
Utils.destroy_model_parallel()
+
+
+class TestMoELayerFP16:
+ """Test MoE layer with FP16 precision."""
+
+ def setup_method(self, method):
+ pass
+
+ @pytest.mark.parametrize("moe_token_dispatcher_type", ["allgather", "alltoall"])
+ @pytest.mark.parametrize("num_moe_experts", [2, 4])
+ @pytest.mark.parametrize("tp_size,ep_size", [(1, 1), (2, 2), (4, 2)])
+ def test_moe_layer_fp16_forward_backward(
+ self, num_moe_experts, moe_token_dispatcher_type, tp_size, ep_size
+ ):
+ """Test MoE layer forward and backward pass with fp16 params and inputs."""
+ Utils.initialize_model_parallel(
+ tensor_model_parallel_size=tp_size, expert_model_parallel_size=ep_size
+ )
+ _set_random_seed(seed_=123, data_parallel_random_init=False)
+
+ hidden_size = 64
+ sequence_length = 32
+ micro_batch_size = 2
+
+ transformer_config = TransformerConfig(
+ num_layers=1,
+ hidden_size=hidden_size,
+ num_attention_heads=4,
+ num_moe_experts=num_moe_experts,
+ use_cpu_initialization=False,
+ moe_token_dispatcher_type=moe_token_dispatcher_type,
+ moe_router_load_balancing_type="aux_loss",
+ moe_router_topk=2,
+ moe_aux_loss_coeff=0.01,
+ moe_grouped_gemm=False, # Use SequentialMLP for fp16 test
+ moe_ffn_hidden_size=256,
+ add_bias_linear=False,
+ tensor_model_parallel_size=tp_size,
+ expert_model_parallel_size=ep_size,
+ sequence_parallel=tp_size > 1,
+ fp16=True,
+ params_dtype=torch.float16,
+ )
+
+ transformer_layer_spec = get_gpt_layer_local_spec(
+ num_experts=num_moe_experts, moe_grouped_gemm=False
+ )
+
+ moe_layer = MoELayer(
+ transformer_config, transformer_layer_spec.submodules.mlp.submodules
+ ).cuda()
+
+ hidden_states = torch.randn(
+ sequence_length,
+ micro_batch_size,
+ hidden_size,
+ device=torch.cuda.current_device(),
+ dtype=torch.float16,
+ requires_grad=True,
+ )
+
+ # Forward pass
+ output, _ = moe_layer(hidden_states)
+
+ assert output.dtype == torch.float16, f"Expected fp16 output, got {output.dtype}"
+ assert output.shape == hidden_states.shape, f"Output shape mismatch"
+
+ # Backward pass
+ loss = output.sum()
+ loss.backward()
+
+ assert hidden_states.grad is not None, "Input gradients should exist"
+ assert (
+ hidden_states.grad.dtype == torch.float16
+ ), f"Expected fp16 gradients, got {hidden_states.grad.dtype}"
+
+ for name, param in moe_layer.named_parameters():
+ if param.requires_grad:
+ assert param.grad is not None, f"Gradient for {name} should exist"
+
+ Utils.destroy_model_parallel()
+
+ def teardown_method(self, method):
+ Utils.destroy_model_parallel()
diff --git a/tests/unit_tests/transformer/moe/test_router_replay.py b/tests/unit_tests/transformer/moe/test_router_replay.py
new file mode 100644
index 00000000000..840fc0fd269
--- /dev/null
+++ b/tests/unit_tests/transformer/moe/test_router_replay.py
@@ -0,0 +1,95 @@
+# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+import pytest
+import torch
+
+from megatron.core.transformer.moe.moe_utils import topk_routing_with_score_function
+from megatron.core.transformer.moe.router_replay import RouterReplay, RouterReplayAction
+
+
+def setup_function():
+ RouterReplay.global_router_replay_instances.clear()
+
+
+def teardown_function():
+ RouterReplay.global_router_replay_instances.clear()
+
+
+def test_record_mode_with_topk_routing_softmax_post():
+ rr = RouterReplay()
+ rr.set_router_replay_action(RouterReplayAction.RECORD)
+ logits = torch.randn(4, 6)
+ probs, routing_map = topk_routing_with_score_function(
+ logits=logits, topk=2, use_pre_softmax=False, router_replay=rr, score_function="softmax"
+ )
+ recorded = rr.get_recorded_indices()
+ expected_idx = torch.topk(logits, k=2, dim=1).indices
+ assert recorded is not None
+ assert torch.equal(recorded, expected_idx)
+ assert probs.shape == (4, 6)
+ assert routing_map.shape == (4, 6)
+ assert routing_map.sum(dim=1).eq(2).all()
+
+
+def test_replay_forward_with_topk_routing_softmax_pre():
+ rr = RouterReplay()
+ rr.set_router_replay_action(RouterReplayAction.REPLAY_FORWARD)
+ logits = torch.randn(3, 5)
+ target = torch.tensor([[1, 2], [0, 3], [2, 4]], dtype=torch.long)
+ rr.set_target_indices(target)
+ probs, routing_map = topk_routing_with_score_function(
+ logits=logits, topk=2, use_pre_softmax=True, router_replay=rr, score_function="softmax"
+ )
+ assert routing_map.sum(dim=1).eq(2).all()
+ scores = torch.softmax(logits, dim=-1)
+ assert torch.equal(probs.gather(1, target), scores.gather(1, target))
+
+
+def test_replay_forward_with_topk_routing_softmax_post():
+ rr = RouterReplay()
+ rr.set_router_replay_action(RouterReplayAction.REPLAY_FORWARD)
+ logits = torch.randn(3, 6)
+ target = torch.tensor([[1, 2], [0, 5], [3, 4]], dtype=torch.long)
+ rr.set_target_indices(target)
+ probs, routing_map = topk_routing_with_score_function(
+ logits=logits, topk=2, use_pre_softmax=False, router_replay=rr, score_function="softmax"
+ )
+ selected = torch.softmax(logits.gather(1, target), dim=-1)
+ assert torch.equal(probs.gather(1, target), selected)
+ assert routing_map.sum(dim=1).eq(2).all()
+
+
+def test_global_set_get_clear_indices():
+ r1 = RouterReplay()
+ r2 = RouterReplay()
+ t1 = torch.tensor([[0, 1]], dtype=torch.long)
+ t2 = torch.tensor([[1, 0]], dtype=torch.long)
+ RouterReplay.set_replay_data([t1, t2])
+ assert torch.equal(r1.target_topk_idx, t1)
+ assert torch.equal(r2.target_topk_idx, t2)
+ r1.record_indices(t1)
+ r2.record_indices(t2)
+ rec = RouterReplay.get_recorded_data()
+ assert len(rec) == 2
+ assert torch.equal(rec[0], t1)
+ assert torch.equal(rec[1], t2)
+ RouterReplay.clear_global_indices()
+ assert r1.target_topk_idx is None and r2.target_topk_idx is None
+ assert r1.get_recorded_indices() is None and r2.get_recorded_indices() is None
+
+
+def test_global_action_set_and_clear():
+ r1 = RouterReplay()
+ r2 = RouterReplay()
+ RouterReplay.set_global_router_replay_action(RouterReplayAction.REPLAY_FORWARD)
+ assert r1.router_replay_action == RouterReplayAction.REPLAY_FORWARD
+ assert r2.router_replay_action == RouterReplayAction.REPLAY_FORWARD
+ RouterReplay.clear_global_router_replay_action()
+ assert r1.router_replay_action is None and r2.router_replay_action is None
+
+
+def test_set_replay_data_length_mismatch():
+ _ = RouterReplay()
+ with pytest.raises(ValueError):
+ RouterReplay.set_replay_data(
+ [torch.tensor([[0, 1]], dtype=torch.long), torch.tensor([[1, 0]], dtype=torch.long)]
+ )
diff --git a/tests/unit_tests/transformer/moe/test_routers.py b/tests/unit_tests/transformer/moe/test_routers.py
index 677d938cdc7..abd1a4db2dc 100644
--- a/tests/unit_tests/transformer/moe/test_routers.py
+++ b/tests/unit_tests/transformer/moe/test_routers.py
@@ -125,6 +125,53 @@ def test_aux_loss(self):
out.sum().mul_(0).backward()
assert self.sequential_mlp.router.weight.grad.abs().sum() > 0
+ @pytest.mark.internal
+ @pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA not available")
+ def test_router_with_padding_mask(self):
+ """Test that padding mask correctly excludes padding tokens from routing."""
+ self.router = self.router.cuda()
+ seq_len = 32
+ batch_size = 2
+ hidden_size = self.router.config.hidden_size
+
+ # Create input with shape [seq_len, batch_size, hidden_size]
+ hidden_states = torch.randn((seq_len, batch_size, hidden_size)).cuda().bfloat16()
+
+ # Create padding mask: first half valid (False), second half padding (True)
+ # padding_mask shape: [seq_len, batch_size]
+ # Convention: True = padding (exclude), False = valid (include)
+ padding_mask = torch.zeros((seq_len, batch_size), dtype=torch.bool, device='cuda')
+ padding_mask[seq_len // 2 :, :] = True # Second half is padding
+
+ # Test forward pass with padding mask
+ with torch.no_grad():
+ probs_with_mask, routing_map_with_mask = self.router(
+ hidden_states, padding_mask=padding_mask
+ )
+
+ # Test forward pass without padding mask (only valid tokens)
+ hidden_states_valid = hidden_states[: seq_len // 2, :, :]
+ probs_without_mask, routing_map_without_mask = self.router(hidden_states_valid)
+
+ # The valid part of routing with mask should match routing without mask
+ probs_valid_part = probs_with_mask.reshape(seq_len, batch_size, -1)[
+ : seq_len // 2, :, :
+ ]
+ probs_valid_part = probs_valid_part.reshape(-1, probs_valid_part.shape[-1])
+
+ # Check that shapes are as expected
+ assert probs_with_mask.shape == (
+ seq_len * batch_size,
+ self.router.config.num_moe_experts,
+ )
+ assert routing_map_with_mask.shape == (
+ seq_len * batch_size,
+ self.router.config.num_moe_experts,
+ )
+
+ # Verify that probs for valid tokens are similar
+ assert torch.equal(probs_valid_part, probs_without_mask)
+
@pytest.mark.internal
@pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA not available")
def test_router_dtype(self):
diff --git a/tests/unit_tests/transformer/moe/test_shared_experts.py b/tests/unit_tests/transformer/moe/test_shared_experts.py
index f721c482937..6df4d2fd369 100644
--- a/tests/unit_tests/transformer/moe/test_shared_experts.py
+++ b/tests/unit_tests/transformer/moe/test_shared_experts.py
@@ -20,7 +20,8 @@ def teardown_method(self, method):
@pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA not available")
@pytest.mark.internal
- def test_gpu_forward(self):
+ @pytest.mark.parametrize("shared_expert_gate", [False, True])
+ def test_gpu_forward(self, shared_expert_gate):
Utils.initialize_model_parallel(1, 1)
model_parallel_cuda_manual_seed(123)
print("done intializing")
@@ -38,6 +39,7 @@ def test_gpu_forward(self):
moe_router_load_balancing_type="sinkhorn",
moe_router_topk=1,
add_bias_linear=False,
+ moe_shared_expert_gate=shared_expert_gate,
)
transformer_layer_spec = get_gpt_layer_local_spec(
num_experts=num_moe_experts, moe_grouped_gemm=False
@@ -49,7 +51,10 @@ def test_gpu_forward(self):
assert isinstance(self.moe_layer, MoELayer)
num_weights = sum([p.numel() for p in self.moe_layer.parameters()])
- assert num_weights == 3480 + 1152
+ if shared_expert_gate:
+ assert num_weights == 3480 + 1152 + 12 # 12 is the weight of the gate
+ else:
+ assert num_weights == 3480 + 1152
assert self.moe_layer.shared_experts is not None
assert self.moe_layer.shared_experts.stream is None
assert self.moe_layer.token_dispatcher.shared_experts is None
diff --git a/tests/unit_tests/transformer/moe/test_token_dispatcher.py b/tests/unit_tests/transformer/moe/test_token_dispatcher.py
index c2462ef73ad..6a155920e2f 100644
--- a/tests/unit_tests/transformer/moe/test_token_dispatcher.py
+++ b/tests/unit_tests/transformer/moe/test_token_dispatcher.py
@@ -1,4 +1,4 @@
-# Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved.
+# Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
import copy
import dataclasses
diff --git a/tests/unit_tests/transformer/test_attention.py b/tests/unit_tests/transformer/test_attention.py
index 247d3db853e..b5f2857d622 100644
--- a/tests/unit_tests/transformer/test_attention.py
+++ b/tests/unit_tests/transformer/test_attention.py
@@ -1,20 +1,44 @@
-# Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved.
+# Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
import copy
+from unittest import mock
+import einops
import pytest
import torch
from packaging import version
+from torch.nn import functional as F
import megatron.core.parallel_state as parallel_state
from megatron.core.hyper_comm_grid import HyperCommGrid
-from megatron.core.models.gpt.gpt_layer_specs import get_gpt_layer_with_transformer_engine_spec
+from megatron.core.models.common.embeddings.rope_utils import (
+ get_pos_emb_on_this_cp_rank as get_tensor_on_this_cp_rank,
+)
+from megatron.core.models.gpt.gpt_layer_specs import (
+ get_gpt_layer_local_spec,
+ get_gpt_layer_with_transformer_engine_spec,
+)
+from megatron.core.models.gpt.gpt_model import GPTModel
from megatron.core.process_groups_config import ProcessGroupCollection
from megatron.core.tensor_parallel.random import model_parallel_cuda_manual_seed
from megatron.core.transformer import TransformerConfig
from megatron.core.transformer.attention import SelfAttention
+from megatron.core.transformer.dot_product_attention_context_parallel import (
+ AttentionFuncionWithContextParallel,
+ to_zz_mask_attn_bias,
+)
from megatron.core.transformer.enums import AttnMaskType
from megatron.core.utils import is_te_min_version
+from megatron.training.arguments import parse_args
+from megatron.training.checkpointing import load_checkpoint, save_checkpoint
+from megatron.training.global_vars import set_args
+from megatron.training.training import get_model
+from megatron.training.utils import unwrap_model
+from tests.unit_tests.dist_checkpointing import (
+ TempNamedDir,
+ init_basic_mock_args,
+ init_checkpointing_mock_args,
+)
from tests.unit_tests.test_utilities import Utils
try:
@@ -25,9 +49,20 @@
HAVE_FUSED_QKV_ROPE = False
+@pytest.mark.parametrize("output_gate", [False, True])
+@pytest.mark.parametrize(
+ ("transformer_impl", "fallback_to_eager_attn"),
+ [("transformer_engine", False), ("transformer_engine", True), ("native", False)],
+)
class TestParallelAttention:
- def setup_method(self, method):
+ @pytest.fixture(scope='function', autouse=True)
+ def setup_method(self, output_gate, transformer_impl, fallback_to_eager_attn):
+ if output_gate:
+ if transformer_impl == "native":
+ pytest.skip("Native implementation does not support output gate.")
+ if fallback_to_eager_attn:
+ pytest.skip("No need to test output gate for fallback_to_eager_attn = True.")
Utils.initialize_model_parallel(1, 1)
model_parallel_cuda_manual_seed(123)
self.transformer_config = TransformerConfig(
@@ -37,14 +72,22 @@ def setup_method(self, method):
use_cpu_initialization=True,
bf16=True,
params_dtype=torch.bfloat16,
+ attention_output_gate=output_gate,
+ transformer_impl=transformer_impl,
+ fallback_to_eager_attn=fallback_to_eager_attn,
)
+ if transformer_impl == "transformer_engine":
+ layer_spec = get_gpt_layer_with_transformer_engine_spec(
+ fallback_to_eager_attn=fallback_to_eager_attn
+ )
+ else:
+ layer_spec = get_gpt_layer_local_spec()
+ attn_layer_spec = layer_spec.submodules.self_attention.submodules
self.parallel_attention = SelfAttention(
- self.transformer_config,
- get_gpt_layer_with_transformer_engine_spec().submodules.self_attention.submodules,
- layer_number=1,
+ self.transformer_config, attn_layer_spec, layer_number=1
)
- def teardown_method(self, method):
+ def teardown_method(self):
Utils.destroy_model_parallel()
def test_constructor(self):
@@ -52,7 +95,19 @@ def test_constructor(self):
assert self.parallel_attention.layer_number == 1
num_weights = sum([p.numel() for p in self.parallel_attention.parameters()])
- assert num_weights == 66304
+
+ hidden_size = self.transformer_config.hidden_size
+ standard_num_weights = (
+ hidden_size * hidden_size * 4 + hidden_size * 4 # QKVO weight # QKVO bias
+ )
+ if self.transformer_config.attention_output_gate:
+ standard_num_weights += hidden_size * hidden_size + hidden_size # Gate weight and bias
+ if self.transformer_config.transformer_impl == "transformer_engine":
+ standard_num_weights += hidden_size * 2 # fused pre layernorm weight and bias
+
+ assert (
+ num_weights == standard_num_weights
+ ), f"{num_weights=} does not match {standard_num_weights=}."
def test_cpu_forward(self):
# we can't currently do this because the global memory buffer is on GPU
@@ -87,9 +142,13 @@ def test_gpu_forward(self):
@pytest.mark.parametrize("rotary_interleaved", [True, False])
@pytest.mark.parametrize("fused_qkv_rope", [True, False])
def test_fused_rope_gpu_forward(self, rotary_interleaved, fused_qkv_rope):
+ if self.transformer_config.fallback_to_eager_attn:
+ pytest.skip("No need to test fused RoPE for fallback_to_eager_attn = True.")
self.parallel_attention.config.apply_rope_fusion = True
if rotary_interleaved and not is_te_min_version("2.3.0"):
pytest.skip("Only TE >= 2.3.0 supports interleaved fused RoPE.")
+ if fused_qkv_rope and self.parallel_attention.config.attention_output_gate:
+ pytest.skip("Fused QKV RoPE does not support gated attention for now.")
if fused_qkv_rope and not HAVE_FUSED_QKV_ROPE:
pytest.skip("Fused QKV RoPE not available.")
self.parallel_attention.config.rotary_interleaved = rotary_interleaved
@@ -346,23 +405,222 @@ def test_clip_qk_mixed_logits(self):
class TestSelfAttention:
def setup_method(self, method):
- Utils.destroy_model_parallel()
+ Utils.initialize_model_parallel(1, 1)
+ model_parallel_cuda_manual_seed(123)
def teardown_method(self, method):
Utils.destroy_model_parallel()
+ def test_clip_qk_disabled_raises_error(self):
+ """Test that clip_qk raises ValueError when qk_clip is not enabled."""
+ transformer_config = TransformerConfig(
+ num_layers=2,
+ hidden_size=128,
+ num_attention_heads=4,
+ use_cpu_initialization=True,
+ qk_clip=False,
+ )
+ attention = SelfAttention(
+ transformer_config,
+ get_gpt_layer_with_transformer_engine_spec().submodules.self_attention.submodules,
+ layer_number=1,
+ )
+
+ with pytest.raises(ValueError, match="qk_clip option needs to be enabled"):
+ attention.clip_qk()
+
+ def test_clip_qk_none_logits_raises_error(self):
+ """Test that clip_qk raises ValueError when current_max_attn_logits is None."""
+ transformer_config = TransformerConfig(
+ num_layers=2,
+ hidden_size=128,
+ num_attention_heads=4,
+ use_cpu_initialization=True,
+ qk_clip=True,
+ qk_clip_threshold=100.0,
+ qk_clip_alpha=0.5,
+ )
+ attention = SelfAttention(
+ transformer_config,
+ get_gpt_layer_with_transformer_engine_spec().submodules.self_attention.submodules,
+ layer_number=1,
+ )
+
+ with pytest.raises(ValueError, match="current_max_attn_logits is None"):
+ attention.clip_qk()
+
+ def test_clip_qk_below_threshold_no_update(self):
+ """Test that weights are not updated when max logits are below threshold."""
+ transformer_config = TransformerConfig(
+ num_layers=2,
+ hidden_size=128,
+ num_attention_heads=4,
+ use_cpu_initialization=True,
+ qk_clip=True,
+ qk_clip_threshold=100.0,
+ qk_clip_alpha=0.5,
+ )
+ attention = SelfAttention(
+ transformer_config,
+ get_gpt_layer_with_transformer_engine_spec().submodules.self_attention.submodules,
+ layer_number=1,
+ )
+ attention.cuda()
+
+ # Save original weights
+ original_weight = attention.linear_qkv.weight.data.clone()
+
+ # Set current_max_attn_logits below threshold
+ attention.core_attention.current_max_attn_logits = torch.tensor(
+ [50.0, 60.0, 70.0, 80.0], device='cuda'
+ )
+
+ # Call clip_qk
+ attention.clip_qk()
+
+ # Weights should not be updated
+ assert torch.equal(attention.linear_qkv.weight.data, original_weight)
+ # current_max_attn_logits should be reset
+ assert attention.core_attention.current_max_attn_logits is None
+
+ def test_clip_qk_above_threshold_updates_weights(self):
+ """Test that weights are updated when max logits exceed threshold."""
+ transformer_config = TransformerConfig(
+ num_layers=2,
+ hidden_size=128,
+ num_attention_heads=4,
+ use_cpu_initialization=True,
+ qk_clip=True,
+ qk_clip_threshold=100.0,
+ qk_clip_alpha=0.5,
+ )
+ attention = SelfAttention(
+ transformer_config,
+ get_gpt_layer_with_transformer_engine_spec().submodules.self_attention.submodules,
+ layer_number=1,
+ )
+ attention.cuda()
+
+ # Save original weights
+ original_weight = attention.linear_qkv.weight.data.clone()
+
+ # Set current_max_attn_logits above threshold
+ attention.core_attention.current_max_attn_logits = torch.tensor(
+ [150.0, 160.0, 170.0, 180.0], device='cuda'
+ )
+
+ # Call clip_qk
+ attention.clip_qk()
+
+ # Weights should be updated
+ assert not torch.equal(attention.linear_qkv.weight.data, original_weight)
+ # current_max_attn_logits should be reset
+ assert attention.core_attention.current_max_attn_logits is None
+
+ def test_clip_qk_gqa_configuration(self):
+ """Test clip_qk with GQA (Grouped Query Attention) configuration."""
+ transformer_config = TransformerConfig(
+ num_layers=2,
+ hidden_size=128,
+ num_attention_heads=8,
+ num_query_groups=4, # GQA with 2 heads per group
+ use_cpu_initialization=True,
+ qk_clip=True,
+ qk_clip_threshold=100.0,
+ qk_clip_alpha=0.5,
+ )
+ attention = SelfAttention(
+ transformer_config,
+ get_gpt_layer_with_transformer_engine_spec().submodules.self_attention.submodules,
+ layer_number=1,
+ )
+ attention.cuda()
+
+ # Save original weights
+ original_weight = attention.linear_qkv.weight.data.clone()
+
+ # Set current_max_attn_logits for all heads (8 heads)
+ attention.core_attention.current_max_attn_logits = torch.tensor(
+ [150.0, 160.0, 170.0, 180.0, 190.0, 200.0, 210.0, 220.0], device='cuda'
+ )
+
+ # Call clip_qk
+ attention.clip_qk()
+
+ # Weights should be updated
+ assert not torch.equal(attention.linear_qkv.weight.data, original_weight)
+ # current_max_attn_logits should be reset
+ assert attention.core_attention.current_max_attn_logits is None
+
+ def test_clip_qk_mixed_logits(self):
+ """Test clip_qk with mixed logits (some above, some below threshold)."""
+ transformer_config = TransformerConfig(
+ num_layers=2,
+ hidden_size=128,
+ num_attention_heads=4,
+ use_cpu_initialization=True,
+ qk_clip=True,
+ qk_clip_threshold=100.0,
+ qk_clip_alpha=0.5,
+ )
+ attention = SelfAttention(
+ transformer_config,
+ get_gpt_layer_with_transformer_engine_spec().submodules.self_attention.submodules,
+ layer_number=1,
+ )
+ attention.cuda()
+
+ # Save original weights
+ original_weight = attention.linear_qkv.weight.data.clone()
+
+ # Set mixed current_max_attn_logits (some above, some below threshold)
+ attention.core_attention.current_max_attn_logits = torch.tensor(
+ [80.0, 150.0, 90.0, 200.0], device='cuda'
+ )
+
+ # Call clip_qk
+ attention.clip_qk()
+
+ # Weights should be updated since at least one head exceeds threshold
+ assert not torch.equal(attention.linear_qkv.weight.data, original_weight)
+ # current_max_attn_logits should be reset
+ assert attention.core_attention.current_max_attn_logits is None
+
+
+@pytest.mark.parametrize("output_gate", [False, True])
+@pytest.mark.parametrize("transformer_impl", ["transformer_engine", "native"])
+class TestSelfAttention:
+
+ @pytest.fixture(scope='function', autouse=True)
+ def setup_method(self, output_gate, transformer_impl):
+ if transformer_impl == "native":
+ if output_gate:
+ pytest.skip("Native implementation does not support output gate.")
+ self.transformer_impl = transformer_impl
+ self.output_gate = output_gate
+ Utils.destroy_model_parallel()
+
+ def teardown_method(self):
+ Utils.destroy_model_parallel()
+
def run_self_attention(self, pg_collection):
tensor_model_parallel_size = torch.distributed.get_world_size(pg_collection.tp)
self.transformer_config = TransformerConfig(
num_layers=2,
hidden_size=128,
num_attention_heads=4,
+ attention_output_gate=self.output_gate,
tensor_model_parallel_size=tensor_model_parallel_size,
use_cpu_initialization=False,
+ transformer_impl=self.transformer_impl,
)
+ if self.transformer_impl == "transformer_engine":
+ get_gpt_layer_spec_fn = get_gpt_layer_with_transformer_engine_spec
+ else:
+ get_gpt_layer_spec_fn = get_gpt_layer_local_spec
self.self_attention = SelfAttention(
self.transformer_config,
- get_gpt_layer_with_transformer_engine_spec().submodules.self_attention.submodules,
+ get_gpt_layer_spec_fn().submodules.self_attention.submodules,
layer_number=1,
attn_mask_type=AttnMaskType.causal,
pg_collection=pg_collection,
@@ -435,3 +693,360 @@ def test_self_attention_independent_pg_smoke(self):
pg_collection = ProcessGroupCollection(tp=tp_group, cp=cp_group)
self.run_self_attention(pg_collection)
+
+
+def _test_parallel_attention_correctness(
+ transformer_config,
+ transformer_layer_spec,
+ tmp_path_dist_ckpt,
+ atol,
+ rtol,
+ tp=1,
+ sp=False,
+ cp=1,
+ seed=123,
+ sequence_length=256,
+ micro_batch_size=4,
+):
+ # Model initialization function
+ def initialize_gpt_model(
+ config, pre_process=True, post_process=True, vp_stage=None, pg_collection=None
+ ):
+ gpt_model = GPTModel(
+ config=config,
+ transformer_layer_spec=transformer_layer_spec,
+ vocab_size=128,
+ max_sequence_length=sequence_length,
+ pre_process=pre_process,
+ post_process=post_process,
+ vp_stage=vp_stage,
+ pg_collection=pg_collection,
+ )
+ return gpt_model
+
+ # Initialize baseline parallel state
+ Utils.initialize_model_parallel(
+ tensor_model_parallel_size=1, pipeline_model_parallel_size=1, context_parallel_size=1
+ )
+
+ # Initialize input hidden states
+ torch.manual_seed(seed)
+ model_parallel_cuda_manual_seed(seed)
+ input_hidden_states = (
+ torch.rand((sequence_length, micro_batch_size, transformer_config.hidden_size))
+ .cuda()
+ .bfloat16()
+ .requires_grad_(True)
+ )
+
+ with TempNamedDir(tmp_path_dist_ckpt / 'test_parallel_attn', sync=True) as ckpt_dir:
+ # Set argument
+ mock_args = parse_args(ignore_unknown_args=True)
+ set_args(mock_args)
+
+ # Initialize baseline model
+ init_basic_mock_args(mock_args, 1, 1, bf16=True)
+ mock_args.context_parallel_size = 1
+ mock_args.sequence_parallel = 1
+ gpt_model = unwrap_model(get_model(initialize_gpt_model, config=transformer_config))
+
+ # Initialize args and save checkpoint
+ init_checkpointing_mock_args(mock_args, ckpt_dir, False)
+ mock_args.no_save_optim = True
+ mock_args.no_save_rng = True
+ mock_args.no_load_optim = True
+ mock_args.no_load_rng = True
+ save_checkpoint(10, gpt_model, None, None, 0)
+
+ # Calculate baseline output
+ attention = gpt_model[0].decoder.layers[0].self_attention
+ output_hidden_states_baseline, bias_hidden_states_baseline = attention(
+ input_hidden_states, attention_mask=None
+ )
+ output_hidden_states_baseline.sum().backward()
+
+ # Save baseline output
+ input_grad_baseline = input_hidden_states.grad.detach()
+ output_hidden_states_baseline = output_hidden_states_baseline.detach()
+ bias_hidden_states_baseline = bias_hidden_states_baseline
+ if bias_hidden_states_baseline is not None:
+ bias_hidden_states_baseline = bias_hidden_states_baseline.detach()
+ has_bias = True
+ else:
+ has_bias = False
+
+ # Initialize parallel model
+ Utils.destroy_model_parallel()
+ Utils.initialize_model_parallel(
+ tensor_model_parallel_size=tp, pipeline_model_parallel_size=1, context_parallel_size=cp
+ )
+ torch.manual_seed(seed)
+ model_parallel_cuda_manual_seed(seed)
+ transformer_config.context_parallel_size = cp
+ transformer_config.tensor_model_parallel_size = tp
+ transformer_config.sequence_parallel = sp
+ init_basic_mock_args(mock_args, tp, 1, bf16=True)
+ mock_args.context_parallel_size = cp
+ mock_args.sequence_parallel = sp
+ gpt_model = unwrap_model(get_model(initialize_gpt_model, config=transformer_config))
+ with mock.patch('megatron.training.checkpointing.check_checkpoint_args'):
+ with mock.patch('megatron.training.checkpointing.update_num_microbatches'):
+ load_checkpoint(gpt_model, None, None)
+
+ # Function to get tensor on this tp and cp rank
+ cp_group = parallel_state.get_context_parallel_group()
+ tp_rank = parallel_state.get_tensor_model_parallel_rank()
+
+ def get_tensor_on_this_rank(tensor):
+ if cp > 1:
+ tensor = get_tensor_on_this_cp_rank(tensor, 0, cp_group)
+ if tp > 1 and sp:
+ sp_seg = sequence_length // tp // cp
+ tensor = tensor[tp_rank * sp_seg : (tp_rank + 1) * sp_seg]
+ return tensor
+
+ # Calculate parallel model output
+ input_hidden_states = get_tensor_on_this_rank(input_hidden_states)
+ input_hidden_states = input_hidden_states.detach().requires_grad_(True)
+ parallel_attention = gpt_model[0].decoder.layers[0].self_attention
+ output_hidden_states_parallel, bias_hidden_states_parallel = parallel_attention(
+ input_hidden_states, attention_mask=None
+ )
+ output_hidden_states_parallel.sum().backward()
+ input_grad_parallel = input_hidden_states.grad.detach()
+
+ # Check if the output is close
+ output_hidden_states_baseline = get_tensor_on_this_rank(output_hidden_states_baseline)
+ input_grad_baseline = get_tensor_on_this_rank(input_grad_baseline)
+
+ assert torch.all(
+ ~torch.isnan(output_hidden_states_baseline)
+ ), "output_hidden_states_baseline contains nan"
+ assert torch.all(
+ ~torch.isinf(output_hidden_states_baseline)
+ ), "output_hidden_states_baseline contains inf"
+ assert torch.all(~torch.isnan(input_grad_baseline)), "input_grad_baseline contains nan"
+ assert torch.all(~torch.isinf(input_grad_baseline)), "input_grad_baseline contains inf"
+ assert torch.all(
+ ~torch.isnan(output_hidden_states_parallel)
+ ), "output_hidden_states_parallel contains nan"
+ assert torch.all(
+ ~torch.isinf(output_hidden_states_parallel)
+ ), "output_hidden_states_parallel contains inf"
+ assert torch.all(~torch.isnan(input_grad_parallel)), "input_grad_parallel contains nan"
+ assert torch.all(~torch.isinf(input_grad_parallel)), "input_grad_parallel contains inf"
+ if has_bias:
+ assert torch.all(
+ ~torch.isnan(bias_hidden_states_baseline)
+ ), "bias_hidden_states_baseline contains nan"
+ assert torch.all(
+ ~torch.isinf(bias_hidden_states_baseline)
+ ), "bias_hidden_states_baseline contains inf"
+ assert torch.all(
+ ~torch.isnan(bias_hidden_states_parallel)
+ ), "bias_hidden_states_parallel contains nan"
+ assert torch.all(
+ ~torch.isinf(bias_hidden_states_parallel)
+ ), "bias_hidden_states_parallel contains inf"
+
+ torch.testing.assert_close(
+ output_hidden_states_baseline,
+ output_hidden_states_parallel,
+ atol=atol,
+ rtol=rtol,
+ msg=lambda msg: f"Mismatch in output_hidden_states: {msg}",
+ )
+ torch.testing.assert_close(
+ input_grad_baseline,
+ input_grad_parallel,
+ atol=atol,
+ rtol=rtol,
+ msg=lambda msg: f"Mismatch in input_grad: {msg}",
+ )
+ if has_bias:
+ torch.testing.assert_close(
+ bias_hidden_states_baseline,
+ bias_hidden_states_parallel,
+ atol=atol,
+ rtol=rtol,
+ msg=lambda msg: f"Mismatch in bias_hidden_states: {msg}",
+ )
+
+ Utils.destroy_model_parallel()
+
+
+# TODO(yuzhongw): Add test case for fallback_to_eager_attn
+@pytest.mark.parametrize("apply_rope_fusion", [False, True])
+@pytest.mark.parametrize(
+ ("tp", "sp", "cp"),
+ [
+ (4, False, 1), # TP w/o SP
+ (4, True, 1), # TP w/ SP
+ (1, False, 4), # CP
+ (2, False, 2), # CP + TP w/o SP
+ (2, True, 2), # CP + TP w/ SP
+ ],
+)
+@pytest.mark.parametrize("qk_layernorm", [False, True])
+@pytest.mark.parametrize("output_gate", [False, True])
+def test_parallel_attention_correctness(
+ tmp_path_dist_ckpt, apply_rope_fusion, tp, sp, cp, qk_layernorm, output_gate
+):
+ transformer_config = TransformerConfig(
+ num_layers=1,
+ hidden_size=128,
+ num_attention_heads=4,
+ normalization="RMSNorm",
+ bf16=True,
+ qk_layernorm=qk_layernorm,
+ apply_rope_fusion=apply_rope_fusion,
+ attention_output_gate=output_gate,
+ hidden_dropout=0.0,
+ attention_dropout=0.0,
+ )
+
+ transformer_layer_spec = get_gpt_layer_with_transformer_engine_spec(qk_layernorm=qk_layernorm)
+ atol, rtol = 1e-2, 1e-2
+
+ _test_parallel_attention_correctness(
+ transformer_config,
+ transformer_layer_spec,
+ tmp_path_dist_ckpt,
+ atol=atol,
+ rtol=rtol,
+ tp=tp,
+ sp=sp,
+ cp=cp,
+ seed=123,
+ sequence_length=256,
+ )
+
+
+def _torch_native_attention(query, key, value, attention_mask, sinks, scaling: float):
+ """Torch native attention implementation
+ This was not in the original implementation and slightly affect results;
+ it prevents overflow in BF16/FP16 when training with batch size > 1 we clamp max values.
+ """
+ # Rearrange query, key, value to (b, h, s, d)
+ query = einops.rearrange(query, 's b h d -> b h s d')
+ key = einops.rearrange(key, 's b h d -> b h s d')
+ value = einops.rearrange(value, 's b h d -> b h s d')
+
+ # Compute attention weights
+ attn_weights = torch.matmul(query, key.transpose(2, 3)) * scaling
+ if attention_mask is not None:
+ nheads = query.shape[1]
+ nheads_k = key.shape[1]
+ heads_k_stride = 1
+ mask_bias = to_zz_mask_attn_bias(
+ attention_mask, 1, nheads, nheads_k, heads_k_stride, query.device, query.dtype
+ )
+ attn_weights = attn_weights + mask_bias
+
+ # Add sinks to attention weights
+ if sinks is None:
+ combined_logits = attn_weights
+ else:
+ sinks = sinks.reshape(1, -1, 1, 1).expand(query.shape[0], -1, query.shape[-2], -1)
+ combined_logits = torch.cat([attn_weights, sinks], dim=-1)
+
+ # Compute attention scores
+ probs = F.softmax(combined_logits, dim=-1, dtype=combined_logits.dtype)
+ if sinks is None:
+ scores = probs
+ else:
+ scores = probs[..., :-1]
+
+ # Compute attention output
+ attn_output = torch.matmul(scores, value)
+ attn_output = einops.rearrange(attn_output, 'b h s d -> s b h d')
+ attn_output = attn_output.contiguous()
+ return attn_output
+
+
+def test_eager_attention_function_correctness():
+ """Test the correctness of the context parallel eager attention function"""
+
+ # Configuration
+ batch_size = 4
+ num_heads = 2
+ head_dim = 256
+ seq_len_q = 512
+ seq_len_k = 2048
+ scale = 1 / (head_dim**2)
+
+ # Initialize inputs
+ q = torch.rand(
+ (seq_len_q, batch_size, num_heads, head_dim),
+ device='cuda',
+ dtype=torch.bfloat16,
+ requires_grad=True,
+ )
+ k = torch.rand(
+ (seq_len_k, batch_size, num_heads, head_dim),
+ device='cuda',
+ dtype=torch.bfloat16,
+ requires_grad=True,
+ )
+ v = torch.rand(
+ (seq_len_k, batch_size, num_heads, head_dim),
+ device='cuda',
+ dtype=torch.bfloat16,
+ requires_grad=True,
+ )
+
+ def randbool(shape, **kwargs):
+ return torch.randn(shape, **kwargs) > 0
+
+ attn_bias = randbool((batch_size, 1, seq_len_q, seq_len_k), device='cuda')
+ sinks = None
+
+ # Torch native attention forward and backward pass
+ out_torch = _torch_native_attention(
+ query=q, key=k, value=v, attention_mask=attn_bias, sinks=sinks, scaling=scale
+ )
+ loss_torch = out_torch.sum()
+ loss_torch.backward()
+ torch_q_grad = q.grad.clone()
+ torch_k_grad = k.grad.clone()
+ torch_v_grad = v.grad.clone()
+ q.grad.zero_()
+ k.grad.zero_()
+ v.grad.zero_()
+ if sinks is not None:
+ torch_sinks_grad = sinks.grad.clone()
+ sinks.grad.zero_()
+ else:
+ torch_sinks_grad = None
+
+ # Custom attention forward and backward pass
+ out_custom = AttentionFuncionWithContextParallel.apply(
+ q, k, v, attn_bias, 0.0, scale, None # dropout
+ )
+ loss_custom = out_custom.sum()
+ loss_custom.backward()
+ custom_q_grad = q.grad.clone()
+ custom_k_grad = k.grad.clone()
+ custom_v_grad = v.grad.clone()
+ q.grad.zero_()
+ k.grad.zero_()
+ v.grad.zero_()
+ if sinks is not None:
+ custom_sinks_grad = sinks.grad.clone()
+ sinks.grad.zero_()
+ else:
+ custom_sinks_grad = None
+
+ # Check attention output and gradients
+ assert torch.equal(out_custom, out_torch), "Mismatch in attention output"
+ tol = {"atol": 1e-4, "rtol": 1e-4}
+ for tensor_name, tensor_torch, tensor_custom in [
+ ("q_grad", torch_q_grad, custom_q_grad),
+ ("k_grad", torch_k_grad, custom_k_grad),
+ ("v_grad", torch_v_grad, custom_v_grad),
+ ("sinks_grad", torch_sinks_grad, custom_sinks_grad),
+ ]:
+ if (tensor_torch is not None) and (tensor_custom is not None):
+ torch.testing.assert_close(
+ out_custom, out_torch, **tol, msg=lambda msg: f"Mismatch in {tensor_name}: {msg}"
+ )
diff --git a/tests/unit_tests/transformer/test_attention_variant_dsa.py b/tests/unit_tests/transformer/test_attention_variant_dsa.py
new file mode 100644
index 00000000000..bd106aa6f0e
--- /dev/null
+++ b/tests/unit_tests/transformer/test_attention_variant_dsa.py
@@ -0,0 +1,1271 @@
+# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved.
+
+from unittest.mock import patch
+
+import pytest
+import torch
+
+import megatron.core.parallel_state as parallel_state
+from megatron.core.models.gpt.gpt_layer_specs import get_gpt_layer_with_transformer_engine_spec
+from megatron.core.process_groups_config import ProcessGroupCollection
+from megatron.core.tensor_parallel.random import model_parallel_cuda_manual_seed
+from megatron.core.transformer import TransformerConfig
+from megatron.core.transformer.enums import AttnMaskType
+from megatron.core.transformer.experimental_attention_variant.dsa import (
+ DSAIndexer,
+ DSAIndexerLossAutoScaler,
+ DSAIndexerSubmodules,
+ DSAttention,
+ DSAttentionSubmodules,
+ compute_dsa_indexer_loss,
+ rotate_activation,
+)
+from megatron.core.transformer.transformer_config import MLATransformerConfig
+from tests.unit_tests.test_utilities import Utils
+
+try:
+ from fast_hadamard_transform import hadamard_transform as _hadamard_transform
+
+ HAVE_HADAMARD = True
+except ImportError:
+ HAVE_HADAMARD = False
+ _hadamard_transform = None
+
+
+def mock_hadamard_transform(x: torch.Tensor, scale: float = 1.0) -> torch.Tensor:
+ """Mock implementation of hadamard_transform for testing without the library installed.
+
+ This is a simple identity-like transformation that preserves shape and applies scaling.
+ """
+ return x * scale
+
+
+@pytest.fixture(autouse=True)
+def patch_hadamard_if_needed():
+ """Automatically patch hadamard_transform in dsa module if not installed."""
+ if not HAVE_HADAMARD:
+ with patch(
+ 'megatron.core.transformer.experimental_attention_variant.dsa.hadamard_transform',
+ mock_hadamard_transform,
+ ):
+ yield
+ else:
+ yield
+
+
+class TestRotateActivation:
+ """Test rotate_activation function."""
+
+ @pytest.fixture(scope='function', autouse=True)
+ def setup_method(self):
+ Utils.initialize_model_parallel(
+ tensor_model_parallel_size=1, pipeline_model_parallel_size=1
+ )
+ yield
+ Utils.destroy_model_parallel()
+
+ def test_rotate_activation_shape(self):
+ """Test that rotate_activation preserves shape."""
+ batch_size = 2
+ seq_len = 16
+ hidden_size = 128
+
+ x = torch.randn(seq_len, batch_size, hidden_size, dtype=torch.bfloat16).cuda()
+ output = rotate_activation(x)
+
+ assert output.shape == x.shape
+ assert output.dtype == torch.bfloat16
+
+ def test_rotate_activation_dtype_check(self):
+ """Test that rotate_activation only accepts bfloat16."""
+ x = torch.randn(16, 2, 128, dtype=torch.float32).cuda()
+
+ with pytest.raises(AssertionError, match="only support bf16"):
+ rotate_activation(x)
+
+
+@pytest.mark.parametrize("seqlen_and_topk", [[16, 32], [64, 32]])
+class TestComputeDSAIndexerLoss:
+ """Test compute_dsa_indexer_loss function."""
+
+ @pytest.fixture(scope='function', autouse=True)
+ def setup_method(self):
+ Utils.initialize_model_parallel(
+ tensor_model_parallel_size=1, pipeline_model_parallel_size=1
+ )
+ self.pg_collection = ProcessGroupCollection.use_mpu_process_groups(required_pgs=['tp'])
+ yield
+ Utils.destroy_model_parallel()
+
+ @pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA not available")
+ def test_dsa_indexer_loss_shape(self, seqlen_and_topk):
+ """Test that indexer loss returns a scalar."""
+ batch_size = 2
+ seqlen = seqlen_and_topk[0]
+ num_heads = 4
+ head_dim = 128
+ index_topk = seqlen_and_topk[1]
+
+ # Create dummy index scores
+ index_scores = torch.randn(batch_size, seqlen, seqlen, dtype=torch.float32).cuda()
+
+ # Apply causal mask to index_scores before computing topk
+ causal_mask = torch.triu(
+ torch.full(
+ (seqlen, seqlen), float('-inf'), dtype=torch.float32, device=index_scores.device
+ ),
+ diagonal=1,
+ )
+ # [batch_size, seqlen, seqlen] + [seqlen, seqlen] -> [batch_size, seqlen, seqlen]
+ masked_index_scores = index_scores + causal_mask
+
+ # Get topk indices from masked index_scores
+ topk_k = min(index_topk, seqlen)
+ topk_indices = masked_index_scores.topk(topk_k, dim=-1)[1]
+
+ query = torch.randn(seqlen, batch_size, num_heads, head_dim, dtype=torch.bfloat16).cuda()
+ key = torch.randn(seqlen, batch_size, num_heads, head_dim, dtype=torch.bfloat16).cuda()
+ softmax_scale = head_dim**-0.5
+
+ loss = compute_dsa_indexer_loss(
+ index_scores=index_scores,
+ topk_indices=topk_indices,
+ query=query,
+ key=key,
+ softmax_scale=softmax_scale,
+ loss_coeff=1.0,
+ sparse_loss=False,
+ pg_collection=self.pg_collection,
+ )
+
+ assert loss.shape == torch.Size([])
+ assert loss.dtype == torch.float32
+ assert loss >= 0 # KL divergence should be non-negative
+
+ @pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA not available")
+ def test_dsa_indexer_loss_sparse(self, seqlen_and_topk):
+ """Test sparse indexer loss computation."""
+ batch_size = 2
+ seqlen = seqlen_and_topk[0]
+ num_heads = 4
+ head_dim = 128
+ index_topk = seqlen_and_topk[1]
+
+ # Create dummy index scores
+ index_scores = torch.randn(batch_size, seqlen, seqlen, dtype=torch.float32).cuda()
+
+ # Apply causal mask to index_scores before computing topk
+ causal_mask = torch.triu(
+ torch.full(
+ (seqlen, seqlen), float('-inf'), dtype=torch.float32, device=index_scores.device
+ ),
+ diagonal=1,
+ )
+ # [batch_size, seqlen, seqlen] + [seqlen, seqlen] -> [batch_size, seqlen, seqlen]
+ masked_index_scores = index_scores + causal_mask
+
+ # Get topk indices from masked index_scores
+ topk_k = min(index_topk, seqlen)
+ topk_indices = masked_index_scores.topk(topk_k, dim=-1)[1]
+
+ query = torch.randn(seqlen, batch_size, num_heads, head_dim, dtype=torch.bfloat16).cuda()
+ key = torch.randn(seqlen, batch_size, num_heads, head_dim, dtype=torch.bfloat16).cuda()
+ softmax_scale = head_dim**-0.5
+
+ loss_sparse = compute_dsa_indexer_loss(
+ index_scores=index_scores,
+ topk_indices=topk_indices,
+ query=query,
+ key=key,
+ softmax_scale=softmax_scale,
+ loss_coeff=1.0,
+ sparse_loss=True,
+ pg_collection=self.pg_collection,
+ )
+
+ loss_dense = compute_dsa_indexer_loss(
+ index_scores=index_scores,
+ topk_indices=topk_indices,
+ query=query,
+ key=key,
+ softmax_scale=softmax_scale,
+ loss_coeff=1.0,
+ sparse_loss=False,
+ pg_collection=self.pg_collection,
+ )
+
+ # Sparse loss should be different from dense loss
+ if seqlen > index_topk:
+ assert loss_sparse != loss_dense
+ else:
+ assert loss_sparse == loss_dense
+ assert loss_sparse >= 0
+ assert loss_dense >= 0
+
+
+class TestDSAIndexerLossAutoScaler:
+ """Test DSAIndexerLossAutoScaler autograd function."""
+
+ @pytest.fixture(scope='function', autouse=True)
+ def setup_method(self):
+ Utils.initialize_model_parallel(
+ tensor_model_parallel_size=1, pipeline_model_parallel_size=1
+ )
+ yield
+ Utils.destroy_model_parallel()
+
+ @pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA not available")
+ def test_forward_pass(self):
+ """Test that forward pass preserves output."""
+ output = torch.randn(16, 2, 128).cuda()
+ output.requires_grad_(True)
+ indexer_loss = torch.tensor(0.5).cuda()
+ indexer_loss.requires_grad_(True)
+
+ result = DSAIndexerLossAutoScaler.apply(output, indexer_loss)
+
+ assert torch.allclose(result, output, atol=0, rtol=0)
+
+ @pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA not available")
+ def test_backward_pass(self):
+ """Test that backward pass triggers indexer loss backward and scales gradient correctly."""
+ output = torch.randn(16, 2, 128).cuda()
+ output.requires_grad_(True)
+
+ # Create indexer_loss with computation graph
+ # This simulates compute_dsa_indexer_loss which computes KL divergence
+ dummy_input = torch.randn(10).cuda()
+ dummy_input.requires_grad_(True)
+ indexer_loss = dummy_input.mean()
+
+ # Set loss scale
+ scale = torch.tensor(2.0).cuda()
+ DSAIndexerLossAutoScaler.set_loss_scale(scale)
+
+ # Apply the autograd function
+ result = DSAIndexerLossAutoScaler.apply(output, indexer_loss)
+
+ # Trigger backward
+ main_loss = result.sum()
+ main_loss.backward()
+
+ # Check that gradients flow back to output
+ assert output.grad is not None, "Gradient should flow back to parameters"
+
+ # Check that indexer_loss backward was triggered
+ assert dummy_input.grad is not None, "Indexer loss backward should be triggered"
+
+ # Verify the gradient is scaled correctly
+ expected_grad_per_element = scale.item() / len(dummy_input)
+ assert torch.allclose(
+ dummy_input.grad,
+ torch.full_like(dummy_input, expected_grad_per_element),
+ rtol=0,
+ atol=0,
+ ), f"Gradient should be scaled by loss scale, expected {expected_grad_per_element}, got {dummy_input.grad[0].item()}"
+
+
+@pytest.mark.parametrize("seqlen", [16, 64])
+class TestDSAIndexer:
+ """Test DSA Indexer module basic functionality with TP=1."""
+
+ @pytest.fixture(scope='function', autouse=True)
+ def setup_method(self):
+ Utils.initialize_model_parallel(
+ tensor_model_parallel_size=1, pipeline_model_parallel_size=1
+ )
+ torch.manual_seed(123)
+ model_parallel_cuda_manual_seed(123)
+
+ # Create MLA config with sparse attention parameters
+ self.index_topk = 32
+ self.config = MLATransformerConfig(
+ num_layers=2,
+ hidden_size=256,
+ num_attention_heads=16,
+ use_cpu_initialization=True,
+ bf16=True,
+ params_dtype=torch.bfloat16,
+ # MLA specific configs
+ q_lora_rank=64,
+ kv_lora_rank=64,
+ qk_head_dim=64,
+ qk_pos_emb_head_dim=32,
+ v_head_dim=64,
+ rope_type='rope',
+ rotary_base=10000,
+ rotary_percent=1.0,
+ # Sparse attention specific configs
+ dsa_indexer_n_heads=8,
+ dsa_indexer_head_dim=64,
+ dsa_indexer_topk=self.index_topk,
+ )
+
+ # Create indexer submodules spec
+ from megatron.core.extensions.transformer_engine import TELinear, TENorm
+ from megatron.core.transformer.spec_utils import ModuleSpec
+
+ indexer_submodules = DSAIndexerSubmodules(
+ linear_wq_b=ModuleSpec(module=TELinear),
+ linear_wk=ModuleSpec(module=TELinear),
+ k_norm=ModuleSpec(module=TENorm),
+ linear_weights_proj=ModuleSpec(module=TELinear),
+ )
+
+ self.pg_collection = ProcessGroupCollection.use_mpu_process_groups(
+ required_pgs=['tp', 'cp']
+ )
+ self.indexer = DSAIndexer(self.config, indexer_submodules, self.pg_collection)
+
+ yield
+ Utils.destroy_model_parallel()
+
+ def test_dsa_indexer_constructor(self, seqlen):
+ """Test indexer initialization."""
+ assert isinstance(self.indexer, DSAIndexer)
+ assert self.indexer.hidden_size == 256
+ assert self.indexer.index_n_heads == 8
+ assert self.indexer.index_head_dim == 64
+ assert self.indexer.index_topk == 32
+
+ @pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA not available")
+ def test_dsa_indexer_forward(self, seqlen):
+ """Test indexer forward pass."""
+ batch_size = 2
+
+ self.indexer.cuda()
+
+ # Create input tensors
+ x = torch.randn(seqlen, batch_size, self.config.hidden_size, dtype=torch.bfloat16).cuda()
+ qr = torch.randn(seqlen, batch_size, self.config.q_lora_rank, dtype=torch.bfloat16).cuda()
+
+ # Forward pass
+ topk_indices = self.indexer(x, qr)
+
+ # Check output shape
+ assert topk_indices.shape == (batch_size, seqlen, min(self.config.dsa_indexer_topk, seqlen))
+ assert topk_indices.dtype == torch.long
+ assert torch.all((topk_indices >= 0) & (topk_indices < seqlen))
+ # Make sure no duplicate indices are selected
+ assert torch.all(
+ torch.sort(topk_indices, dim=-1).values[:, :, 1:]
+ != torch.sort(topk_indices, dim=-1).values[:, :, :-1]
+ )
+
+ @pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA not available")
+ def test_dsa_indexer_forward_with_scores(self, seqlen):
+ """Test indexer forward pass with scores."""
+ batch_size = 2
+
+ self.indexer.cuda()
+
+ # Create input tensors
+ x = torch.randn(seqlen, batch_size, self.config.hidden_size, dtype=torch.bfloat16).cuda()
+ qr = torch.randn(seqlen, batch_size, self.config.q_lora_rank, dtype=torch.bfloat16).cuda()
+
+ # Forward pass with scores
+ index_scores, topk_indices = self.indexer.forward_with_scores(x, qr)
+
+ # Check output shapes
+ assert index_scores.shape == (batch_size, seqlen, seqlen)
+ assert topk_indices.shape == (batch_size, seqlen, min(self.config.dsa_indexer_topk, seqlen))
+ assert index_scores.dtype == torch.float32
+ assert topk_indices.dtype == torch.long
+ assert torch.all((topk_indices >= 0) & (topk_indices < seqlen))
+ # Make sure no duplicate indices are selected
+ assert torch.all(
+ torch.sort(topk_indices, dim=-1).values[:, :, 1:]
+ != torch.sort(topk_indices, dim=-1).values[:, :, :-1]
+ )
+
+ @pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA not available")
+ def test_dsa_indexer_with_mask(self, seqlen):
+ """Test indexer with attention mask."""
+ batch_size = 2
+
+ self.indexer.cuda()
+
+ # Create input tensors
+ x = torch.randn(seqlen, batch_size, self.config.hidden_size, dtype=torch.bfloat16).cuda()
+ qr = torch.randn(seqlen, batch_size, self.config.q_lora_rank, dtype=torch.bfloat16).cuda()
+ mask = torch.triu(
+ torch.full((batch_size, seqlen, seqlen), float('-inf'), dtype=torch.float32).cuda(),
+ diagonal=1,
+ )
+
+ # Forward pass with mask
+ index_scores, topk_indices = self.indexer.forward_with_scores(x, qr, mask=mask)
+
+ # Check that masked positions are not selected
+ # For causal mask, topk_indices[b, i, :] should all be <= i (except for the case that
+ # i < index_topk).
+ for b in range(batch_size):
+ for i in range(seqlen):
+ assert torch.all(topk_indices[b, i] <= max(self.index_topk, i))
+
+
+class TestDSAttention:
+ """Test DSAttention module basic functionality with TP=1."""
+
+ @pytest.fixture(scope='function', autouse=True)
+ def setup_method(self):
+ Utils.initialize_model_parallel(
+ tensor_model_parallel_size=1, pipeline_model_parallel_size=1
+ )
+ torch.manual_seed(123)
+ model_parallel_cuda_manual_seed(123)
+
+ # Create MLA config with sparse attention parameters
+ self.config = MLATransformerConfig(
+ num_layers=2,
+ hidden_size=256,
+ num_attention_heads=16,
+ use_cpu_initialization=True,
+ bf16=True,
+ params_dtype=torch.bfloat16,
+ # MLA specific configs
+ q_lora_rank=64,
+ kv_lora_rank=64,
+ qk_head_dim=64,
+ qk_pos_emb_head_dim=32,
+ v_head_dim=64,
+ rope_type='rope',
+ rotary_base=10000,
+ rotary_percent=1.0,
+ # Sparse attention specific configs
+ dsa_indexer_n_heads=8,
+ dsa_indexer_head_dim=64,
+ dsa_indexer_topk=32,
+ dsa_indexer_loss_coeff=1.0,
+ dsa_indexer_use_sparse_loss=False,
+ )
+
+ # Create sparse attention submodules spec
+ from megatron.core.extensions.transformer_engine import TELinear, TENorm
+ from megatron.core.transformer.spec_utils import ModuleSpec
+
+ indexer_submodules = DSAIndexerSubmodules(
+ linear_wq_b=ModuleSpec(module=TELinear),
+ linear_wk=ModuleSpec(module=TELinear),
+ k_norm=ModuleSpec(module=TENorm),
+ linear_weights_proj=ModuleSpec(module=TELinear),
+ )
+ indexer_spec = ModuleSpec(module=DSAIndexer, submodules=indexer_submodules)
+ sparse_attention_submodules = DSAttentionSubmodules(indexer=indexer_spec)
+
+ self.pg_collection = ProcessGroupCollection.use_mpu_process_groups(
+ required_pgs=['tp', 'cp']
+ )
+
+ self.sparse_attention = DSAttention(
+ config=self.config,
+ submodules=sparse_attention_submodules,
+ layer_number=1,
+ attn_mask_type=AttnMaskType.causal,
+ attention_type='self',
+ pg_collection=self.pg_collection,
+ )
+
+ yield
+ Utils.destroy_model_parallel()
+
+ def test_dsa_constructor(self):
+ """Test sparse attention initialization."""
+ assert isinstance(self.sparse_attention, DSAttention)
+ assert hasattr(self.sparse_attention, 'indexer')
+ assert isinstance(self.sparse_attention.indexer, DSAIndexer)
+
+ @pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA not available")
+ def test_dsa_forward(self):
+ """Test sparse attention forward pass."""
+ seq_len = 16
+ batch_size = 2
+ num_heads = self.config.num_attention_heads
+ head_dim = self.config.hidden_size // num_heads
+
+ self.sparse_attention.cuda()
+
+ # Create input tensors [seq_len, batch, num_heads, head_dim]
+ query = (
+ torch.randn(seq_len, batch_size, num_heads, head_dim, dtype=torch.bfloat16)
+ .cuda()
+ .requires_grad_(True)
+ )
+ key = (
+ torch.randn(seq_len, batch_size, num_heads, head_dim, dtype=torch.bfloat16)
+ .cuda()
+ .requires_grad_(True)
+ )
+ value = (
+ torch.randn(seq_len, batch_size, num_heads, head_dim, dtype=torch.bfloat16)
+ .cuda()
+ .requires_grad_(True)
+ )
+
+ # Original hidden states and low-rank query
+ x = torch.randn(seq_len, batch_size, self.config.hidden_size, dtype=torch.bfloat16).cuda()
+ qr = torch.randn(seq_len, batch_size, self.config.q_lora_rank, dtype=torch.bfloat16).cuda()
+
+ # Create causal attention mask
+ attention_mask = torch.ones(batch_size, 1, seq_len, seq_len, dtype=torch.bool).cuda()
+ attention_mask = torch.tril(attention_mask)
+
+ # Forward pass
+ output = self.sparse_attention(
+ query=query,
+ key=key,
+ value=value,
+ x=x,
+ qr=qr,
+ attention_mask=attention_mask,
+ attn_mask_type=AttnMaskType.causal,
+ )
+
+ # Check output shape
+ assert output.shape == (seq_len, batch_size, self.config.hidden_size)
+ assert output.dtype == torch.bfloat16
+
+ @pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA not available")
+ def test_dsa_backward(self):
+ """Test sparse attention backward pass with indexer loss."""
+ seq_len = 16
+ batch_size = 2
+ num_heads = self.config.num_attention_heads
+ head_dim = self.config.hidden_size // num_heads
+
+ self.sparse_attention.train()
+ self.sparse_attention.cuda()
+
+ # Create input tensors
+ query = (
+ torch.randn(seq_len, batch_size, num_heads, head_dim, dtype=torch.bfloat16)
+ .cuda()
+ .requires_grad_(True)
+ )
+ key = (
+ torch.randn(seq_len, batch_size, num_heads, head_dim, dtype=torch.bfloat16)
+ .cuda()
+ .requires_grad_(True)
+ )
+ value = (
+ torch.randn(seq_len, batch_size, num_heads, head_dim, dtype=torch.bfloat16)
+ .cuda()
+ .requires_grad_(True)
+ )
+
+ # Original hidden states and low-rank query
+ x = torch.randn(seq_len, batch_size, self.config.hidden_size, dtype=torch.bfloat16).cuda()
+ qr = torch.randn(seq_len, batch_size, self.config.q_lora_rank, dtype=torch.bfloat16).cuda()
+
+ # Create causal attention mask
+ attention_mask = torch.ones(batch_size, 1, seq_len, seq_len, dtype=torch.bool).cuda()
+ attention_mask = torch.tril(attention_mask)
+
+ # Forward pass
+ output = self.sparse_attention(
+ query=query,
+ key=key,
+ value=value,
+ x=x,
+ qr=qr,
+ attention_mask=attention_mask,
+ attn_mask_type=AttnMaskType.causal,
+ )
+
+ # Backward pass
+ loss = output.sum()
+ loss.backward()
+
+ # Check that gradients are computed for inputs
+ assert query.grad is not None
+ assert key.grad is not None
+ assert value.grad is not None
+
+ # Check that indexer parameters have gradients
+ for name, param in self.sparse_attention.indexer.named_parameters():
+ if param.requires_grad:
+ assert param.grad is not None, f"Indexer parameter {name} has no gradient"
+
+ @pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA not available")
+ def test_dsa_topk_selection(self):
+ """Test that sparse attention correctly selects top-k indices."""
+ seq_len = 16
+ batch_size = 2
+ num_heads = self.config.num_attention_heads
+ head_dim = self.config.hidden_size // num_heads
+
+ self.sparse_attention.eval()
+ self.sparse_attention.cuda()
+
+ # Create input tensors
+ query = torch.randn(seq_len, batch_size, num_heads, head_dim, dtype=torch.bfloat16).cuda()
+ key = torch.randn(seq_len, batch_size, num_heads, head_dim, dtype=torch.bfloat16).cuda()
+ value = torch.randn(seq_len, batch_size, num_heads, head_dim, dtype=torch.bfloat16).cuda()
+
+ # Original hidden states and low-rank query
+ x = torch.randn(seq_len, batch_size, self.config.hidden_size, dtype=torch.bfloat16).cuda()
+ qr = torch.randn(seq_len, batch_size, self.config.q_lora_rank, dtype=torch.bfloat16).cuda()
+
+ # Create causal attention mask
+ attention_mask = torch.ones(batch_size, 1, seq_len, seq_len, dtype=torch.bool).cuda()
+ attention_mask = torch.tril(attention_mask)
+
+ with torch.no_grad():
+ # Get topk indices from indexer
+ _, topk_indices = self.sparse_attention.indexer.forward_with_scores(x, qr)
+
+ # Forward pass
+ output = self.sparse_attention(
+ query=query,
+ key=key,
+ value=value,
+ x=x,
+ qr=qr,
+ attention_mask=attention_mask,
+ attn_mask_type=AttnMaskType.causal,
+ )
+
+ # Check that topk_indices are valid
+ assert torch.all(topk_indices >= 0)
+ assert torch.all(topk_indices < seq_len)
+ assert topk_indices.shape[2] == min(self.config.dsa_indexer_topk, seq_len)
+
+
+# ======================================================================================
+# Tensor Parallel Consistency Tests
+# ======================================================================================
+
+
+@pytest.mark.parametrize("tensor_model_parallel_size", [2, 4, 8])
+@pytest.mark.parametrize("sequence_parallel", [False, True])
+class TestIndexerTensorParallel:
+ """Test DSA Indexer with different TP sizes and SP settings, compare with TP=1 baseline."""
+
+ def _create_config(self, sequence_parallel=False):
+ """Helper to create MLA config."""
+ # Get TP size from parallel_state
+ tensor_model_parallel_size = parallel_state.get_tensor_model_parallel_world_size()
+
+ return MLATransformerConfig(
+ num_layers=2,
+ hidden_size=256,
+ num_attention_heads=16,
+ use_cpu_initialization=True,
+ bf16=True,
+ params_dtype=torch.bfloat16,
+ tensor_model_parallel_size=tensor_model_parallel_size,
+ sequence_parallel=sequence_parallel,
+ # MLA specific configs
+ q_lora_rank=64,
+ kv_lora_rank=64,
+ qk_head_dim=64,
+ qk_pos_emb_head_dim=32,
+ v_head_dim=64,
+ rope_type='rope',
+ rotary_base=10000,
+ rotary_percent=1.0,
+ # Sparse attention specific configs
+ dsa_indexer_n_heads=8,
+ dsa_indexer_head_dim=64,
+ dsa_indexer_topk=32,
+ )
+
+ def _create_indexer(self, config, pg_collection):
+ """Helper to create indexer."""
+ from megatron.core.extensions.transformer_engine import TELinear, TENorm
+ from megatron.core.transformer.spec_utils import ModuleSpec
+
+ indexer_submodules = DSAIndexerSubmodules(
+ linear_wq_b=ModuleSpec(module=TELinear),
+ linear_wk=ModuleSpec(module=TELinear),
+ k_norm=ModuleSpec(module=TENorm),
+ linear_weights_proj=ModuleSpec(module=TELinear),
+ )
+
+ return DSAIndexer(config, indexer_submodules, pg_collection)
+
+ @pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA not available")
+ def test_dsa_indexer_weight_consistency(self, tensor_model_parallel_size, sequence_parallel):
+ """Test that indexer weights are identical across ALL GPUs."""
+ Utils.initialize_model_parallel(
+ tensor_model_parallel_size=tensor_model_parallel_size, pipeline_model_parallel_size=1
+ )
+ torch.manual_seed(123)
+ model_parallel_cuda_manual_seed(123)
+
+ config = self._create_config(sequence_parallel=sequence_parallel)
+ pg_collection = ProcessGroupCollection.use_mpu_process_groups(required_pgs=['tp', 'cp'])
+ indexer = self._create_indexer(config, pg_collection).cuda()
+
+ # Check that all weights are identical across ALL ranks (not just TP group)
+ world_size = torch.distributed.get_world_size()
+ world_rank = torch.distributed.get_rank()
+
+ if world_size > 1:
+ for name, param in indexer.named_parameters():
+ # Gather weights from ALL ranks in WORLD group
+ param_list = [torch.zeros_like(param.data) for _ in range(world_size)]
+ torch.distributed.all_gather(param_list, param.data)
+
+ # All weights should be identical across all GPUs
+ for i in range(1, world_size):
+ assert torch.allclose(
+ param_list[0], param_list[i], rtol=0, atol=0
+ ), f"Parameter {name} differs between rank 0 and rank {i} (world)"
+
+ Utils.destroy_model_parallel()
+
+ @pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA not available")
+ def test_dsa_indexer_forward_consistency(self, tensor_model_parallel_size, sequence_parallel):
+ """Test that indexer gives consistent results across different TP sizes and SP settings."""
+ # First run with TP=1 to get baseline
+ Utils.initialize_model_parallel(
+ tensor_model_parallel_size=1, pipeline_model_parallel_size=1
+ )
+ torch.manual_seed(123)
+ model_parallel_cuda_manual_seed(123)
+
+ config_tp1 = self._create_config(sequence_parallel=False) # TP=1 doesn't use SP
+ pg_collection_tp1 = ProcessGroupCollection.use_mpu_process_groups(required_pgs=['tp', 'cp'])
+ indexer_tp1 = self._create_indexer(config_tp1, pg_collection_tp1).cuda()
+
+ seq_len = 64
+ batch_size = 2
+
+ # Create one common input (all ranks create same input with same seed)
+ x_input = torch.randn(
+ seq_len, batch_size, config_tp1.hidden_size, dtype=torch.bfloat16
+ ).cuda()
+ qr_input = torch.randn(
+ seq_len, batch_size, config_tp1.q_lora_rank, dtype=torch.bfloat16
+ ).cuda()
+
+ # Forward pass with gradients enabled
+ index_scores_tp1, topk_indices_tp1 = indexer_tp1.forward_with_scores(x_input, qr_input)
+
+ # Backward pass
+ loss_tp1 = index_scores_tp1.sum()
+ loss_tp1.backward()
+
+ # Save gradients from TP=1
+ indexer_tp1_grads = {
+ name: param.grad.clone().cpu()
+ for name, param in indexer_tp1.named_parameters()
+ if param.grad is not None
+ }
+
+ Utils.destroy_model_parallel()
+
+ # Now run with target TP size
+ Utils.initialize_model_parallel(
+ tensor_model_parallel_size=tensor_model_parallel_size, pipeline_model_parallel_size=1
+ )
+ torch.manual_seed(123)
+ model_parallel_cuda_manual_seed(123)
+
+ config_tpn = self._create_config(sequence_parallel=sequence_parallel)
+ pg_collection_tpn = ProcessGroupCollection.use_mpu_process_groups(required_pgs=['tp', 'cp'])
+ indexer_tpn = self._create_indexer(config_tpn, pg_collection_tpn).cuda()
+
+ # Prepare input: split along seqlen if SP is enabled
+ if sequence_parallel:
+ tp_rank = parallel_state.get_tensor_model_parallel_rank()
+ seq_per_rank = seq_len // tensor_model_parallel_size
+ start_idx = tp_rank * seq_per_rank
+ end_idx = (tp_rank + 1) * seq_per_rank
+ x_tpn = x_input[start_idx:end_idx]
+ qr_tpn = qr_input[start_idx:end_idx]
+ else:
+ # No SP: all TP ranks see full input
+ x_tpn = x_input
+ qr_tpn = qr_input
+
+ # Forward pass with gradients enabled
+ index_scores_tpn, topk_indices_tpn = indexer_tpn.forward_with_scores(x_tpn, qr_tpn)
+
+ # Backward pass
+ loss_tpn = index_scores_tpn.sum()
+ loss_tpn.backward()
+
+ # Compare forward outputs
+ assert index_scores_tpn.shape == index_scores_tp1.shape
+ assert topk_indices_tpn.shape == topk_indices_tp1.shape
+
+ # Check that index scores are close (allow for floating point accumulation errors)
+ assert torch.allclose(
+ index_scores_tpn, index_scores_tp1, rtol=0, atol=0
+ ), f"Index scores mismatch between TP=1 and TP={tensor_model_parallel_size}, SP={sequence_parallel}"
+
+ # Check that topk indices are exactly the same
+ assert torch.equal(
+ topk_indices_tpn, topk_indices_tp1
+ ), f"Top-k indices mismatch between TP=1 and TP={tensor_model_parallel_size}, SP={sequence_parallel}"
+
+ # Compare gradients - indexer grads should be identical (duplicated weights)
+ for name, param in indexer_tpn.named_parameters():
+ if param.grad is not None and name in indexer_tp1_grads:
+ assert torch.allclose(
+ param.grad.cpu(), indexer_tp1_grads[name], rtol=0, atol=0
+ ), f"Indexer gradient {name} mismatch between TP=1 and TP={tensor_model_parallel_size}, SP={sequence_parallel}"
+
+ Utils.destroy_model_parallel()
+
+ @pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA not available")
+ def test_dsa_indexer_gradient_sync(self, tensor_model_parallel_size, sequence_parallel):
+ """Test that gradients are properly synchronized within TP group."""
+ Utils.initialize_model_parallel(
+ tensor_model_parallel_size=tensor_model_parallel_size, pipeline_model_parallel_size=1
+ )
+ torch.manual_seed(123)
+ model_parallel_cuda_manual_seed(123)
+
+ config = self._create_config(sequence_parallel=sequence_parallel)
+ pg_collection = ProcessGroupCollection.use_mpu_process_groups(required_pgs=['tp', 'cp'])
+ indexer = self._create_indexer(config, pg_collection).cuda()
+
+ seq_len = 64
+ batch_size = 2
+
+ # Create one common input (all ranks create same input with same seed)
+ x_input = torch.randn(seq_len, batch_size, config.hidden_size, dtype=torch.bfloat16).cuda()
+ qr_input = torch.randn(seq_len, batch_size, config.q_lora_rank, dtype=torch.bfloat16).cuda()
+
+ # Prepare input: split along seqlen if SP is enabled
+ if sequence_parallel:
+ tp_rank = parallel_state.get_tensor_model_parallel_rank()
+ tp_size = parallel_state.get_tensor_model_parallel_world_size()
+ seq_per_rank = seq_len // tp_size
+ start_idx = tp_rank * seq_per_rank
+ end_idx = (tp_rank + 1) * seq_per_rank
+ x = x_input[start_idx:end_idx]
+ qr = qr_input[start_idx:end_idx]
+ else:
+ # No SP: all TP ranks see full input
+ x = x_input
+ qr = qr_input
+
+ # Forward and backward
+ index_scores, topk_indices = indexer.forward_with_scores(x, qr)
+ loss = index_scores.sum()
+ loss.backward()
+
+ # Check that all parameters have gradients
+ for name, param in indexer.named_parameters():
+ if param.requires_grad:
+ assert param.grad is not None, f"Parameter {name} has no gradient"
+
+ # After TP sync, check that gradients are identical within TP group
+ # Note: We only check TP group because DDP sync happens separately
+ tp_size = parallel_state.get_tensor_model_parallel_world_size()
+ if tp_size > 1:
+ for name, param in indexer.named_parameters():
+ if param.requires_grad and param.grad is not None:
+ # Gather gradients from all ranks in TP group only
+ grad_list = [torch.zeros_like(param.grad) for _ in range(tp_size)]
+ torch.distributed.all_gather(grad_list, param.grad, group=pg_collection.tp)
+
+ # All gradients should be identical within TP group after sync
+ for i in range(1, tp_size):
+ assert torch.allclose(
+ grad_list[0], grad_list[i], rtol=0, atol=0
+ ), f"Gradient for {name} differs between TP rank 0 and rank {i} after TP sync"
+
+ Utils.destroy_model_parallel()
+
+
+@pytest.mark.parametrize("tensor_model_parallel_size", [2, 4])
+@pytest.mark.parametrize("sequence_parallel", [False, True])
+@pytest.mark.parametrize("use_sparse_indexer_loss", [False, True])
+class TestDSAttentionTensorParallel:
+ """Test DSAttention with different TP sizes, SP settings, and sparse indexer loss."""
+
+ def _create_config(self, sequence_parallel=False, use_sparse_indexer_loss=False):
+ """Helper to create MLA config."""
+ # Get TP size from parallel_state
+ tensor_model_parallel_size = parallel_state.get_tensor_model_parallel_world_size()
+
+ return MLATransformerConfig(
+ num_layers=2,
+ hidden_size=256,
+ num_attention_heads=16,
+ use_cpu_initialization=True,
+ bf16=True,
+ params_dtype=torch.bfloat16,
+ tensor_model_parallel_size=tensor_model_parallel_size,
+ sequence_parallel=sequence_parallel,
+ # MLA specific configs
+ q_lora_rank=64,
+ kv_lora_rank=64,
+ qk_head_dim=64,
+ qk_pos_emb_head_dim=32,
+ v_head_dim=64,
+ rope_type='rope',
+ rotary_base=10000,
+ rotary_percent=1.0,
+ # Sparse attention specific configs
+ dsa_indexer_n_heads=8,
+ dsa_indexer_head_dim=64,
+ dsa_indexer_topk=32,
+ dsa_indexer_loss_coeff=1.0,
+ dsa_indexer_use_sparse_loss=use_sparse_indexer_loss,
+ )
+
+ def _create_sparse_attention(self, config, pg_collection):
+ """Helper to create sparse attention."""
+ from megatron.core.extensions.transformer_engine import TELinear, TENorm
+ from megatron.core.transformer.spec_utils import ModuleSpec
+
+ indexer_submodules = DSAIndexerSubmodules(
+ linear_wq_b=ModuleSpec(module=TELinear),
+ linear_wk=ModuleSpec(module=TELinear),
+ k_norm=ModuleSpec(module=TENorm),
+ linear_weights_proj=ModuleSpec(module=TELinear),
+ )
+ indexer_spec = ModuleSpec(module=DSAIndexer, submodules=indexer_submodules)
+ sparse_attention_submodules = DSAttentionSubmodules(indexer=indexer_spec)
+
+ return DSAttention(
+ config=config,
+ submodules=sparse_attention_submodules,
+ layer_number=1,
+ attn_mask_type=AttnMaskType.causal,
+ attention_type='self',
+ pg_collection=pg_collection,
+ )
+
+ @pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA not available")
+ def test_dsa_weight_consistency(
+ self, tensor_model_parallel_size, sequence_parallel, use_sparse_indexer_loss
+ ):
+ """Test that sparse attention indexer weights are identical across ALL GPUs."""
+ Utils.initialize_model_parallel(
+ tensor_model_parallel_size=tensor_model_parallel_size, pipeline_model_parallel_size=1
+ )
+ torch.manual_seed(123)
+ model_parallel_cuda_manual_seed(123)
+
+ config = self._create_config(
+ sequence_parallel=sequence_parallel, use_sparse_indexer_loss=use_sparse_indexer_loss
+ )
+ pg_collection = ProcessGroupCollection.use_mpu_process_groups(required_pgs=['tp', 'cp'])
+ sparse_attention = self._create_sparse_attention(config, pg_collection).cuda()
+
+ # Check that all indexer weights are identical across ALL ranks
+ world_size = torch.distributed.get_world_size()
+ world_rank = torch.distributed.get_rank()
+
+ if world_size > 1:
+ for name, param in sparse_attention.indexer.named_parameters():
+ # Gather weights from ALL ranks in WORLD group
+ param_list = [torch.zeros_like(param.data) for _ in range(world_size)]
+ torch.distributed.all_gather(param_list, param.data)
+
+ # All weights should be identical across all GPUs
+ for i in range(1, world_size):
+ torch.testing.assert_close(param_list[0], param_list[i], rtol=0, atol=0)
+
+ Utils.destroy_model_parallel()
+
+ @pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA not available")
+ def test_dsa_forward_consistency(
+ self, tensor_model_parallel_size, sequence_parallel, use_sparse_indexer_loss
+ ):
+ """Test that sparse attention gives consistent results across different TP, SP, and sparse loss settings."""
+ # First run with TP=1 to get baseline
+ Utils.initialize_model_parallel(
+ tensor_model_parallel_size=1, pipeline_model_parallel_size=1
+ )
+ torch.manual_seed(123)
+ model_parallel_cuda_manual_seed(123)
+
+ config_tp1 = self._create_config(
+ sequence_parallel=False, use_sparse_indexer_loss=use_sparse_indexer_loss
+ ) # TP=1 doesn't use SP
+ pg_collection_tp1 = ProcessGroupCollection.use_mpu_process_groups(required_pgs=['tp', 'cp'])
+ sparse_attention_tp1 = self._create_sparse_attention(config_tp1, pg_collection_tp1).cuda()
+
+ seq_len = 64
+ batch_size = 2
+ num_heads = config_tp1.num_attention_heads
+ head_dim = config_tp1.hidden_size // num_heads
+
+ # Create one common input (all ranks create same input with same seed)
+ query_input = (
+ torch.randn(seq_len, batch_size, num_heads, head_dim, dtype=torch.bfloat16)
+ .cuda()
+ .requires_grad_(True)
+ )
+ key_input = (
+ torch.randn(seq_len, batch_size, num_heads, head_dim, dtype=torch.bfloat16)
+ .cuda()
+ .requires_grad_(True)
+ )
+ value_input = (
+ torch.randn(seq_len, batch_size, num_heads, head_dim, dtype=torch.bfloat16)
+ .cuda()
+ .requires_grad_(True)
+ )
+ x_input = torch.randn(
+ seq_len, batch_size, config_tp1.hidden_size, dtype=torch.bfloat16
+ ).cuda()
+ qr_input = torch.randn(
+ seq_len, batch_size, config_tp1.q_lora_rank, dtype=torch.bfloat16
+ ).cuda()
+ attention_mask = torch.ones(batch_size, 1, seq_len, seq_len, dtype=torch.bool).cuda()
+ attention_mask = torch.tril(attention_mask)
+
+ # Forward pass with gradients enabled
+ sparse_attention_tp1.train()
+ output_tp1 = sparse_attention_tp1(
+ query=query_input,
+ key=key_input,
+ value=value_input,
+ x=x_input,
+ qr=qr_input,
+ attention_mask=attention_mask,
+ attn_mask_type=AttnMaskType.causal,
+ )
+
+ # Backward pass
+ loss_tp1 = output_tp1.sum()
+ loss_tp1.backward()
+
+ # Save gradients from TP=1
+ indexer_tp1_grads = {
+ name: param.grad.clone()
+ for name, param in sparse_attention_tp1.indexer.named_parameters()
+ if param.grad is not None
+ }
+ query_tp1_grad = query_input.grad.clone().cpu()
+ key_tp1_grad = key_input.grad.clone().cpu()
+ value_tp1_grad = value_input.grad.clone().cpu()
+
+ Utils.destroy_model_parallel()
+
+ # Now run with target TP size
+ Utils.initialize_model_parallel(
+ tensor_model_parallel_size=tensor_model_parallel_size, pipeline_model_parallel_size=1
+ )
+ torch.manual_seed(123)
+ model_parallel_cuda_manual_seed(123)
+
+ config_tpn = self._create_config(
+ sequence_parallel=sequence_parallel, use_sparse_indexer_loss=use_sparse_indexer_loss
+ )
+ pg_collection_tpn = ProcessGroupCollection.use_mpu_process_groups(required_pgs=['tp', 'cp'])
+ sparse_attention_tpn = self._create_sparse_attention(config_tpn, pg_collection_tpn).cuda()
+
+ # Create one common input (all ranks create same input with same seed)
+ query_input = torch.randn(
+ seq_len, batch_size, num_heads, head_dim, dtype=torch.bfloat16
+ ).cuda()
+ key_input = torch.randn(
+ seq_len, batch_size, num_heads, head_dim, dtype=torch.bfloat16
+ ).cuda()
+ value_input = torch.randn(
+ seq_len, batch_size, num_heads, head_dim, dtype=torch.bfloat16
+ ).cuda()
+ x_input = torch.randn(
+ seq_len, batch_size, config_tp1.hidden_size, dtype=torch.bfloat16
+ ).cuda()
+ qr_input = torch.randn(
+ seq_len, batch_size, config_tp1.q_lora_rank, dtype=torch.bfloat16
+ ).cuda()
+ attention_mask = torch.ones(batch_size, 1, seq_len, seq_len, dtype=torch.bool).cuda()
+ attention_mask = torch.tril(attention_mask)
+
+ # Prepare input: split along seqlen if SP is enabled
+ tp_rank = parallel_state.get_tensor_model_parallel_rank()
+ if sequence_parallel:
+ seq_per_rank = seq_len // tensor_model_parallel_size
+ start_idx = tp_rank * seq_per_rank
+ end_idx = (tp_rank + 1) * seq_per_rank
+ x_tpn = x_input[start_idx:end_idx]
+ qr_tpn = qr_input[start_idx:end_idx]
+ else:
+ x_tpn = x_input
+ qr_tpn = qr_input
+
+ query_input = query_input.detach()
+ key_input = key_input.detach()
+ value_input = value_input.detach()
+ head_per_rank = num_heads // tensor_model_parallel_size
+ start_head = tp_rank * head_per_rank
+ end_head = (tp_rank + 1) * head_per_rank
+ query_tpn = query_input[:, :, start_head:end_head, :].clone().requires_grad_(True)
+ key_tpn = key_input[:, :, start_head:end_head, :].clone().requires_grad_(True)
+ value_tpn = value_input[:, :, start_head:end_head, :].clone().requires_grad_(True)
+ attention_mask_tpn = attention_mask
+
+ # Forward pass with gradients enabled
+ sparse_attention_tpn.train()
+ output_tpn = sparse_attention_tpn(
+ query=query_tpn,
+ key=key_tpn,
+ value=value_tpn,
+ x=x_tpn,
+ qr=qr_tpn,
+ attention_mask=attention_mask_tpn,
+ attn_mask_type=AttnMaskType.causal,
+ )
+
+ # Backward pass
+ loss_tpn = output_tpn.sum()
+ loss_tpn.backward()
+
+ from megatron.core.tensor_parallel.mappings import gather_from_tensor_model_parallel_region
+
+ output_tpn_gathered = gather_from_tensor_model_parallel_region(
+ output_tpn, group=pg_collection_tpn.tp
+ )
+ assert output_tpn_gathered.shape == output_tp1.shape
+ assert torch.allclose(
+ output_tpn_gathered.detach(), output_tp1.detach(), rtol=0, atol=0
+ ), f"Sparse attention outputs mismatch between TP=1 and TP={tensor_model_parallel_size}, SP={sequence_parallel}, sparse_loss={use_sparse_indexer_loss}"
+
+ # 1. Check indexer gradients.
+ for name, param in sparse_attention_tpn.indexer.named_parameters():
+ if param.grad is not None and name in indexer_tp1_grads:
+ torch.testing.assert_close(
+ param.grad, indexer_tp1_grads[name], rtol=1e-5, atol=1e-5
+ )
+
+ # 2. Query/Key/Value gradients need to be gathered along num_heads dim (dim 2) if SP is enabled
+ # Flatten last two dims: [seq_len, batch, num_heads, head_dim] -> [seq_len, batch, num_heads * head_dim]
+ sq, b, nh, hd = query_tpn.grad.shape
+ query_grad_flat = query_tpn.grad.reshape(sq, b, nh * hd)
+ key_grad_flat = key_tpn.grad.reshape(sq, b, nh * hd)
+ value_grad_flat = value_tpn.grad.reshape(sq, b, nh * hd)
+
+ # Gather along last dim
+ query_grad_gathered_flat = gather_from_tensor_model_parallel_region(
+ query_grad_flat, group=pg_collection_tpn.tp
+ )
+ key_grad_gathered_flat = gather_from_tensor_model_parallel_region(
+ key_grad_flat, group=pg_collection_tpn.tp
+ )
+ value_grad_gathered_flat = gather_from_tensor_model_parallel_region(
+ value_grad_flat, group=pg_collection_tpn.tp
+ )
+
+ # Reshape back: [seq_len, batch, num_heads * head_dim] -> [seq_len, batch, num_heads, head_dim]
+ query_tpn_grad_gathered = query_grad_gathered_flat.reshape(sq, b, num_heads, hd)
+ key_tpn_grad_gathered = key_grad_gathered_flat.reshape(sq, b, num_heads, hd)
+ value_tpn_grad_gathered = value_grad_gathered_flat.reshape(sq, b, num_heads, hd)
+
+ assert torch.allclose(
+ query_tpn_grad_gathered.cpu(), query_tp1_grad, rtol=0, atol=0
+ ), f"Query gradient mismatch between TP=1 and TP={tensor_model_parallel_size}"
+ assert torch.allclose(
+ key_tpn_grad_gathered.cpu(), key_tp1_grad, rtol=0, atol=0
+ ), f"Key gradient mismatch between TP=1 and TP={tensor_model_parallel_size}"
+ assert torch.allclose(
+ value_tpn_grad_gathered.cpu(), value_tp1_grad, rtol=0, atol=0
+ ), f"Value gradient mismatch between TP=1 and TP={tensor_model_parallel_size}"
+
+ Utils.destroy_model_parallel()
+
+ @pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA not available")
+ def test_dsa_gradient_sync(
+ self, tensor_model_parallel_size, sequence_parallel, use_sparse_indexer_loss
+ ):
+ """Test that indexer gradients are properly synchronized within TP group."""
+ Utils.initialize_model_parallel(
+ tensor_model_parallel_size=tensor_model_parallel_size, pipeline_model_parallel_size=1
+ )
+ torch.manual_seed(123)
+ model_parallel_cuda_manual_seed(123)
+
+ config = self._create_config(
+ sequence_parallel=sequence_parallel, use_sparse_indexer_loss=use_sparse_indexer_loss
+ )
+ pg_collection = ProcessGroupCollection.use_mpu_process_groups(required_pgs=['tp', 'cp'])
+ sparse_attention = self._create_sparse_attention(config, pg_collection).cuda()
+ sparse_attention.train()
+
+ seq_len = 64
+ batch_size = 2
+ num_heads = config.num_attention_heads
+ head_dim = config.hidden_size // num_heads
+
+ # Create one common input (all ranks create same input with same seed)
+ query_input = torch.randn(
+ seq_len, batch_size, num_heads, head_dim, dtype=torch.bfloat16
+ ).cuda()
+ key_input = torch.randn(
+ seq_len, batch_size, num_heads, head_dim, dtype=torch.bfloat16
+ ).cuda()
+ value_input = torch.randn(
+ seq_len, batch_size, num_heads, head_dim, dtype=torch.bfloat16
+ ).cuda()
+ x_input = torch.randn(seq_len, batch_size, config.hidden_size, dtype=torch.bfloat16).cuda()
+ qr_input = torch.randn(seq_len, batch_size, config.q_lora_rank, dtype=torch.bfloat16).cuda()
+
+ # Prepare input: split along seqlen if SP is enabled
+ tp_rank = parallel_state.get_tensor_model_parallel_rank()
+ if sequence_parallel:
+ tp_size = parallel_state.get_tensor_model_parallel_world_size()
+ seq_per_rank = seq_len // tp_size
+ start_idx = tp_rank * seq_per_rank
+ end_idx = (tp_rank + 1) * seq_per_rank
+ x = x_input[start_idx:end_idx]
+ qr = qr_input[start_idx:end_idx]
+ else:
+ x = x_input
+ qr = qr_input
+
+ # query, key, value should be split along num_heads dim
+ head_per_rank = num_heads // tensor_model_parallel_size
+ start_head = tp_rank * head_per_rank
+ end_head = (tp_rank + 1) * head_per_rank
+ query = query_input[:, :, start_head:end_head, :]
+ key = key_input[:, :, start_head:end_head, :]
+ value = value_input[:, :, start_head:end_head, :]
+
+ attention_mask = torch.ones(batch_size, 1, seq_len, seq_len, dtype=torch.bool).cuda()
+ attention_mask = torch.tril(attention_mask)
+
+ query.requires_grad_(True)
+ key.requires_grad_(True)
+ value.requires_grad_(True)
+
+ # Forward and backward
+ output = sparse_attention(
+ query=query,
+ key=key,
+ value=value,
+ x=x,
+ qr=qr,
+ attention_mask=attention_mask,
+ attn_mask_type=AttnMaskType.causal,
+ )
+
+ loss = output.sum()
+ loss.backward()
+
+ # Check that gradients exist before sync
+ assert query.grad is not None
+ assert key.grad is not None
+ assert value.grad is not None
+
+ # Check that indexer parameters have gradients
+ for name, param in sparse_attention.indexer.named_parameters():
+ if param.requires_grad:
+ assert param.grad is not None, f"Indexer parameter {name} has no gradient"
+
+ # Check that indexer gradients are identical within TP group
+ tp_size = parallel_state.get_tensor_model_parallel_world_size()
+ if tp_size > 1:
+ for name, param in sparse_attention.indexer.named_parameters():
+ if param.requires_grad and param.grad is not None:
+ # Gather gradients from all ranks in TP group only
+ grad_list = [torch.zeros_like(param.grad) for _ in range(tp_size)]
+ torch.distributed.all_gather(grad_list, param.grad, group=pg_collection.tp)
+
+ # All gradients should be identical within TP group after sync
+ for i in range(1, tp_size):
+ assert torch.allclose(
+ grad_list[0], grad_list[i], rtol=0, atol=0
+ ), f"Indexer gradient for {name} differs between TP rank 0 and rank {i} after TP sync"
+
+ Utils.destroy_model_parallel()
diff --git a/tests/unit_tests/transformer/test_cuda_graphs.py b/tests/unit_tests/transformer/test_cuda_graphs.py
index b92ff383d82..7f49a559f32 100644
--- a/tests/unit_tests/transformer/test_cuda_graphs.py
+++ b/tests/unit_tests/transformer/test_cuda_graphs.py
@@ -1,11 +1,42 @@
# Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+import gc
+import os
+import random
+import sys
+import time
+import types
+
import pytest
import torch
-
-from megatron.core.models.gpt.gpt_layer_specs import get_gpt_layer_with_transformer_engine_spec
+from transformer_engine.pytorch.fp8 import check_fp8_support
+
+from megatron.core import parallel_state
+from megatron.core.enums import ModelType
+from megatron.core.inference.contexts import DynamicInferenceContext
+from megatron.core.inference.engines import DynamicInferenceEngine
+from megatron.core.inference.model_inference_wrappers.gpt.gpt_inference_wrapper import (
+ GPTInferenceWrapper,
+)
+from megatron.core.inference.model_inference_wrappers.inference_wrapper_config import (
+ InferenceWrapperConfig,
+)
+from megatron.core.inference.sampling_params import SamplingParams
+from megatron.core.inference.text_generation_controllers.text_generation_controller import (
+ TextGenerationController,
+)
+from megatron.core.models.gpt.gpt_layer_specs import (
+ get_gpt_decoder_block_spec,
+ get_gpt_layer_local_spec,
+ get_gpt_layer_with_transformer_engine_spec,
+ get_gpt_mtp_block_spec,
+)
from megatron.core.models.gpt.gpt_model import GPTModel
from megatron.core.models.mamba.mamba_layer_specs import mamba_stack_spec
+from megatron.core.num_microbatches_calculator import (
+ destroy_num_microbatches_calculator,
+ init_num_microbatches_calculator,
+)
from megatron.core.pipeline_parallel.schedules import set_current_microbatch
from megatron.core.process_groups_config import ProcessGroupCollection
from megatron.core.ssm.mamba_block import MambaStack
@@ -14,12 +45,28 @@
initialize_rng_tracker,
model_parallel_cuda_manual_seed,
)
-from megatron.core.transformer.cuda_graphs import CudaGraphManager, _CudagraphGlobalRecord
+from megatron.core.transformer.cuda_graphs import (
+ CudaGraphManager,
+ TECudaGraphHelper,
+ _CudagraphGlobalRecord,
+)
+from megatron.core.transformer.enums import CudaGraphScope
+from megatron.core.transformer.moe.fused_a2a import reset_hybrid_ep_buffer
from megatron.core.transformer.transformer_block import TransformerBlock
from megatron.core.transformer.transformer_config import TransformerConfig
from megatron.core.utils import is_fa_min_version, is_te_min_version
+from megatron.training.arguments import core_transformer_config_from_args, parse_args, validate_args
+from megatron.training.global_vars import (
+ destroy_global_vars,
+ get_args,
+ set_args,
+ set_global_variables,
+)
+from megatron.training.training import setup_model_and_optimizer
from tests.unit_tests.test_utilities import Utils
+fp8_available, _ = check_fp8_support()
+
class TestParallelTransformerBlockCudagraphs:
def setup_method(self, method):
@@ -497,6 +544,761 @@ def test_gpu_cudagraph(self):
del parallel_mamba_block.layers[_].cudagraph_manager.cudagraph_runners[0].fwd_graph
+class TestCaptureFreezeGC:
+
+ def capture_cuda_graphs(self, cuda_graph_capture_freeze_gc: bool) -> None:
+ """Capture multiple cuda graphs by initializing the `DynamicInferenceEngine`.
+
+ The `DynamicInferenceEngine` is used here because it is currently (as of
+ August 2025) one of the heaviest users of multiple cuda graphs, and so
+ its setup tests a realistic use-case of multi-batch size cuda graphs.
+
+ Args:
+ cuda_graph_capture_freeze_gc (bool): Flag that determines whether to
+ freeze garbage collection.
+ """
+
+ # Set freeze-gc environment variable.
+ os.environ["CUDA_GRAPH_CAPTURE_FREEZE_GC"] = str(int(cuda_graph_capture_freeze_gc))
+
+ # Configuration.
+ random_seed = 123
+ vocab_size = 100
+ num_tokens_to_prompt = 128
+ num_tokens_to_generate = 32
+ max_sequence_length = num_tokens_to_prompt + num_tokens_to_generate
+ num_cuda_graphs = 4
+
+ # Rounder values.
+ rounder = 4
+ DynamicInferenceContext.ROUNDER = rounder # For backwards compatibility
+ DynamicInferenceContext.TOKEN_ROUNDER = rounder
+ DynamicInferenceContext.REQUEST_ROUNDER = rounder
+
+ # Random state.
+ random.seed(random_seed)
+ torch.manual_seed(random_seed)
+ model_parallel_cuda_manual_seed(
+ seed=random_seed,
+ inference_rng_tracker=True,
+ use_cudagraphable_rng=False,
+ force_reset_rng=True,
+ )
+
+ # Transformer config.
+ transformer_config = TransformerConfig(
+ params_dtype=torch.bfloat16,
+ num_layers=4,
+ hidden_size=32,
+ num_attention_heads=4,
+ use_cpu_initialization=True,
+ cuda_graph_impl="local",
+ inference_rng_tracker=True,
+ tensor_model_parallel_size=1, # needed?
+ )
+
+ # Sampling params.
+ sampling_params = SamplingParams(num_tokens_to_generate=num_tokens_to_generate)
+
+ # GPT model.
+ model = GPTModel(
+ config=transformer_config,
+ transformer_layer_spec=get_gpt_layer_local_spec(),
+ vocab_size=vocab_size,
+ max_sequence_length=max_sequence_length,
+ parallel_output=True,
+ ).cuda()
+
+ for param in model.parameters():
+ param.data = param.data.to(transformer_config.params_dtype)
+
+ model.eval()
+
+ # Inference config.
+ inference_config = InferenceWrapperConfig(
+ hidden_size=transformer_config.hidden_size,
+ inference_batch_times_seqlen_threshold=400,
+ fp32_residual_connection=False,
+ params_dtype=transformer_config.params_dtype,
+ padded_vocab_size=vocab_size,
+ )
+
+ # Inference context.
+ context = DynamicInferenceContext(
+ params_dtype=transformer_config.params_dtype,
+ num_layers=transformer_config.num_layers,
+ kv_channels=transformer_config.kv_channels,
+ num_attention_heads=transformer_config.num_query_groups,
+ max_sequence_length=max_sequence_length,
+ num_cuda_graphs=num_cuda_graphs,
+ buffer_size_gb=20,
+ buffer_guaranteed_fraction=0.05,
+ block_size_tokens=256,
+ buffer_overflow_factor=1.1,
+ max_requests_override=512,
+ max_tokens_override=8196,
+ tensor_model_parallel_size=transformer_config.tensor_model_parallel_size,
+ )
+
+ # Inference model wrapper.
+ inference_wrapped_model = GPTInferenceWrapper(model, inference_config, context)
+
+ # Note: the following is taken from AbstractModelInferenceWrapper.prep_model_for_inference().
+ inference_wrapped_model.model_is_pipeline_parallel = not (
+ parallel_state.is_pipeline_first_stage() and parallel_state.is_pipeline_last_stage()
+ )
+
+ # Text generation controller.
+ text_generation_controller = TextGenerationController(
+ inference_wrapped_model=inference_wrapped_model,
+ tokenizer=types.SimpleNamespace(vocab_size=vocab_size),
+ )
+
+ # Inference engine.
+ engine = DynamicInferenceEngine(
+ text_generation_controller,
+ context,
+ termination_id=vocab_size - 1,
+ random_seed=random_seed,
+ )
+
+ return engine.capture_stats
+
+ @pytest.mark.flaky_in_dev # Issue #2855
+ @pytest.mark.flaky
+ @pytest.mark.experimental
+ @pytest.mark.skipif(
+ not is_fa_min_version("2.7.3"), reason="need latest flash attn for dynamic batching"
+ )
+ def test_capture_freeze_gc(self):
+ """Test cuda graph capture while freezing the GC."""
+
+ Utils.initialize_model_parallel(
+ tensor_model_parallel_size=1, pipeline_model_parallel_size=1
+ )
+
+ # Run tests with GC freeze off/on.
+ result_map = {}
+ for freeze_gc in (False, True):
+
+ # Reset global cuda graph state.
+ _CudagraphGlobalRecord.cudagraph_created = False
+ _CudagraphGlobalRecord.cudagraph_record = []
+ CudaGraphManager.global_mempool = None
+
+ # Capture multiple cuda graphs by initializing DynamicInferenceEngine.
+ mem_stats_start = torch.cuda.memory_stats()
+ time_start = time.time()
+ internal_stats = self.capture_cuda_graphs(freeze_gc)
+ time_end = time.time()
+ mem_stats_end = torch.cuda.memory_stats()
+
+ # Track local (external) stats, in addition to internal stats.
+ external_stats = {
+ "time": time_end - time_start,
+ "allocated_bytes": (
+ mem_stats_end["allocated_bytes.all.current"]
+ - mem_stats_start["allocated_bytes.all.current"]
+ ),
+ "reserved_bytes": (
+ mem_stats_end["reserved_bytes.all.current"]
+ - mem_stats_start["reserved_bytes.all.current"]
+ ),
+ }
+
+ # Record results.
+ result_map[freeze_gc] = {"internal": internal_stats, "external": external_stats}
+
+ # Extract results.
+ freeze_off_results = result_map[False]
+ freeze_on_results = result_map[True]
+ print(
+ "test capture | freeze off: internal %.3f, external %.3f."
+ % (freeze_off_results["internal"]["time"], freeze_off_results["external"]["time"])
+ )
+ print(
+ "test capture | freeze on: internal %.3f, external %.3f."
+ % (freeze_on_results["internal"]["time"], freeze_on_results["external"]["time"])
+ )
+
+ # Validate time and memory usage.
+ assert freeze_on_results["internal"]["time"] < 0.3 * freeze_off_results["internal"]["time"]
+ assert freeze_on_results["external"]["time"] < 0.3 * freeze_off_results["external"]["time"]
+ assert (
+ freeze_on_results["internal"]["allocated_bytes"]
+ <= freeze_off_results["internal"]["allocated_bytes"]
+ )
+ assert (
+ freeze_on_results["external"]["allocated_bytes"]
+ <= freeze_off_results["external"]["allocated_bytes"]
+ )
+ assert (
+ freeze_on_results["internal"]["reserved_bytes"]
+ <= freeze_off_results["internal"]["reserved_bytes"]
+ )
+ assert (
+ freeze_on_results["external"]["reserved_bytes"]
+ <= freeze_off_results["external"]["reserved_bytes"]
+ )
+
+
+# Global storage for comparing unique buffer counts across different num_microbatches,
+# keyed by (pp_size, vpp_size)
+_unique_buffer_counts = {}
+
+
+class TestTECudaGraphHelper:
+ def setup_method(self, method):
+ # Initialize parallel state
+ initialize_rng_tracker(use_te_rng_tracker=True, force_reset=True)
+
+ def teardown_method(self, method):
+ Utils.destroy_model_parallel()
+ destroy_global_vars()
+ destroy_num_microbatches_calculator()
+ # Note: _unique_buffer_counts is intentionally NOT cleared here so we can
+ # compare values across parametrized test runs
+
+ @pytest.mark.parametrize("num_microbatches", [16, 64, 256])
+ @pytest.mark.parametrize("pp_size", [1, 2, 4])
+ @pytest.mark.parametrize("vpp_size", [None, 2])
+ def test_get_cuda_graph_input_data(self, num_microbatches, pp_size, vpp_size):
+ """Test _get_cuda_graph_input_data function in TECudaGraphHelper."""
+
+ if vpp_size and pp_size == 1:
+ pytest.skip("vpp_size must be None when pp_size is 1")
+
+ Utils.initialize_model_parallel(
+ tensor_model_parallel_size=1,
+ pipeline_model_parallel_size=pp_size,
+ virtual_pipeline_model_parallel_size=vpp_size,
+ )
+
+ # Set up test configuration
+ seq_length = 128
+ micro_batch_size = 2
+ num_layers = 8
+ vocab_size = 1024
+ hidden_size = 64
+ num_attention_heads = 4
+
+ # Initialize num_microbatches calculator
+ init_num_microbatches_calculator(
+ rank=0,
+ rampup_batch_size=None,
+ global_batch_size=micro_batch_size * num_microbatches,
+ micro_batch_size=micro_batch_size,
+ data_parallel_size=1,
+ decrease_batch_size_if_needed=False,
+ )
+
+ # Create transformer config directly
+ transformer_config = TransformerConfig(
+ num_layers=num_layers,
+ hidden_size=hidden_size,
+ num_attention_heads=num_attention_heads,
+ use_cpu_initialization=True,
+ cuda_graph_impl="transformer_engine",
+ use_te_rng_tracker=True,
+ bf16=True,
+ tensor_model_parallel_size=1,
+ pipeline_model_parallel_size=pp_size,
+ virtual_pipeline_model_parallel_size=vpp_size,
+ pipeline_dtype=torch.bfloat16,
+ context_parallel_size=1,
+ )
+
+ # Create model
+ torch.manual_seed(123)
+ model_parallel_cuda_manual_seed(123)
+
+ model = []
+ for i in range(vpp_size or 1):
+ this_model = GPTModel(
+ config=transformer_config,
+ transformer_layer_spec=get_gpt_layer_with_transformer_engine_spec(),
+ vocab_size=vocab_size,
+ max_sequence_length=seq_length,
+ parallel_output=True,
+ position_embedding_type="rope",
+ vp_stage=i if vpp_size else None,
+ ).cuda()
+ model.append(this_model)
+
+ # Initialize TECudaGraphHelper
+ cuda_graph_helper = TECudaGraphHelper(
+ model=model,
+ config=transformer_config,
+ seq_length=seq_length,
+ micro_batch_size=micro_batch_size,
+ optimizers=[],
+ )
+
+ # Call _get_cuda_graph_input_data (which internally calls _get_sample_arguments)
+ sample_args, make_graphed_callables_kwargs = cuda_graph_helper._get_cuda_graph_input_data()
+
+ # Extract sample_kwargs from the kwargs dict
+ # For TE >= 1.10.0, sample_kwargs should always be present
+ assert (
+ 'sample_kwargs' in make_graphed_callables_kwargs
+ ), "sample_kwargs should be present in make_graphed_callables_kwargs for TE >= 1.10.0"
+ sample_kwargs = make_graphed_callables_kwargs['sample_kwargs']
+
+ # Basic checks
+ num_graphable_layers = len(cuda_graph_helper.flattened_callables)
+ if pp_size > 1:
+ expected_length = num_graphable_layers * num_microbatches
+ else:
+ expected_length = num_graphable_layers
+ assert len(sample_args) == expected_length, (
+ f"sample_args length mismatch: expected {expected_length}, " f"got {len(sample_args)}"
+ )
+ assert len(sample_kwargs) == expected_length, (
+ f"sample_kwargs length mismatch: expected {expected_length}, "
+ f"got {len(sample_kwargs)}"
+ )
+
+ # Check that all elements are not None
+ for i, (args_item, kwargs_item) in enumerate(zip(sample_args, sample_kwargs)):
+ assert args_item is not None, f"sample_args[{i}] is None"
+ assert kwargs_item is not None, f"sample_kwargs[{i}] is None"
+ assert isinstance(args_item, tuple), f"sample_args[{i}] should be a tuple"
+ assert isinstance(kwargs_item, dict), f"sample_kwargs[{i}] should be a dict"
+ assert len(args_item) > 0, f"sample_args[{i}] should not be empty"
+ # Check that hidden_states is present
+ assert "hidden_states" in kwargs_item or (
+ len(args_item) > 0 and torch.is_tensor(args_item[0])
+ ), f"sample_args[{i}] or sample_kwargs[{i}] should contain hidden_states"
+
+ # Check tensor properties
+ for i, (args_item, kwargs_item) in enumerate(zip(sample_args, sample_kwargs)):
+ # Get hidden_states from args or kwargs
+ if len(args_item) > 0 and torch.is_tensor(args_item[0]):
+ hidden_states = args_item[0]
+ elif "hidden_states" in kwargs_item:
+ hidden_states = kwargs_item["hidden_states"]
+ else:
+ continue
+
+ assert torch.is_tensor(hidden_states), f"hidden_states at index {i} should be a tensor"
+ # Check shape matches expected (accounting for TP/CP)
+ expected_seq_len = seq_length // transformer_config.context_parallel_size
+ if transformer_config.sequence_parallel:
+ expected_seq_len = expected_seq_len // transformer_config.tensor_model_parallel_size
+ assert hidden_states.shape[0] == expected_seq_len, (
+ f"hidden_states seq_len mismatch at index {i}: "
+ f"expected {expected_seq_len}, got {hidden_states.shape[0]}"
+ )
+ assert hidden_states.shape[1] == micro_batch_size, (
+ f"hidden_states batch_size mismatch at index {i}: "
+ f"expected {micro_batch_size}, got {hidden_states.shape[1]}"
+ )
+ assert hidden_states.shape[2] == transformer_config.hidden_size, (
+ f"hidden_states hidden_size mismatch at index {i}: "
+ f"expected {transformer_config.hidden_size}, got {hidden_states.shape[2]}"
+ )
+
+ # Memory optimization check: verify that buffers with same signature are reused
+ # Create a mapping of sample_keys to indices
+ sample_keys_to_indices = {}
+ for idx, (args_item, kwargs_item) in enumerate(zip(sample_args, sample_kwargs)):
+ # Create sample_keys similar to the function
+ args_keys = tuple((t.shape, t.dtype, t.layout) for t in args_item if torch.is_tensor(t))
+ kwargs_keys = tuple(
+ (k, v.shape, v.dtype, v.layout)
+ for k, v in sorted(kwargs_item.items())
+ if torch.is_tensor(v)
+ )
+ sample_keys = args_keys + kwargs_keys
+
+ if sample_keys not in sample_keys_to_indices:
+ sample_keys_to_indices[sample_keys] = []
+ sample_keys_to_indices[sample_keys].append(idx)
+
+ # Check that buffers with same signature share references (memory optimization)
+ # The optimization reuses buffers when:
+ # 1. They have the same signature (shape, dtype, layout)
+ # 2. The backward pass of the original buffer has completed
+ # 3. A new forward pass with matching signature needs a buffer
+ # Count how many times each tensor is reused
+ unique_tensors = set()
+ tensor_reuse_count = {}
+ for idx, (args_item, kwargs_item) in enumerate(zip(sample_args, sample_kwargs)):
+ # Get the first tensor from args (hidden_states)
+ if len(args_item) > 0 and torch.is_tensor(args_item[0]):
+ tensor_ptr = args_item[0].data_ptr()
+ unique_tensors.add(tensor_ptr)
+ tensor_reuse_count[tensor_ptr] = tensor_reuse_count.get(tensor_ptr, 0) + 1
+
+ # With memory optimization, we should see some buffers reused
+ # (i.e., some tensors should appear multiple times)
+ max_reuse = max(tensor_reuse_count.values()) if tensor_reuse_count else 0
+ total_entries = len(sample_args)
+ unique_buffer_count = len(unique_tensors)
+
+ # Verify that memory optimization is working:
+ # - The number of unique buffers should be <= total entries
+ # - With the 1F1B schedule and multiple microbatches, we should see some buffer reuse
+ # - The number of unique buffers should be bounded as num_microbatches grows.
+ assert unique_buffer_count <= total_entries, (
+ f"Memory optimization check: unique_buffer_count ({unique_buffer_count}) "
+ f"should be <= total_entries ({total_entries})"
+ )
+ global _unique_buffer_counts
+ # Use (pp_size, vpp_size) as key to track unique buffer counts per configuration
+ config_key = (pp_size, vpp_size)
+ if config_key not in _unique_buffer_counts:
+ _unique_buffer_counts[config_key] = unique_buffer_count
+ else:
+ assert unique_buffer_count == _unique_buffer_counts[config_key], (
+ f"Unique buffer count mismatch: expected {_unique_buffer_counts[config_key]}, "
+ f"got {unique_buffer_count}"
+ )
+
+ # Verify that buffers with the same signature can potentially be reused
+ # (the actual reuse depends on the schedule, but the mechanism should work)
+ if expected_length > 1:
+ # Check that we have multiple entries with the same signature
+ has_duplicate_signatures = any(
+ len(indices) > 1 for indices in sample_keys_to_indices.values()
+ )
+ assert has_duplicate_signatures, (
+ "Memory optimization: expected duplicate signatures for buffer reuse, "
+ "but all signatures are unique"
+ )
+
+ # We tested with a large number of microbatches, so we should see some buffer reuse.
+ if pp_size > 1:
+ assert max_reuse > 1, "Expected some buffer reuse"
+
+ # Verify that make_graphed_callables_kwargs contains expected keys
+ assert (
+ '_order' in make_graphed_callables_kwargs
+ ), "make_graphed_callables_kwargs should contain '_order'"
+ assert (
+ 'num_warmup_iters' in make_graphed_callables_kwargs
+ ), "make_graphed_callables_kwargs should contain 'num_warmup_iters'"
+ assert (
+ 'allow_unused_input' in make_graphed_callables_kwargs
+ ), "make_graphed_callables_kwargs should contain 'allow_unused_input'"
+
+ # Verify the order in kwargs matches expectations
+ order = make_graphed_callables_kwargs['_order']
+ num_model_chunks = cuda_graph_helper.num_model_chunks
+ forward_count = sum(1 for chunk_id in order if chunk_id > 0)
+ if pp_size > 1:
+ # Verify that all forward passes in order have corresponding entries in sample_args
+ assert forward_count == num_microbatches * num_model_chunks, (
+ f"Forward count mismatch: expected {num_microbatches * num_model_chunks}, "
+ f"got {forward_count}"
+ )
+ expected_order_length = num_microbatches * num_model_chunks * 2
+ else:
+ assert num_model_chunks == 1, "Expected only one model chunk for pp_size == 1"
+ assert forward_count == 1, "Expected only one forward pass for pp_size == 1"
+ expected_order_length = 2
+ assert (
+ len(order) == expected_order_length
+ ), f"Order length mismatch: expected {expected_order_length}, got {len(order)}"
+
+
+def is_deep_ep_available():
+ from megatron.core.transformer.moe.fused_a2a import HAVE_DEEP_EP
+
+ return HAVE_DEEP_EP
+
+
+def is_hybrid_ep_available():
+ from megatron.core.transformer.moe.fused_a2a import HAVE_HYBRIDEP
+
+ return HAVE_HYBRIDEP
+
+
+class TestPartialCudaGraph:
+ """Test that CUDA graph outputs match non-CUDA graph outputs for various scopes."""
+
+ def setup_method(self, method):
+ self.seq_length = 512
+ self.micro_batch_size = 2
+ self.tp_size = 2
+ self.cp_size = 2
+ self.cuda_graph_helper = None
+ # Store original environment variable values
+ self.original_env = {
+ 'CUDA_DEVICE_MAX_CONNECTIONS': os.environ.get('CUDA_DEVICE_MAX_CONNECTIONS'),
+ 'NVTE_ALLOW_NONDETERMINISTIC_ALGO': os.environ.get('NVTE_ALLOW_NONDETERMINISTIC_ALGO'),
+ }
+ os.environ['CUDA_DEVICE_MAX_CONNECTIONS'] = '1'
+ os.environ['NVTE_ALLOW_NONDETERMINISTIC_ALGO'] = '0'
+
+ def teardown_method(self, method):
+ # Restore original environment variable values
+ for key, value in self.original_env.items():
+ if value is None:
+ os.environ.pop(key, None)
+ else:
+ os.environ[key] = value
+ destroy_global_vars()
+ destroy_num_microbatches_calculator()
+ if self.cuda_graph_helper is not None and self.cuda_graph_helper.graphs_created():
+ self.cuda_graph_helper.delete_cuda_graphs()
+ self.cuda_graph_helper = None
+ gc.collect()
+
+ def model_provider(
+ self,
+ pre_process=True,
+ post_process=True,
+ layer_spec_fn=get_gpt_decoder_block_spec,
+ **config_kwargs,
+ ):
+ args = get_args()
+ config = core_transformer_config_from_args(args)
+ transformer_layer_spec = layer_spec_fn(
+ config,
+ use_transformer_engine=True,
+ normalization=args.normalization,
+ qk_l2_norm=args.qk_l2_norm,
+ )
+ if args.mtp_num_layers:
+ mtp_block_spec = get_gpt_mtp_block_spec(
+ config, transformer_layer_spec, use_transformer_engine=True
+ )
+ else:
+ mtp_block_spec = None
+ return GPTModel(
+ config=config,
+ transformer_layer_spec=transformer_layer_spec,
+ vocab_size=args.vocab_size,
+ max_sequence_length=args.max_position_embeddings,
+ pre_process=pre_process,
+ post_process=post_process,
+ fp16_lm_cross_entropy=args.fp16_lm_cross_entropy,
+ parallel_output=True,
+ share_embeddings_and_output_weights=not args.untie_embeddings_and_output_weights,
+ position_embedding_type=args.position_embedding_type,
+ rotary_percent=args.rotary_percent,
+ mtp_block_spec=mtp_block_spec,
+ )
+
+ def create_test_args(
+ self, cuda_graph_impl, cuda_graph_scope, cuda_graph_warmup_steps, ep_size, **kwargs
+ ):
+ destroy_global_vars()
+ destroy_num_microbatches_calculator()
+
+ sys.argv = ['test_cuda_graphs.py']
+ args = parse_args()
+ args.num_layers = 4
+ args.mtp_num_layers = 1
+ args.vocab_size = 1024
+ args.hidden_size = 512
+ args.num_attention_heads = 8
+ args.max_position_embeddings = 512
+ args.global_batch_size = self.micro_batch_size * 8 // self.tp_size // self.cp_size
+ args.micro_batch_size = self.micro_batch_size
+ args.create_attention_mask_in_dataloader = True
+ args.seq_length = self.seq_length
+ args.tensor_model_parallel_size = self.tp_size
+ args.sequence_parallel = True if self.tp_size > 1 else False
+ args.pipeline_model_parallel_size = 1
+ args.context_parallel_size = self.cp_size
+ args.train_iters = 10
+ args.lr = 3e-5
+ args.bf16 = True
+ args.add_bias_linear = False
+ args.swiglu = True
+ args.use_distributed_optimizer = True
+ args.position_embedding_type = "rope"
+ args.rotary_percent = 1.0
+ args.hidden_dropout = 0.0
+ args.attention_dropout = 0.0
+
+ # MoE settings
+ args.num_experts = 4
+ args.expert_model_parallel_size = ep_size
+ args.expert_tensor_parallel_size = 1 if ep_size > 1 else self.tp_size
+ args.moe_shared_expert_intermediate_size = 1024
+ args.moe_layer_freq = [0, 0, 1, 1]
+ args.moe_permute_fusion = True
+ args.moe_router_fusion = True
+ args.moe_router_topk = 2
+ args.moe_router_dtype = "fp32"
+
+ # CUDA graph settings
+ args.cuda_graph_impl = cuda_graph_impl
+ args.cuda_graph_scope = cuda_graph_scope
+ args.cuda_graph_warmup_steps = cuda_graph_warmup_steps
+
+ # fp8 settings
+ if fp8_available:
+ args.fp8 = "e4m3"
+ args.fp8_recipe = "tensorwise"
+ args.first_last_layers_bf16 = True
+ args.num_layers_at_start_in_bf16 = 1
+ args.num_layers_at_end_in_bf16 = 1
+
+ for key, value in kwargs.items():
+ assert hasattr(args, key)
+ setattr(args, key, value)
+
+ validate_args(args)
+ set_global_variables(args, False)
+ return args
+
+ def get_batch(self, seq_length, micro_batch_size, cp_size):
+ data = list(range(seq_length // cp_size))
+ input_ids = torch.tensor(data, dtype=torch.int64).repeat((micro_batch_size, 1)).cuda()
+ labels = 1 + torch.tensor(data, dtype=torch.int64).repeat((micro_batch_size, 1)).cuda()
+ position_ids = torch.tensor(data, dtype=torch.int64).repeat((micro_batch_size, 1)).cuda()
+ attention_mask = torch.ones(
+ (micro_batch_size, 1, seq_length // cp_size, seq_length), dtype=bool
+ ).cuda()
+ loss_mask = torch.ones(seq_length // cp_size).repeat((micro_batch_size, 1)).cuda()
+ return input_ids, labels, position_ids, attention_mask, loss_mask
+
+ def _run_test_helper(
+ self, ep_size, cuda_graph_impl, cuda_graph_scope, cuda_graph_warmup_steps, **kwargs
+ ):
+ """Test fp8_param with gpt_model."""
+ args = self.create_test_args(
+ cuda_graph_impl, cuda_graph_scope, cuda_graph_warmup_steps, ep_size, **kwargs
+ )
+
+ set_args(args)
+ torch.manual_seed(123)
+ model_parallel_cuda_manual_seed(123)
+
+ input_ids, labels, position_ids, attention_mask, loss_mask = self.get_batch(
+ self.seq_length, self.micro_batch_size, self.cp_size
+ )
+
+ gpt_model, optimizer, _ = setup_model_and_optimizer(
+ self.model_provider, ModelType.encoder_or_decoder
+ )
+ assert len(gpt_model) == 1 # Assume only one model in the model provider.
+
+ if cuda_graph_impl == "transformer_engine":
+ self.cuda_graph_helper = TECudaGraphHelper(
+ model=gpt_model,
+ config=gpt_model[0].config,
+ seq_length=self.seq_length,
+ micro_batch_size=self.micro_batch_size,
+ optimizers=[optimizer],
+ )
+
+ loss_list = []
+
+ for i in range(100):
+ gpt_model[0].zero_grad_buffer()
+ optimizer.zero_grad()
+
+ # Capture CUDA graphs after warmup if helper is provided
+ if self.cuda_graph_helper is not None and i == cuda_graph_warmup_steps:
+ self.cuda_graph_helper.create_cudagraphs()
+
+ gpt_model[0].set_is_first_microbatch()
+ output = gpt_model[0].forward(
+ input_ids=input_ids,
+ position_ids=position_ids,
+ attention_mask=attention_mask,
+ labels=labels,
+ loss_mask=loss_mask,
+ )
+
+ # Check output shapes
+ assert output.shape[0] == self.micro_batch_size
+ assert output.shape[1] == self.seq_length // self.cp_size
+
+ # Verify gradients
+ loss = output.mean()
+ loss.backward()
+
+ for param in gpt_model[0].parameters():
+ assert param.main_grad is not None
+
+ update_successful, _, _ = optimizer.step()
+ assert update_successful
+
+ loss_list.append(loss.item())
+
+ if self.cuda_graph_helper is not None and self.cuda_graph_helper.graphs_created():
+ self.cuda_graph_helper.delete_cuda_graphs()
+ self.cuda_graph_helper = None
+
+ return torch.tensor(loss_list)
+
+ @pytest.mark.skipif(
+ not (HAVE_TE and is_te_min_version("2.10.0")),
+ reason="Partial CUDA graph UT support requires TransformerEngine version >= 2.10.0",
+ )
+ @pytest.mark.parametrize("ep_size", [1, 4])
+ @pytest.mark.parametrize("moe_dropless_dispatcher", [False, True])
+ @pytest.mark.parametrize("moe_dispatcher_type", ["alltoall", "deepep", "hybridep"])
+ def test_moe_partial_cudagraph(self, ep_size, moe_dropless_dispatcher, moe_dispatcher_type):
+ initialize_rng_tracker(use_te_rng_tracker=True, force_reset=True)
+ Utils.initialize_model_parallel(
+ tensor_model_parallel_size=self.tp_size,
+ context_parallel_size=self.cp_size,
+ pipeline_model_parallel_size=1,
+ expert_tensor_parallel_size=1 if ep_size > 1 else self.tp_size,
+ expert_model_parallel_size=ep_size,
+ )
+
+ extra_kwargs = {}
+ if moe_dispatcher_type == "deepep":
+ if not is_deep_ep_available():
+ pytest.skip("Deep EP is not available")
+ extra_kwargs["moe_token_dispatcher_type"] = "flex"
+ extra_kwargs["moe_flex_dispatcher_backend"] = "deepep"
+ elif moe_dispatcher_type == "hybridep":
+ if not is_hybrid_ep_available():
+ pytest.skip("Hybrid EP is not available")
+ extra_kwargs["moe_token_dispatcher_type"] = "flex"
+ extra_kwargs["moe_flex_dispatcher_backend"] = "hybridep"
+ else:
+ extra_kwargs["moe_token_dispatcher_type"] = moe_dispatcher_type
+ if not moe_dropless_dispatcher:
+ if moe_dispatcher_type == "deepep":
+ pytest.skip("Deep EP doesn't support drop&pad MoE")
+ if moe_dispatcher_type == "hybridep" and ep_size == 1:
+ pytest.skip("Hybrid EP doesn't support drop&pad MoE with ep_size == 1")
+ extra_kwargs["moe_expert_capacity_factor"] = 1.0
+ extra_kwargs["moe_pad_expert_input_to_capacity"] = True
+
+ loss_list_ref = self._run_test_helper(ep_size, "none", None, 0, **extra_kwargs)
+ for cuda_graph_scope in [
+ None,
+ [CudaGraphScope.attn],
+ [CudaGraphScope.moe],
+ [CudaGraphScope.mlp, CudaGraphScope.moe_router],
+ [
+ CudaGraphScope.attn,
+ CudaGraphScope.mlp,
+ CudaGraphScope.moe_router,
+ CudaGraphScope.moe_preprocess,
+ ],
+ ]:
+ if (moe_dropless_dispatcher or moe_dispatcher_type == "hybridep") and (
+ cuda_graph_scope is None or CudaGraphScope.moe in cuda_graph_scope
+ ):
+ # Dropless MoE or Hybrid EP doesn't work with "moe" scope cudagraph. Skip.
+ continue
+ cuda_graph_warmup_steps = 3
+ loss_list = self._run_test_helper(
+ ep_size,
+ "transformer_engine",
+ cuda_graph_scope,
+ cuda_graph_warmup_steps,
+ **extra_kwargs,
+ )
+ assert torch.equal(loss_list, loss_list_ref)
+
+ if moe_dispatcher_type == "hybridep":
+ reset_hybrid_ep_buffer()
+ Utils.destroy_model_parallel()
+
+
if __name__ == "__main__":
test = TestParallelTransformerBlockCudagraphs()
@@ -508,3 +1310,11 @@ def test_gpu_cudagraph(self):
llava_test.setup_method(method=None)
llava_test.test_llava_cudagraph_is_last_layer_logic()
llava_test.teardown_method(method=None)
+
+ test = TestCaptureFreezeGC()
+ test.test_capture_freeze_gc()
+
+ test = TestPartialCudaGraph()
+ test.setup_method(method=None)
+ test.test_moe_partial_cudagraph(4, True, "alltoall")
+ test.teardown_method(method=None)
diff --git a/tests/unit_tests/transformer/test_multi_token_prediction.py b/tests/unit_tests/transformer/test_multi_token_prediction.py
index 05fb2c4fe63..ddfa9bfba16 100644
--- a/tests/unit_tests/transformer/test_multi_token_prediction.py
+++ b/tests/unit_tests/transformer/test_multi_token_prediction.py
@@ -104,7 +104,7 @@ def test_constructor_local(self, tp):
assert num_weights == 15216 * config.mtp_num_layers
@pytest.mark.skipif(not HAVE_TE, reason="transformer_engine not available")
- @pytest.mark.parametrize(('tp', 'cp'), [(1, 1), (1, 2), (2, 1), (2, 2)])
+ @pytest.mark.parametrize(('tp', 'cp'), [(1, 1), (2, 1), (2, 2)])
def test_constructor_ues_te(self, tp, cp):
"""Test basic construction of MTP module."""
torch.manual_seed(_SEED)
@@ -312,7 +312,7 @@ def get_packed_batch(self, seq_lengths, micro_batch_size):
not HAVE_TE or not is_te_min_version("2.1.0"),
reason="grouped_gemm requires TransformerEngine >= 2.1.0",
)
- @pytest.mark.parametrize(("tp", "cp"), [(1, 1), (1, 2), (2, 1), (2, 2)])
+ @pytest.mark.parametrize(("tp", "cp"), [(2, 1), (2, 2)])
def test_sharded_state_dict(self, tp, cp):
"""Test MTP with different tensor parallel sizes."""
args = self.create_test_args(tp, cp, self.seq_length, self.micro_batch_size)
@@ -331,9 +331,8 @@ def test_sharded_state_dict(self, tp, cp):
not HAVE_TE or not is_te_min_version("2.1.0"),
reason="grouped_gemm requires TransformerEngine >= 2.1.0",
)
- @pytest.mark.parametrize("full_recompute", [False, True])
@pytest.mark.parametrize(
- ("tp", "cp"), [(1, 1), (1, 2), (1, 4), (2, 1), (2, 2), (2, 4), (4, 1), (4, 2)]
+ ("tp", "cp", "full_recompute"), [(1, 1, False), (1, 4, False), (2, 4, False), (4, 1, True)]
)
def test_forward_backward(self, tmp_path_dist_ckpt, tp, cp, full_recompute):
"""Test MTP forward and backward with gptmodel."""
diff --git a/tests/unit_tests/transformer/test_submodule_callables.py b/tests/unit_tests/transformer/test_submodule_callables.py
index 1ccb6fd5be8..73059495c06 100644
--- a/tests/unit_tests/transformer/test_submodule_callables.py
+++ b/tests/unit_tests/transformer/test_submodule_callables.py
@@ -64,7 +64,7 @@ def run_model_submodules_with_capture(model, input_tensors, microbatches):
output_tensors = []
# get callables
callables, dw = build_layer_callables(model)
- attn, post_attn, dispatch, moe, combine, post_process = callables
+ attn, dispatch, moe, combine, post_process = callables
assert post_process is None
dummy_model = DummyState()
dummy_model.decoder = DummyState()
@@ -76,24 +76,16 @@ def run_model_submodules_with_capture(model, input_tensors, microbatches):
node.chunk_state.model = dummy_model
# attn fwd
- hidden_states = attn(node, input_tensors[i])
-
- # post attn fwd
- local_tokens, probs = post_attn(node, hidden_states)
+ local_tokens, probs = attn(node, input_tensors[i])
# dispatch fwd
dispatched_tokens = dispatch(node, local_tokens, probs)
# moe fwd
- expert_outputs = moe(node, dispatched_tokens)
- if model.mlp.use_shared_expert:
- expert_output, shared_expert_output = expert_outputs
- else:
- expert_output = expert_outputs
- shared_expert_output = None
+ expert_output = moe(node, dispatched_tokens)
# combine fwd
- hidden_states = combine(node, expert_output, shared_expert_output)
+ hidden_states = combine(node, expert_output)
# loss
output_tensors.append(hidden_states)
diff --git a/tools/checkpoint/checkpoint_inspector.py b/tools/checkpoint/checkpoint_inspector.py
index c62f0ca7417..3d03f4db959 100644
--- a/tools/checkpoint/checkpoint_inspector.py
+++ b/tools/checkpoint/checkpoint_inspector.py
@@ -1,3 +1,5 @@
+# Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+
# python checkpoint_inspector.py inspect /path/to/checkpoint
# torchrun --nproc_per_node=8 --nnodes=1 checkpoint_inspector.py convert-torch-dist-to-fsdp-dtensor /path/to/input_checkpoint /path/to/output_checkpoint --swiglu
import gc
diff --git a/uv.lock b/uv.lock
index 4ff93717fa5..15892827c83 100644
--- a/uv.lock
+++ b/uv.lock
@@ -75,7 +75,7 @@ wheels = [
[[package]]
name = "aiohttp"
-version = "3.13.2"
+version = "3.13.3"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "aiohappyeyeballs" },
@@ -87,110 +87,110 @@ dependencies = [
{ name = "propcache" },
{ name = "yarl" },
]
-sdist = { url = "https://files.pythonhosted.org/packages/1c/ce/3b83ebba6b3207a7135e5fcaba49706f8a4b6008153b4e30540c982fae26/aiohttp-3.13.2.tar.gz", hash = "sha256:40176a52c186aefef6eb3cad2cdd30cd06e3afbe88fe8ab2af9c0b90f228daca", size = 7837994, upload-time = "2025-10-28T20:59:39.937Z" }
-wheels = [
- { url = "https://files.pythonhosted.org/packages/6d/34/939730e66b716b76046dedfe0842995842fa906ccc4964bba414ff69e429/aiohttp-3.13.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:2372b15a5f62ed37789a6b383ff7344fc5b9f243999b0cd9b629d8bc5f5b4155", size = 736471, upload-time = "2025-10-28T20:55:27.924Z" },
- { url = "https://files.pythonhosted.org/packages/fd/cf/dcbdf2df7f6ca72b0bb4c0b4509701f2d8942cf54e29ca197389c214c07f/aiohttp-3.13.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:e7f8659a48995edee7229522984bd1009c1213929c769c2daa80b40fe49a180c", size = 493985, upload-time = "2025-10-28T20:55:29.456Z" },
- { url = "https://files.pythonhosted.org/packages/9d/87/71c8867e0a1d0882dcbc94af767784c3cb381c1c4db0943ab4aae4fed65e/aiohttp-3.13.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:939ced4a7add92296b0ad38892ce62b98c619288a081170695c6babe4f50e636", size = 489274, upload-time = "2025-10-28T20:55:31.134Z" },
- { url = "https://files.pythonhosted.org/packages/38/0f/46c24e8dae237295eaadd113edd56dee96ef6462adf19b88592d44891dc5/aiohttp-3.13.2-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6315fb6977f1d0dd41a107c527fee2ed5ab0550b7d885bc15fee20ccb17891da", size = 1668171, upload-time = "2025-10-28T20:55:36.065Z" },
- { url = "https://files.pythonhosted.org/packages/eb/c6/4cdfb4440d0e28483681a48f69841fa5e39366347d66ef808cbdadddb20e/aiohttp-3.13.2-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:6e7352512f763f760baaed2637055c49134fd1d35b37c2dedfac35bfe5cf8725", size = 1636036, upload-time = "2025-10-28T20:55:37.576Z" },
- { url = "https://files.pythonhosted.org/packages/84/37/8708cf678628216fb678ab327a4e1711c576d6673998f4f43e86e9ae90dd/aiohttp-3.13.2-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e09a0a06348a2dd73e7213353c90d709502d9786219f69b731f6caa0efeb46f5", size = 1727975, upload-time = "2025-10-28T20:55:39.457Z" },
- { url = "https://files.pythonhosted.org/packages/e6/2e/3ebfe12fdcb9b5f66e8a0a42dffcd7636844c8a018f261efb2419f68220b/aiohttp-3.13.2-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a09a6d073fb5789456545bdee2474d14395792faa0527887f2f4ec1a486a59d3", size = 1815823, upload-time = "2025-10-28T20:55:40.958Z" },
- { url = "https://files.pythonhosted.org/packages/a1/4f/ca2ef819488cbb41844c6cf92ca6dd15b9441e6207c58e5ae0e0fc8d70ad/aiohttp-3.13.2-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b59d13c443f8e049d9e94099c7e412e34610f1f49be0f230ec656a10692a5802", size = 1669374, upload-time = "2025-10-28T20:55:42.745Z" },
- { url = "https://files.pythonhosted.org/packages/f8/fe/1fe2e1179a0d91ce09c99069684aab619bf2ccde9b20bd6ca44f8837203e/aiohttp-3.13.2-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:20db2d67985d71ca033443a1ba2001c4b5693fe09b0e29f6d9358a99d4d62a8a", size = 1555315, upload-time = "2025-10-28T20:55:44.264Z" },
- { url = "https://files.pythonhosted.org/packages/5a/2b/f3781899b81c45d7cbc7140cddb8a3481c195e7cbff8e36374759d2ab5a5/aiohttp-3.13.2-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:960c2fc686ba27b535f9fd2b52d87ecd7e4fd1cf877f6a5cba8afb5b4a8bd204", size = 1639140, upload-time = "2025-10-28T20:55:46.626Z" },
- { url = "https://files.pythonhosted.org/packages/72/27/c37e85cd3ece6f6c772e549bd5a253d0c122557b25855fb274224811e4f2/aiohttp-3.13.2-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:6c00dbcf5f0d88796151e264a8eab23de2997c9303dd7c0bf622e23b24d3ce22", size = 1645496, upload-time = "2025-10-28T20:55:48.933Z" },
- { url = "https://files.pythonhosted.org/packages/66/20/3af1ab663151bd3780b123e907761cdb86ec2c4e44b2d9b195ebc91fbe37/aiohttp-3.13.2-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:fed38a5edb7945f4d1bcabe2fcd05db4f6ec7e0e82560088b754f7e08d93772d", size = 1697625, upload-time = "2025-10-28T20:55:50.377Z" },
- { url = "https://files.pythonhosted.org/packages/95/eb/ae5cab15efa365e13d56b31b0d085a62600298bf398a7986f8388f73b598/aiohttp-3.13.2-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:b395bbca716c38bef3c764f187860e88c724b342c26275bc03e906142fc5964f", size = 1542025, upload-time = "2025-10-28T20:55:51.861Z" },
- { url = "https://files.pythonhosted.org/packages/e9/2d/1683e8d67ec72d911397fe4e575688d2a9b8f6a6e03c8fdc9f3fd3d4c03f/aiohttp-3.13.2-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:204ffff2426c25dfda401ba08da85f9c59525cdc42bda26660463dd1cbcfec6f", size = 1714918, upload-time = "2025-10-28T20:55:53.515Z" },
- { url = "https://files.pythonhosted.org/packages/99/a2/ffe8e0e1c57c5e542d47ffa1fcf95ef2b3ea573bf7c4d2ee877252431efc/aiohttp-3.13.2-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:05c4dd3c48fb5f15db31f57eb35374cb0c09afdde532e7fb70a75aede0ed30f6", size = 1656113, upload-time = "2025-10-28T20:55:55.438Z" },
- { url = "https://files.pythonhosted.org/packages/0d/42/d511aff5c3a2b06c09d7d214f508a4ad8ac7799817f7c3d23e7336b5e896/aiohttp-3.13.2-cp310-cp310-win32.whl", hash = "sha256:e574a7d61cf10351d734bcddabbe15ede0eaa8a02070d85446875dc11189a251", size = 432290, upload-time = "2025-10-28T20:55:56.96Z" },
- { url = "https://files.pythonhosted.org/packages/8b/ea/1c2eb7098b5bad4532994f2b7a8228d27674035c9b3234fe02c37469ef14/aiohttp-3.13.2-cp310-cp310-win_amd64.whl", hash = "sha256:364f55663085d658b8462a1c3f17b2b84a5c2e1ba858e1b79bff7b2e24ad1514", size = 455075, upload-time = "2025-10-28T20:55:58.373Z" },
- { url = "https://files.pythonhosted.org/packages/35/74/b321e7d7ca762638cdf8cdeceb39755d9c745aff7a64c8789be96ddf6e96/aiohttp-3.13.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:4647d02df098f6434bafd7f32ad14942f05a9caa06c7016fdcc816f343997dd0", size = 743409, upload-time = "2025-10-28T20:56:00.354Z" },
- { url = "https://files.pythonhosted.org/packages/99/3d/91524b905ec473beaf35158d17f82ef5a38033e5809fe8742e3657cdbb97/aiohttp-3.13.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:e3403f24bcb9c3b29113611c3c16a2a447c3953ecf86b79775e7be06f7ae7ccb", size = 497006, upload-time = "2025-10-28T20:56:01.85Z" },
- { url = "https://files.pythonhosted.org/packages/eb/d3/7f68bc02a67716fe80f063e19adbd80a642e30682ce74071269e17d2dba1/aiohttp-3.13.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:43dff14e35aba17e3d6d5ba628858fb8cb51e30f44724a2d2f0c75be492c55e9", size = 493195, upload-time = "2025-10-28T20:56:03.314Z" },
- { url = "https://files.pythonhosted.org/packages/98/31/913f774a4708775433b7375c4f867d58ba58ead833af96c8af3621a0d243/aiohttp-3.13.2-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e2a9ea08e8c58bb17655630198833109227dea914cd20be660f52215f6de5613", size = 1747759, upload-time = "2025-10-28T20:56:04.904Z" },
- { url = "https://files.pythonhosted.org/packages/e8/63/04efe156f4326f31c7c4a97144f82132c3bb21859b7bb84748d452ccc17c/aiohttp-3.13.2-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:53b07472f235eb80e826ad038c9d106c2f653584753f3ddab907c83f49eedead", size = 1704456, upload-time = "2025-10-28T20:56:06.986Z" },
- { url = "https://files.pythonhosted.org/packages/8e/02/4e16154d8e0a9cf4ae76f692941fd52543bbb148f02f098ca73cab9b1c1b/aiohttp-3.13.2-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e736c93e9c274fce6419af4aac199984d866e55f8a4cec9114671d0ea9688780", size = 1807572, upload-time = "2025-10-28T20:56:08.558Z" },
- { url = "https://files.pythonhosted.org/packages/34/58/b0583defb38689e7f06798f0285b1ffb3a6fb371f38363ce5fd772112724/aiohttp-3.13.2-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:ff5e771f5dcbc81c64898c597a434f7682f2259e0cd666932a913d53d1341d1a", size = 1895954, upload-time = "2025-10-28T20:56:10.545Z" },
- { url = "https://files.pythonhosted.org/packages/6b/f3/083907ee3437425b4e376aa58b2c915eb1a33703ec0dc30040f7ae3368c6/aiohttp-3.13.2-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a3b6fb0c207cc661fa0bf8c66d8d9b657331ccc814f4719468af61034b478592", size = 1747092, upload-time = "2025-10-28T20:56:12.118Z" },
- { url = "https://files.pythonhosted.org/packages/ac/61/98a47319b4e425cc134e05e5f3fc512bf9a04bf65aafd9fdcda5d57ec693/aiohttp-3.13.2-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:97a0895a8e840ab3520e2288db7cace3a1981300d48babeb50e7425609e2e0ab", size = 1606815, upload-time = "2025-10-28T20:56:14.191Z" },
- { url = "https://files.pythonhosted.org/packages/97/4b/e78b854d82f66bb974189135d31fce265dee0f5344f64dd0d345158a5973/aiohttp-3.13.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:9e8f8afb552297aca127c90cb840e9a1d4bfd6a10d7d8f2d9176e1acc69bad30", size = 1723789, upload-time = "2025-10-28T20:56:16.101Z" },
- { url = "https://files.pythonhosted.org/packages/ed/fc/9d2ccc794fc9b9acd1379d625c3a8c64a45508b5091c546dea273a41929e/aiohttp-3.13.2-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:ed2f9c7216e53c3df02264f25d824b079cc5914f9e2deba94155190ef648ee40", size = 1718104, upload-time = "2025-10-28T20:56:17.655Z" },
- { url = "https://files.pythonhosted.org/packages/66/65/34564b8765ea5c7d79d23c9113135d1dd3609173da13084830f1507d56cf/aiohttp-3.13.2-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:99c5280a329d5fa18ef30fd10c793a190d996567667908bef8a7f81f8202b948", size = 1785584, upload-time = "2025-10-28T20:56:19.238Z" },
- { url = "https://files.pythonhosted.org/packages/30/be/f6a7a426e02fc82781afd62016417b3948e2207426d90a0e478790d1c8a4/aiohttp-3.13.2-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:2ca6ffef405fc9c09a746cb5d019c1672cd7f402542e379afc66b370833170cf", size = 1595126, upload-time = "2025-10-28T20:56:20.836Z" },
- { url = "https://files.pythonhosted.org/packages/e5/c7/8e22d5d28f94f67d2af496f14a83b3c155d915d1fe53d94b66d425ec5b42/aiohttp-3.13.2-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:47f438b1a28e926c37632bff3c44df7d27c9b57aaf4e34b1def3c07111fdb782", size = 1800665, upload-time = "2025-10-28T20:56:22.922Z" },
- { url = "https://files.pythonhosted.org/packages/d1/11/91133c8b68b1da9fc16555706aa7276fdf781ae2bb0876c838dd86b8116e/aiohttp-3.13.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:9acda8604a57bb60544e4646a4615c1866ee6c04a8edef9b8ee6fd1d8fa2ddc8", size = 1739532, upload-time = "2025-10-28T20:56:25.924Z" },
- { url = "https://files.pythonhosted.org/packages/17/6b/3747644d26a998774b21a616016620293ddefa4d63af6286f389aedac844/aiohttp-3.13.2-cp311-cp311-win32.whl", hash = "sha256:868e195e39b24aaa930b063c08bb0c17924899c16c672a28a65afded9c46c6ec", size = 431876, upload-time = "2025-10-28T20:56:27.524Z" },
- { url = "https://files.pythonhosted.org/packages/c3/63/688462108c1a00eb9f05765331c107f95ae86f6b197b865d29e930b7e462/aiohttp-3.13.2-cp311-cp311-win_amd64.whl", hash = "sha256:7fd19df530c292542636c2a9a85854fab93474396a52f1695e799186bbd7f24c", size = 456205, upload-time = "2025-10-28T20:56:29.062Z" },
- { url = "https://files.pythonhosted.org/packages/29/9b/01f00e9856d0a73260e86dd8ed0c2234a466c5c1712ce1c281548df39777/aiohttp-3.13.2-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:b1e56bab2e12b2b9ed300218c351ee2a3d8c8fdab5b1ec6193e11a817767e47b", size = 737623, upload-time = "2025-10-28T20:56:30.797Z" },
- { url = "https://files.pythonhosted.org/packages/5a/1b/4be39c445e2b2bd0aab4ba736deb649fabf14f6757f405f0c9685019b9e9/aiohttp-3.13.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:364e25edaabd3d37b1db1f0cbcee8c73c9a3727bfa262b83e5e4cf3489a2a9dc", size = 492664, upload-time = "2025-10-28T20:56:32.708Z" },
- { url = "https://files.pythonhosted.org/packages/28/66/d35dcfea8050e131cdd731dff36434390479b4045a8d0b9d7111b0a968f1/aiohttp-3.13.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:c5c94825f744694c4b8db20b71dba9a257cd2ba8e010a803042123f3a25d50d7", size = 491808, upload-time = "2025-10-28T20:56:34.57Z" },
- { url = "https://files.pythonhosted.org/packages/00/29/8e4609b93e10a853b65f8291e64985de66d4f5848c5637cddc70e98f01f8/aiohttp-3.13.2-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ba2715d842ffa787be87cbfce150d5e88c87a98e0b62e0f5aa489169a393dbbb", size = 1738863, upload-time = "2025-10-28T20:56:36.377Z" },
- { url = "https://files.pythonhosted.org/packages/9d/fa/4ebdf4adcc0def75ced1a0d2d227577cd7b1b85beb7edad85fcc87693c75/aiohttp-3.13.2-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:585542825c4bc662221fb257889e011a5aa00f1ae4d75d1d246a5225289183e3", size = 1700586, upload-time = "2025-10-28T20:56:38.034Z" },
- { url = "https://files.pythonhosted.org/packages/da/04/73f5f02ff348a3558763ff6abe99c223381b0bace05cd4530a0258e52597/aiohttp-3.13.2-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:39d02cb6025fe1aabca329c5632f48c9532a3dabccd859e7e2f110668972331f", size = 1768625, upload-time = "2025-10-28T20:56:39.75Z" },
- { url = "https://files.pythonhosted.org/packages/f8/49/a825b79ffec124317265ca7d2344a86bcffeb960743487cb11988ffb3494/aiohttp-3.13.2-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:e67446b19e014d37342f7195f592a2a948141d15a312fe0e700c2fd2f03124f6", size = 1867281, upload-time = "2025-10-28T20:56:41.471Z" },
- { url = "https://files.pythonhosted.org/packages/b9/48/adf56e05f81eac31edcfae45c90928f4ad50ef2e3ea72cb8376162a368f8/aiohttp-3.13.2-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4356474ad6333e41ccefd39eae869ba15a6c5299c9c01dfdcfdd5c107be4363e", size = 1752431, upload-time = "2025-10-28T20:56:43.162Z" },
- { url = "https://files.pythonhosted.org/packages/30/ab/593855356eead019a74e862f21523db09c27f12fd24af72dbc3555b9bfd9/aiohttp-3.13.2-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:eeacf451c99b4525f700f078becff32c32ec327b10dcf31306a8a52d78166de7", size = 1562846, upload-time = "2025-10-28T20:56:44.85Z" },
- { url = "https://files.pythonhosted.org/packages/39/0f/9f3d32271aa8dc35036e9668e31870a9d3b9542dd6b3e2c8a30931cb27ae/aiohttp-3.13.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:d8a9b889aeabd7a4e9af0b7f4ab5ad94d42e7ff679aaec6d0db21e3b639ad58d", size = 1699606, upload-time = "2025-10-28T20:56:46.519Z" },
- { url = "https://files.pythonhosted.org/packages/2c/3c/52d2658c5699b6ef7692a3f7128b2d2d4d9775f2a68093f74bca06cf01e1/aiohttp-3.13.2-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:fa89cb11bc71a63b69568d5b8a25c3ca25b6d54c15f907ca1c130d72f320b76b", size = 1720663, upload-time = "2025-10-28T20:56:48.528Z" },
- { url = "https://files.pythonhosted.org/packages/9b/d4/8f8f3ff1fb7fb9e3f04fcad4e89d8a1cd8fc7d05de67e3de5b15b33008ff/aiohttp-3.13.2-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:8aa7c807df234f693fed0ecd507192fc97692e61fee5702cdc11155d2e5cadc8", size = 1737939, upload-time = "2025-10-28T20:56:50.77Z" },
- { url = "https://files.pythonhosted.org/packages/03/d3/ddd348f8a27a634daae39a1b8e291ff19c77867af438af844bf8b7e3231b/aiohttp-3.13.2-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:9eb3e33fdbe43f88c3c75fa608c25e7c47bbd80f48d012763cb67c47f39a7e16", size = 1555132, upload-time = "2025-10-28T20:56:52.568Z" },
- { url = "https://files.pythonhosted.org/packages/39/b8/46790692dc46218406f94374903ba47552f2f9f90dad554eed61bfb7b64c/aiohttp-3.13.2-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:9434bc0d80076138ea986833156c5a48c9c7a8abb0c96039ddbb4afc93184169", size = 1764802, upload-time = "2025-10-28T20:56:54.292Z" },
- { url = "https://files.pythonhosted.org/packages/ba/e4/19ce547b58ab2a385e5f0b8aa3db38674785085abcf79b6e0edd1632b12f/aiohttp-3.13.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:ff15c147b2ad66da1f2cbb0622313f2242d8e6e8f9b79b5206c84523a4473248", size = 1719512, upload-time = "2025-10-28T20:56:56.428Z" },
- { url = "https://files.pythonhosted.org/packages/70/30/6355a737fed29dcb6dfdd48682d5790cb5eab050f7b4e01f49b121d3acad/aiohttp-3.13.2-cp312-cp312-win32.whl", hash = "sha256:27e569eb9d9e95dbd55c0fc3ec3a9335defbf1d8bc1d20171a49f3c4c607b93e", size = 426690, upload-time = "2025-10-28T20:56:58.736Z" },
- { url = "https://files.pythonhosted.org/packages/0a/0d/b10ac09069973d112de6ef980c1f6bb31cb7dcd0bc363acbdad58f927873/aiohttp-3.13.2-cp312-cp312-win_amd64.whl", hash = "sha256:8709a0f05d59a71f33fd05c17fc11fcb8c30140506e13c2f5e8ee1b8964e1b45", size = 453465, upload-time = "2025-10-28T20:57:00.795Z" },
- { url = "https://files.pythonhosted.org/packages/bf/78/7e90ca79e5aa39f9694dcfd74f4720782d3c6828113bb1f3197f7e7c4a56/aiohttp-3.13.2-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:7519bdc7dfc1940d201651b52bf5e03f5503bda45ad6eacf64dda98be5b2b6be", size = 732139, upload-time = "2025-10-28T20:57:02.455Z" },
- { url = "https://files.pythonhosted.org/packages/db/ed/1f59215ab6853fbaa5c8495fa6cbc39edfc93553426152b75d82a5f32b76/aiohttp-3.13.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:088912a78b4d4f547a1f19c099d5a506df17eacec3c6f4375e2831ec1d995742", size = 490082, upload-time = "2025-10-28T20:57:04.784Z" },
- { url = "https://files.pythonhosted.org/packages/68/7b/fe0fe0f5e05e13629d893c760465173a15ad0039c0a5b0d0040995c8075e/aiohttp-3.13.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:5276807b9de9092af38ed23ce120539ab0ac955547b38563a9ba4f5b07b95293", size = 489035, upload-time = "2025-10-28T20:57:06.894Z" },
- { url = "https://files.pythonhosted.org/packages/d2/04/db5279e38471b7ac801d7d36a57d1230feeee130bbe2a74f72731b23c2b1/aiohttp-3.13.2-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1237c1375eaef0db4dcd7c2559f42e8af7b87ea7d295b118c60c36a6e61cb811", size = 1720387, upload-time = "2025-10-28T20:57:08.685Z" },
- { url = "https://files.pythonhosted.org/packages/31/07/8ea4326bd7dae2bd59828f69d7fdc6e04523caa55e4a70f4a8725a7e4ed2/aiohttp-3.13.2-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:96581619c57419c3d7d78703d5b78c1e5e5fc0172d60f555bdebaced82ded19a", size = 1688314, upload-time = "2025-10-28T20:57:10.693Z" },
- { url = "https://files.pythonhosted.org/packages/48/ab/3d98007b5b87ffd519d065225438cc3b668b2f245572a8cb53da5dd2b1bc/aiohttp-3.13.2-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:a2713a95b47374169409d18103366de1050fe0ea73db358fc7a7acb2880422d4", size = 1756317, upload-time = "2025-10-28T20:57:12.563Z" },
- { url = "https://files.pythonhosted.org/packages/97/3d/801ca172b3d857fafb7b50c7c03f91b72b867a13abca982ed6b3081774ef/aiohttp-3.13.2-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:228a1cd556b3caca590e9511a89444925da87d35219a49ab5da0c36d2d943a6a", size = 1858539, upload-time = "2025-10-28T20:57:14.623Z" },
- { url = "https://files.pythonhosted.org/packages/f7/0d/4764669bdf47bd472899b3d3db91fffbe925c8e3038ec591a2fd2ad6a14d/aiohttp-3.13.2-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ac6cde5fba8d7d8c6ac963dbb0256a9854e9fafff52fbcc58fdf819357892c3e", size = 1739597, upload-time = "2025-10-28T20:57:16.399Z" },
- { url = "https://files.pythonhosted.org/packages/c4/52/7bd3c6693da58ba16e657eb904a5b6decfc48ecd06e9ac098591653b1566/aiohttp-3.13.2-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:f2bef8237544f4e42878c61cef4e2839fee6346dc60f5739f876a9c50be7fcdb", size = 1555006, upload-time = "2025-10-28T20:57:18.288Z" },
- { url = "https://files.pythonhosted.org/packages/48/30/9586667acec5993b6f41d2ebcf96e97a1255a85f62f3c653110a5de4d346/aiohttp-3.13.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:16f15a4eac3bc2d76c45f7ebdd48a65d41b242eb6c31c2245463b40b34584ded", size = 1683220, upload-time = "2025-10-28T20:57:20.241Z" },
- { url = "https://files.pythonhosted.org/packages/71/01/3afe4c96854cfd7b30d78333852e8e851dceaec1c40fd00fec90c6402dd2/aiohttp-3.13.2-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:bb7fb776645af5cc58ab804c58d7eba545a97e047254a52ce89c157b5af6cd0b", size = 1712570, upload-time = "2025-10-28T20:57:22.253Z" },
- { url = "https://files.pythonhosted.org/packages/11/2c/22799d8e720f4697a9e66fd9c02479e40a49de3de2f0bbe7f9f78a987808/aiohttp-3.13.2-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:e1b4951125ec10c70802f2cb09736c895861cd39fd9dcb35107b4dc8ae6220b8", size = 1733407, upload-time = "2025-10-28T20:57:24.37Z" },
- { url = "https://files.pythonhosted.org/packages/34/cb/90f15dd029f07cebbd91f8238a8b363978b530cd128488085b5703683594/aiohttp-3.13.2-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:550bf765101ae721ee1d37d8095f47b1f220650f85fe1af37a90ce75bab89d04", size = 1550093, upload-time = "2025-10-28T20:57:26.257Z" },
- { url = "https://files.pythonhosted.org/packages/69/46/12dce9be9d3303ecbf4d30ad45a7683dc63d90733c2d9fe512be6716cd40/aiohttp-3.13.2-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:fe91b87fc295973096251e2d25a811388e7d8adf3bd2b97ef6ae78bc4ac6c476", size = 1758084, upload-time = "2025-10-28T20:57:28.349Z" },
- { url = "https://files.pythonhosted.org/packages/f9/c8/0932b558da0c302ffd639fc6362a313b98fdf235dc417bc2493da8394df7/aiohttp-3.13.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:e0c8e31cfcc4592cb200160344b2fb6ae0f9e4effe06c644b5a125d4ae5ebe23", size = 1716987, upload-time = "2025-10-28T20:57:30.233Z" },
- { url = "https://files.pythonhosted.org/packages/5d/8b/f5bd1a75003daed099baec373aed678f2e9b34f2ad40d85baa1368556396/aiohttp-3.13.2-cp313-cp313-win32.whl", hash = "sha256:0740f31a60848d6edb296a0df827473eede90c689b8f9f2a4cdde74889eb2254", size = 425859, upload-time = "2025-10-28T20:57:32.105Z" },
- { url = "https://files.pythonhosted.org/packages/5d/28/a8a9fc6957b2cee8902414e41816b5ab5536ecf43c3b1843c10e82c559b2/aiohttp-3.13.2-cp313-cp313-win_amd64.whl", hash = "sha256:a88d13e7ca367394908f8a276b89d04a3652044612b9a408a0bb22a5ed976a1a", size = 452192, upload-time = "2025-10-28T20:57:34.166Z" },
- { url = "https://files.pythonhosted.org/packages/9b/36/e2abae1bd815f01c957cbf7be817b3043304e1c87bad526292a0410fdcf9/aiohttp-3.13.2-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:2475391c29230e063ef53a66669b7b691c9bfc3f1426a0f7bcdf1216bdbac38b", size = 735234, upload-time = "2025-10-28T20:57:36.415Z" },
- { url = "https://files.pythonhosted.org/packages/ca/e3/1ee62dde9b335e4ed41db6bba02613295a0d5b41f74a783c142745a12763/aiohttp-3.13.2-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:f33c8748abef4d8717bb20e8fb1b3e07c6adacb7fd6beaae971a764cf5f30d61", size = 490733, upload-time = "2025-10-28T20:57:38.205Z" },
- { url = "https://files.pythonhosted.org/packages/1a/aa/7a451b1d6a04e8d15a362af3e9b897de71d86feac3babf8894545d08d537/aiohttp-3.13.2-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:ae32f24bbfb7dbb485a24b30b1149e2f200be94777232aeadba3eecece4d0aa4", size = 491303, upload-time = "2025-10-28T20:57:40.122Z" },
- { url = "https://files.pythonhosted.org/packages/57/1e/209958dbb9b01174870f6a7538cd1f3f28274fdbc88a750c238e2c456295/aiohttp-3.13.2-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5d7f02042c1f009ffb70067326ef183a047425bb2ff3bc434ead4dd4a4a66a2b", size = 1717965, upload-time = "2025-10-28T20:57:42.28Z" },
- { url = "https://files.pythonhosted.org/packages/08/aa/6a01848d6432f241416bc4866cae8dc03f05a5a884d2311280f6a09c73d6/aiohttp-3.13.2-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:93655083005d71cd6c072cdab54c886e6570ad2c4592139c3fb967bfc19e4694", size = 1667221, upload-time = "2025-10-28T20:57:44.869Z" },
- { url = "https://files.pythonhosted.org/packages/87/4f/36c1992432d31bbc789fa0b93c768d2e9047ec8c7177e5cd84ea85155f36/aiohttp-3.13.2-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:0db1e24b852f5f664cd728db140cf11ea0e82450471232a394b3d1a540b0f906", size = 1757178, upload-time = "2025-10-28T20:57:47.216Z" },
- { url = "https://files.pythonhosted.org/packages/ac/b4/8e940dfb03b7e0f68a82b88fd182b9be0a65cb3f35612fe38c038c3112cf/aiohttp-3.13.2-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:b009194665bcd128e23eaddef362e745601afa4641930848af4c8559e88f18f9", size = 1838001, upload-time = "2025-10-28T20:57:49.337Z" },
- { url = "https://files.pythonhosted.org/packages/d7/ef/39f3448795499c440ab66084a9db7d20ca7662e94305f175a80f5b7e0072/aiohttp-3.13.2-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c038a8fdc8103cd51dbd986ecdce141473ffd9775a7a8057a6ed9c3653478011", size = 1716325, upload-time = "2025-10-28T20:57:51.327Z" },
- { url = "https://files.pythonhosted.org/packages/d7/51/b311500ffc860b181c05d91c59a1313bdd05c82960fdd4035a15740d431e/aiohttp-3.13.2-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:66bac29b95a00db411cd758fea0e4b9bdba6d549dfe333f9a945430f5f2cc5a6", size = 1547978, upload-time = "2025-10-28T20:57:53.554Z" },
- { url = "https://files.pythonhosted.org/packages/31/64/b9d733296ef79815226dab8c586ff9e3df41c6aff2e16c06697b2d2e6775/aiohttp-3.13.2-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:4ebf9cfc9ba24a74cf0718f04aac2a3bbe745902cc7c5ebc55c0f3b5777ef213", size = 1682042, upload-time = "2025-10-28T20:57:55.617Z" },
- { url = "https://files.pythonhosted.org/packages/3f/30/43d3e0f9d6473a6db7d472104c4eff4417b1e9df01774cb930338806d36b/aiohttp-3.13.2-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:a4b88ebe35ce54205c7074f7302bd08a4cb83256a3e0870c72d6f68a3aaf8e49", size = 1680085, upload-time = "2025-10-28T20:57:57.59Z" },
- { url = "https://files.pythonhosted.org/packages/16/51/c709f352c911b1864cfd1087577760ced64b3e5bee2aa88b8c0c8e2e4972/aiohttp-3.13.2-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:98c4fb90bb82b70a4ed79ca35f656f4281885be076f3f970ce315402b53099ae", size = 1728238, upload-time = "2025-10-28T20:57:59.525Z" },
- { url = "https://files.pythonhosted.org/packages/19/e2/19bd4c547092b773caeb48ff5ae4b1ae86756a0ee76c16727fcfd281404b/aiohttp-3.13.2-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:ec7534e63ae0f3759df3a1ed4fa6bc8f75082a924b590619c0dd2f76d7043caa", size = 1544395, upload-time = "2025-10-28T20:58:01.914Z" },
- { url = "https://files.pythonhosted.org/packages/cf/87/860f2803b27dfc5ed7be532832a3498e4919da61299b4a1f8eb89b8ff44d/aiohttp-3.13.2-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:5b927cf9b935a13e33644cbed6c8c4b2d0f25b713d838743f8fe7191b33829c4", size = 1742965, upload-time = "2025-10-28T20:58:03.972Z" },
- { url = "https://files.pythonhosted.org/packages/67/7f/db2fc7618925e8c7a601094d5cbe539f732df4fb570740be88ed9e40e99a/aiohttp-3.13.2-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:88d6c017966a78c5265d996c19cdb79235be5e6412268d7e2ce7dee339471b7a", size = 1697585, upload-time = "2025-10-28T20:58:06.189Z" },
- { url = "https://files.pythonhosted.org/packages/0c/07/9127916cb09bb38284db5036036042b7b2c514c8ebaeee79da550c43a6d6/aiohttp-3.13.2-cp314-cp314-win32.whl", hash = "sha256:f7c183e786e299b5d6c49fb43a769f8eb8e04a2726a2bd5887b98b5cc2d67940", size = 431621, upload-time = "2025-10-28T20:58:08.636Z" },
- { url = "https://files.pythonhosted.org/packages/fb/41/554a8a380df6d3a2bba8a7726429a23f4ac62aaf38de43bb6d6cde7b4d4d/aiohttp-3.13.2-cp314-cp314-win_amd64.whl", hash = "sha256:fe242cd381e0fb65758faf5ad96c2e460df6ee5b2de1072fe97e4127927e00b4", size = 457627, upload-time = "2025-10-28T20:58:11Z" },
- { url = "https://files.pythonhosted.org/packages/c7/8e/3824ef98c039d3951cb65b9205a96dd2b20f22241ee17d89c5701557c826/aiohttp-3.13.2-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:f10d9c0b0188fe85398c61147bbd2a657d616c876863bfeff43376e0e3134673", size = 767360, upload-time = "2025-10-28T20:58:13.358Z" },
- { url = "https://files.pythonhosted.org/packages/a4/0f/6a03e3fc7595421274fa34122c973bde2d89344f8a881b728fa8c774e4f1/aiohttp-3.13.2-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:e7c952aefdf2460f4ae55c5e9c3e80aa72f706a6317e06020f80e96253b1accd", size = 504616, upload-time = "2025-10-28T20:58:15.339Z" },
- { url = "https://files.pythonhosted.org/packages/c6/aa/ed341b670f1bc8a6f2c6a718353d13b9546e2cef3544f573c6a1ff0da711/aiohttp-3.13.2-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:c20423ce14771d98353d2e25e83591fa75dfa90a3c1848f3d7c68243b4fbded3", size = 509131, upload-time = "2025-10-28T20:58:17.693Z" },
- { url = "https://files.pythonhosted.org/packages/7f/f0/c68dac234189dae5c4bbccc0f96ce0cc16b76632cfc3a08fff180045cfa4/aiohttp-3.13.2-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e96eb1a34396e9430c19d8338d2ec33015e4a87ef2b4449db94c22412e25ccdf", size = 1864168, upload-time = "2025-10-28T20:58:20.113Z" },
- { url = "https://files.pythonhosted.org/packages/8f/65/75a9a76db8364b5d0e52a0c20eabc5d52297385d9af9c35335b924fafdee/aiohttp-3.13.2-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:23fb0783bc1a33640036465019d3bba069942616a6a2353c6907d7fe1ccdaf4e", size = 1719200, upload-time = "2025-10-28T20:58:22.583Z" },
- { url = "https://files.pythonhosted.org/packages/f5/55/8df2ed78d7f41d232f6bd3ff866b6f617026551aa1d07e2f03458f964575/aiohttp-3.13.2-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2e1a9bea6244a1d05a4e57c295d69e159a5c50d8ef16aa390948ee873478d9a5", size = 1843497, upload-time = "2025-10-28T20:58:24.672Z" },
- { url = "https://files.pythonhosted.org/packages/e9/e0/94d7215e405c5a02ccb6a35c7a3a6cfff242f457a00196496935f700cde5/aiohttp-3.13.2-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:0a3d54e822688b56e9f6b5816fb3de3a3a64660efac64e4c2dc435230ad23bad", size = 1935703, upload-time = "2025-10-28T20:58:26.758Z" },
- { url = "https://files.pythonhosted.org/packages/0b/78/1eeb63c3f9b2d1015a4c02788fb543141aad0a03ae3f7a7b669b2483f8d4/aiohttp-3.13.2-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7a653d872afe9f33497215745da7a943d1dc15b728a9c8da1c3ac423af35178e", size = 1792738, upload-time = "2025-10-28T20:58:29.787Z" },
- { url = "https://files.pythonhosted.org/packages/41/75/aaf1eea4c188e51538c04cc568040e3082db263a57086ea74a7d38c39e42/aiohttp-3.13.2-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:56d36e80d2003fa3fc0207fac644216d8532e9504a785ef9a8fd013f84a42c61", size = 1624061, upload-time = "2025-10-28T20:58:32.529Z" },
- { url = "https://files.pythonhosted.org/packages/9b/c2/3b6034de81fbcc43de8aeb209073a2286dfb50b86e927b4efd81cf848197/aiohttp-3.13.2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:78cd586d8331fb8e241c2dd6b2f4061778cc69e150514b39a9e28dd050475661", size = 1789201, upload-time = "2025-10-28T20:58:34.618Z" },
- { url = "https://files.pythonhosted.org/packages/c9/38/c15dcf6d4d890217dae79d7213988f4e5fe6183d43893a9cf2fe9e84ca8d/aiohttp-3.13.2-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:20b10bbfbff766294fe99987f7bb3b74fdd2f1a2905f2562132641ad434dcf98", size = 1776868, upload-time = "2025-10-28T20:58:38.835Z" },
- { url = "https://files.pythonhosted.org/packages/04/75/f74fd178ac81adf4f283a74847807ade5150e48feda6aef024403716c30c/aiohttp-3.13.2-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:9ec49dff7e2b3c85cdeaa412e9d438f0ecd71676fde61ec57027dd392f00c693", size = 1790660, upload-time = "2025-10-28T20:58:41.507Z" },
- { url = "https://files.pythonhosted.org/packages/e7/80/7368bd0d06b16b3aba358c16b919e9c46cf11587dc572091031b0e9e3ef0/aiohttp-3.13.2-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:94f05348c4406450f9d73d38efb41d669ad6cd90c7ee194810d0eefbfa875a7a", size = 1617548, upload-time = "2025-10-28T20:58:43.674Z" },
- { url = "https://files.pythonhosted.org/packages/7d/4b/a6212790c50483cb3212e507378fbe26b5086d73941e1ec4b56a30439688/aiohttp-3.13.2-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:fa4dcb605c6f82a80c7f95713c2b11c3b8e9893b3ebd2bc9bde93165ed6107be", size = 1817240, upload-time = "2025-10-28T20:58:45.787Z" },
- { url = "https://files.pythonhosted.org/packages/ff/f7/ba5f0ba4ea8d8f3c32850912944532b933acbf0f3a75546b89269b9b7dde/aiohttp-3.13.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:cf00e5db968c3f67eccd2778574cf64d8b27d95b237770aa32400bd7a1ca4f6c", size = 1762334, upload-time = "2025-10-28T20:58:47.936Z" },
- { url = "https://files.pythonhosted.org/packages/7e/83/1a5a1856574588b1cad63609ea9ad75b32a8353ac995d830bf5da9357364/aiohttp-3.13.2-cp314-cp314t-win32.whl", hash = "sha256:d23b5fe492b0805a50d3371e8a728a9134d8de5447dce4c885f5587294750734", size = 464685, upload-time = "2025-10-28T20:58:50.642Z" },
- { url = "https://files.pythonhosted.org/packages/9f/4d/d22668674122c08f4d56972297c51a624e64b3ed1efaa40187607a7cb66e/aiohttp-3.13.2-cp314-cp314t-win_amd64.whl", hash = "sha256:ff0a7b0a82a7ab905cbda74006318d1b12e37c797eb1b0d4eb3e316cf47f658f", size = 498093, upload-time = "2025-10-28T20:58:52.782Z" },
+sdist = { url = "https://files.pythonhosted.org/packages/50/42/32cf8e7704ceb4481406eb87161349abb46a57fee3f008ba9cb610968646/aiohttp-3.13.3.tar.gz", hash = "sha256:a949eee43d3782f2daae4f4a2819b2cb9b0c5d3b7f7a927067cc84dafdbb9f88", size = 7844556, upload-time = "2026-01-03T17:33:05.204Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/36/d6/5aec9313ee6ea9c7cde8b891b69f4ff4001416867104580670a31daeba5b/aiohttp-3.13.3-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:d5a372fd5afd301b3a89582817fdcdb6c34124787c70dbcc616f259013e7eef7", size = 738950, upload-time = "2026-01-03T17:29:13.002Z" },
+ { url = "https://files.pythonhosted.org/packages/68/03/8fa90a7e6d11ff20a18837a8e2b5dd23db01aabc475aa9271c8ad33299f5/aiohttp-3.13.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:147e422fd1223005c22b4fe080f5d93ced44460f5f9c105406b753612b587821", size = 496099, upload-time = "2026-01-03T17:29:15.268Z" },
+ { url = "https://files.pythonhosted.org/packages/d2/23/b81f744d402510a8366b74eb420fc0cc1170d0c43daca12d10814df85f10/aiohttp-3.13.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:859bd3f2156e81dd01432f5849fc73e2243d4a487c4fd26609b1299534ee1845", size = 491072, upload-time = "2026-01-03T17:29:16.922Z" },
+ { url = "https://files.pythonhosted.org/packages/d5/e1/56d1d1c0dd334cd203dd97706ce004c1aa24b34a813b0b8daf3383039706/aiohttp-3.13.3-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:dca68018bf48c251ba17c72ed479f4dafe9dbd5a73707ad8d28a38d11f3d42af", size = 1671588, upload-time = "2026-01-03T17:29:18.539Z" },
+ { url = "https://files.pythonhosted.org/packages/5f/34/8d7f962604f4bc2b4e39eb1220dac7d4e4cba91fb9ba0474b4ecd67db165/aiohttp-3.13.3-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:fee0c6bc7db1de362252affec009707a17478a00ec69f797d23ca256e36d5940", size = 1640334, upload-time = "2026-01-03T17:29:21.028Z" },
+ { url = "https://files.pythonhosted.org/packages/94/1d/fcccf2c668d87337ddeef9881537baee13c58d8f01f12ba8a24215f2b804/aiohttp-3.13.3-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c048058117fd649334d81b4b526e94bde3ccaddb20463a815ced6ecbb7d11160", size = 1722656, upload-time = "2026-01-03T17:29:22.531Z" },
+ { url = "https://files.pythonhosted.org/packages/aa/98/c6f3b081c4c606bc1e5f2ec102e87d6411c73a9ef3616fea6f2d5c98c062/aiohttp-3.13.3-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:215a685b6fbbfcf71dfe96e3eba7a6f58f10da1dfdf4889c7dd856abe430dca7", size = 1817625, upload-time = "2026-01-03T17:29:24.276Z" },
+ { url = "https://files.pythonhosted.org/packages/2c/c0/cfcc3d2e11b477f86e1af2863f3858c8850d751ce8dc39c4058a072c9e54/aiohttp-3.13.3-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:de2c184bb1fe2cbd2cefba613e9db29a5ab559323f994b6737e370d3da0ac455", size = 1672604, upload-time = "2026-01-03T17:29:26.099Z" },
+ { url = "https://files.pythonhosted.org/packages/1e/77/6b4ffcbcac4c6a5d041343a756f34a6dd26174ae07f977a64fe028dda5b0/aiohttp-3.13.3-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:75ca857eba4e20ce9f546cd59c7007b33906a4cd48f2ff6ccf1ccfc3b646f279", size = 1554370, upload-time = "2026-01-03T17:29:28.121Z" },
+ { url = "https://files.pythonhosted.org/packages/f2/f0/e3ddfa93f17d689dbe014ba048f18e0c9f9b456033b70e94349a2e9048be/aiohttp-3.13.3-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:81e97251d9298386c2b7dbeb490d3d1badbdc69107fb8c9299dd04eb39bddc0e", size = 1642023, upload-time = "2026-01-03T17:29:30.002Z" },
+ { url = "https://files.pythonhosted.org/packages/eb/45/c14019c9ec60a8e243d06d601b33dcc4fd92379424bde3021725859d7f99/aiohttp-3.13.3-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:c0e2d366af265797506f0283487223146af57815b388623f0357ef7eac9b209d", size = 1649680, upload-time = "2026-01-03T17:29:31.782Z" },
+ { url = "https://files.pythonhosted.org/packages/9c/fd/09c9451dae5aa5c5ed756df95ff9ef549d45d4be663bafd1e4954fd836f0/aiohttp-3.13.3-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:4e239d501f73d6db1522599e14b9b321a7e3b1de66ce33d53a765d975e9f4808", size = 1692407, upload-time = "2026-01-03T17:29:33.392Z" },
+ { url = "https://files.pythonhosted.org/packages/a6/81/938bc2ec33c10efd6637ccb3d22f9f3160d08e8f3aa2587a2c2d5ab578eb/aiohttp-3.13.3-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:0db318f7a6f065d84cb1e02662c526294450b314a02bd9e2a8e67f0d8564ce40", size = 1543047, upload-time = "2026-01-03T17:29:34.855Z" },
+ { url = "https://files.pythonhosted.org/packages/f7/23/80488ee21c8d567c83045e412e1d9b7077d27171591a4eb7822586e8c06a/aiohttp-3.13.3-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:bfc1cc2fe31a6026a8a88e4ecfb98d7f6b1fec150cfd708adbfd1d2f42257c29", size = 1715264, upload-time = "2026-01-03T17:29:36.389Z" },
+ { url = "https://files.pythonhosted.org/packages/e2/83/259a8da6683182768200b368120ab3deff5370bed93880fb9a3a86299f34/aiohttp-3.13.3-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:af71fff7bac6bb7508956696dce8f6eec2bbb045eceb40343944b1ae62b5ef11", size = 1657275, upload-time = "2026-01-03T17:29:38.162Z" },
+ { url = "https://files.pythonhosted.org/packages/3f/4f/2c41f800a0b560785c10fb316216ac058c105f9be50bdc6a285de88db625/aiohttp-3.13.3-cp310-cp310-win32.whl", hash = "sha256:37da61e244d1749798c151421602884db5270faf479cf0ef03af0ff68954c9dd", size = 434053, upload-time = "2026-01-03T17:29:40.074Z" },
+ { url = "https://files.pythonhosted.org/packages/80/df/29cd63c7ecfdb65ccc12f7d808cac4fa2a19544660c06c61a4a48462de0c/aiohttp-3.13.3-cp310-cp310-win_amd64.whl", hash = "sha256:7e63f210bc1b57ef699035f2b4b6d9ce096b5914414a49b0997c839b2bd2223c", size = 456687, upload-time = "2026-01-03T17:29:41.819Z" },
+ { url = "https://files.pythonhosted.org/packages/f1/4c/a164164834f03924d9a29dc3acd9e7ee58f95857e0b467f6d04298594ebb/aiohttp-3.13.3-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:5b6073099fb654e0a068ae678b10feff95c5cae95bbfcbfa7af669d361a8aa6b", size = 746051, upload-time = "2026-01-03T17:29:43.287Z" },
+ { url = "https://files.pythonhosted.org/packages/82/71/d5c31390d18d4f58115037c432b7e0348c60f6f53b727cad33172144a112/aiohttp-3.13.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1cb93e166e6c28716c8c6aeb5f99dfb6d5ccf482d29fe9bf9a794110e6d0ab64", size = 499234, upload-time = "2026-01-03T17:29:44.822Z" },
+ { url = "https://files.pythonhosted.org/packages/0e/c9/741f8ac91e14b1d2e7100690425a5b2b919a87a5075406582991fb7de920/aiohttp-3.13.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:28e027cf2f6b641693a09f631759b4d9ce9165099d2b5d92af9bd4e197690eea", size = 494979, upload-time = "2026-01-03T17:29:46.405Z" },
+ { url = "https://files.pythonhosted.org/packages/75/b5/31d4d2e802dfd59f74ed47eba48869c1c21552c586d5e81a9d0d5c2ad640/aiohttp-3.13.3-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3b61b7169ababd7802f9568ed96142616a9118dd2be0d1866e920e77ec8fa92a", size = 1748297, upload-time = "2026-01-03T17:29:48.083Z" },
+ { url = "https://files.pythonhosted.org/packages/1a/3e/eefad0ad42959f226bb79664826883f2687d602a9ae2941a18e0484a74d3/aiohttp-3.13.3-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:80dd4c21b0f6237676449c6baaa1039abae86b91636b6c91a7f8e61c87f89540", size = 1707172, upload-time = "2026-01-03T17:29:49.648Z" },
+ { url = "https://files.pythonhosted.org/packages/c5/3a/54a64299fac2891c346cdcf2aa6803f994a2e4beeaf2e5a09dcc54acc842/aiohttp-3.13.3-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:65d2ccb7eabee90ce0503c17716fc77226be026dcc3e65cce859a30db715025b", size = 1805405, upload-time = "2026-01-03T17:29:51.244Z" },
+ { url = "https://files.pythonhosted.org/packages/6c/70/ddc1b7169cf64075e864f64595a14b147a895a868394a48f6a8031979038/aiohttp-3.13.3-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5b179331a481cb5529fca8b432d8d3c7001cb217513c94cd72d668d1248688a3", size = 1899449, upload-time = "2026-01-03T17:29:53.938Z" },
+ { url = "https://files.pythonhosted.org/packages/a1/7e/6815aab7d3a56610891c76ef79095677b8b5be6646aaf00f69b221765021/aiohttp-3.13.3-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9d4c940f02f49483b18b079d1c27ab948721852b281f8b015c058100e9421dd1", size = 1748444, upload-time = "2026-01-03T17:29:55.484Z" },
+ { url = "https://files.pythonhosted.org/packages/6b/f2/073b145c4100da5511f457dc0f7558e99b2987cf72600d42b559db856fbc/aiohttp-3.13.3-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:f9444f105664c4ce47a2a7171a2418bce5b7bae45fb610f4e2c36045d85911d3", size = 1606038, upload-time = "2026-01-03T17:29:57.179Z" },
+ { url = "https://files.pythonhosted.org/packages/0a/c1/778d011920cae03ae01424ec202c513dc69243cf2db303965615b81deeea/aiohttp-3.13.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:694976222c711d1d00ba131904beb60534f93966562f64440d0c9d41b8cdb440", size = 1724156, upload-time = "2026-01-03T17:29:58.914Z" },
+ { url = "https://files.pythonhosted.org/packages/0e/cb/3419eabf4ec1e9ec6f242c32b689248365a1cf621891f6f0386632525494/aiohttp-3.13.3-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:f33ed1a2bf1997a36661874b017f5c4b760f41266341af36febaf271d179f6d7", size = 1722340, upload-time = "2026-01-03T17:30:01.962Z" },
+ { url = "https://files.pythonhosted.org/packages/7a/e5/76cf77bdbc435bf233c1f114edad39ed4177ccbfab7c329482b179cff4f4/aiohttp-3.13.3-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:e636b3c5f61da31a92bf0d91da83e58fdfa96f178ba682f11d24f31944cdd28c", size = 1783041, upload-time = "2026-01-03T17:30:03.609Z" },
+ { url = "https://files.pythonhosted.org/packages/9d/d4/dd1ca234c794fd29c057ce8c0566b8ef7fd6a51069de5f06fa84b9a1971c/aiohttp-3.13.3-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:5d2d94f1f5fcbe40838ac51a6ab5704a6f9ea42e72ceda48de5e6b898521da51", size = 1596024, upload-time = "2026-01-03T17:30:05.132Z" },
+ { url = "https://files.pythonhosted.org/packages/55/58/4345b5f26661a6180afa686c473620c30a66afdf120ed3dd545bbc809e85/aiohttp-3.13.3-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:2be0e9ccf23e8a94f6f0650ce06042cefc6ac703d0d7ab6c7a917289f2539ad4", size = 1804590, upload-time = "2026-01-03T17:30:07.135Z" },
+ { url = "https://files.pythonhosted.org/packages/7b/06/05950619af6c2df7e0a431d889ba2813c9f0129cec76f663e547a5ad56f2/aiohttp-3.13.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:9af5e68ee47d6534d36791bbe9b646d2a7c7deb6fc24d7943628edfbb3581f29", size = 1740355, upload-time = "2026-01-03T17:30:09.083Z" },
+ { url = "https://files.pythonhosted.org/packages/3e/80/958f16de79ba0422d7c1e284b2abd0c84bc03394fbe631d0a39ffa10e1eb/aiohttp-3.13.3-cp311-cp311-win32.whl", hash = "sha256:a2212ad43c0833a873d0fb3c63fa1bacedd4cf6af2fee62bf4b739ceec3ab239", size = 433701, upload-time = "2026-01-03T17:30:10.869Z" },
+ { url = "https://files.pythonhosted.org/packages/dc/f2/27cdf04c9851712d6c1b99df6821a6623c3c9e55956d4b1e318c337b5a48/aiohttp-3.13.3-cp311-cp311-win_amd64.whl", hash = "sha256:642f752c3eb117b105acbd87e2c143de710987e09860d674e068c4c2c441034f", size = 457678, upload-time = "2026-01-03T17:30:12.719Z" },
+ { url = "https://files.pythonhosted.org/packages/a0/be/4fc11f202955a69e0db803a12a062b8379c970c7c84f4882b6da17337cc1/aiohttp-3.13.3-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:b903a4dfee7d347e2d87697d0713be59e0b87925be030c9178c5faa58ea58d5c", size = 739732, upload-time = "2026-01-03T17:30:14.23Z" },
+ { url = "https://files.pythonhosted.org/packages/97/2c/621d5b851f94fa0bb7430d6089b3aa970a9d9b75196bc93bb624b0db237a/aiohttp-3.13.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:a45530014d7a1e09f4a55f4f43097ba0fd155089372e105e4bff4ca76cb1b168", size = 494293, upload-time = "2026-01-03T17:30:15.96Z" },
+ { url = "https://files.pythonhosted.org/packages/5d/43/4be01406b78e1be8320bb8316dc9c42dbab553d281c40364e0f862d5661c/aiohttp-3.13.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:27234ef6d85c914f9efeb77ff616dbf4ad2380be0cda40b4db086ffc7ddd1b7d", size = 493533, upload-time = "2026-01-03T17:30:17.431Z" },
+ { url = "https://files.pythonhosted.org/packages/8d/a8/5a35dc56a06a2c90d4742cbf35294396907027f80eea696637945a106f25/aiohttp-3.13.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d32764c6c9aafb7fb55366a224756387cd50bfa720f32b88e0e6fa45b27dcf29", size = 1737839, upload-time = "2026-01-03T17:30:19.422Z" },
+ { url = "https://files.pythonhosted.org/packages/bf/62/4b9eeb331da56530bf2e198a297e5303e1c1ebdceeb00fe9b568a65c5a0c/aiohttp-3.13.3-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:b1a6102b4d3ebc07dad44fbf07b45bb600300f15b552ddf1851b5390202ea2e3", size = 1703932, upload-time = "2026-01-03T17:30:21.756Z" },
+ { url = "https://files.pythonhosted.org/packages/7c/f6/af16887b5d419e6a367095994c0b1332d154f647e7dc2bd50e61876e8e3d/aiohttp-3.13.3-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c014c7ea7fb775dd015b2d3137378b7be0249a448a1612268b5a90c2d81de04d", size = 1771906, upload-time = "2026-01-03T17:30:23.932Z" },
+ { url = "https://files.pythonhosted.org/packages/ce/83/397c634b1bcc24292fa1e0c7822800f9f6569e32934bdeef09dae7992dfb/aiohttp-3.13.3-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:2b8d8ddba8f95ba17582226f80e2de99c7a7948e66490ef8d947e272a93e9463", size = 1871020, upload-time = "2026-01-03T17:30:26Z" },
+ { url = "https://files.pythonhosted.org/packages/86/f6/a62cbbf13f0ac80a70f71b1672feba90fdb21fd7abd8dbf25c0105fb6fa3/aiohttp-3.13.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9ae8dd55c8e6c4257eae3a20fd2c8f41edaea5992ed67156642493b8daf3cecc", size = 1755181, upload-time = "2026-01-03T17:30:27.554Z" },
+ { url = "https://files.pythonhosted.org/packages/0a/87/20a35ad487efdd3fba93d5843efdfaa62d2f1479eaafa7453398a44faf13/aiohttp-3.13.3-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:01ad2529d4b5035578f5081606a465f3b814c542882804e2e8cda61adf5c71bf", size = 1561794, upload-time = "2026-01-03T17:30:29.254Z" },
+ { url = "https://files.pythonhosted.org/packages/de/95/8fd69a66682012f6716e1bc09ef8a1a2a91922c5725cb904689f112309c4/aiohttp-3.13.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:bb4f7475e359992b580559e008c598091c45b5088f28614e855e42d39c2f1033", size = 1697900, upload-time = "2026-01-03T17:30:31.033Z" },
+ { url = "https://files.pythonhosted.org/packages/e5/66/7b94b3b5ba70e955ff597672dad1691333080e37f50280178967aff68657/aiohttp-3.13.3-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:c19b90316ad3b24c69cd78d5c9b4f3aa4497643685901185b65166293d36a00f", size = 1728239, upload-time = "2026-01-03T17:30:32.703Z" },
+ { url = "https://files.pythonhosted.org/packages/47/71/6f72f77f9f7d74719692ab65a2a0252584bf8d5f301e2ecb4c0da734530a/aiohttp-3.13.3-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:96d604498a7c782cb15a51c406acaea70d8c027ee6b90c569baa6e7b93073679", size = 1740527, upload-time = "2026-01-03T17:30:34.695Z" },
+ { url = "https://files.pythonhosted.org/packages/fa/b4/75ec16cbbd5c01bdaf4a05b19e103e78d7ce1ef7c80867eb0ace42ff4488/aiohttp-3.13.3-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:084911a532763e9d3dd95adf78a78f4096cd5f58cdc18e6fdbc1b58417a45423", size = 1554489, upload-time = "2026-01-03T17:30:36.864Z" },
+ { url = "https://files.pythonhosted.org/packages/52/8f/bc518c0eea29f8406dcf7ed1f96c9b48e3bc3995a96159b3fc11f9e08321/aiohttp-3.13.3-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:7a4a94eb787e606d0a09404b9c38c113d3b099d508021faa615d70a0131907ce", size = 1767852, upload-time = "2026-01-03T17:30:39.433Z" },
+ { url = "https://files.pythonhosted.org/packages/9d/f2/a07a75173124f31f11ea6f863dc44e6f09afe2bca45dd4e64979490deab1/aiohttp-3.13.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:87797e645d9d8e222e04160ee32aa06bc5c163e8499f24db719e7852ec23093a", size = 1722379, upload-time = "2026-01-03T17:30:41.081Z" },
+ { url = "https://files.pythonhosted.org/packages/3c/4a/1a3fee7c21350cac78e5c5cef711bac1b94feca07399f3d406972e2d8fcd/aiohttp-3.13.3-cp312-cp312-win32.whl", hash = "sha256:b04be762396457bef43f3597c991e192ee7da460a4953d7e647ee4b1c28e7046", size = 428253, upload-time = "2026-01-03T17:30:42.644Z" },
+ { url = "https://files.pythonhosted.org/packages/d9/b7/76175c7cb4eb73d91ad63c34e29fc4f77c9386bba4a65b53ba8e05ee3c39/aiohttp-3.13.3-cp312-cp312-win_amd64.whl", hash = "sha256:e3531d63d3bdfa7e3ac5e9b27b2dd7ec9df3206a98e0b3445fa906f233264c57", size = 455407, upload-time = "2026-01-03T17:30:44.195Z" },
+ { url = "https://files.pythonhosted.org/packages/97/8a/12ca489246ca1faaf5432844adbfce7ff2cc4997733e0af120869345643a/aiohttp-3.13.3-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:5dff64413671b0d3e7d5918ea490bdccb97a4ad29b3f311ed423200b2203e01c", size = 734190, upload-time = "2026-01-03T17:30:45.832Z" },
+ { url = "https://files.pythonhosted.org/packages/32/08/de43984c74ed1fca5c014808963cc83cb00d7bb06af228f132d33862ca76/aiohttp-3.13.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:87b9aab6d6ed88235aa2970294f496ff1a1f9adcd724d800e9b952395a80ffd9", size = 491783, upload-time = "2026-01-03T17:30:47.466Z" },
+ { url = "https://files.pythonhosted.org/packages/17/f8/8dd2cf6112a5a76f81f81a5130c57ca829d101ad583ce57f889179accdda/aiohttp-3.13.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:425c126c0dc43861e22cb1c14ba4c8e45d09516d0a3ae0a3f7494b79f5f233a3", size = 490704, upload-time = "2026-01-03T17:30:49.373Z" },
+ { url = "https://files.pythonhosted.org/packages/6d/40/a46b03ca03936f832bc7eaa47cfbb1ad012ba1be4790122ee4f4f8cba074/aiohttp-3.13.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:7f9120f7093c2a32d9647abcaf21e6ad275b4fbec5b55969f978b1a97c7c86bf", size = 1720652, upload-time = "2026-01-03T17:30:50.974Z" },
+ { url = "https://files.pythonhosted.org/packages/f7/7e/917fe18e3607af92657e4285498f500dca797ff8c918bd7d90b05abf6c2a/aiohttp-3.13.3-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:697753042d57f4bf7122cab985bf15d0cef23c770864580f5af4f52023a56bd6", size = 1692014, upload-time = "2026-01-03T17:30:52.729Z" },
+ { url = "https://files.pythonhosted.org/packages/71/b6/cefa4cbc00d315d68973b671cf105b21a609c12b82d52e5d0c9ae61d2a09/aiohttp-3.13.3-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:6de499a1a44e7de70735d0b39f67c8f25eb3d91eb3103be99ca0fa882cdd987d", size = 1759777, upload-time = "2026-01-03T17:30:54.537Z" },
+ { url = "https://files.pythonhosted.org/packages/fb/e3/e06ee07b45e59e6d81498b591fc589629be1553abb2a82ce33efe2a7b068/aiohttp-3.13.3-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:37239e9f9a7ea9ac5bf6b92b0260b01f8a22281996da609206a84df860bc1261", size = 1861276, upload-time = "2026-01-03T17:30:56.512Z" },
+ { url = "https://files.pythonhosted.org/packages/7c/24/75d274228acf35ceeb2850b8ce04de9dd7355ff7a0b49d607ee60c29c518/aiohttp-3.13.3-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f76c1e3fe7d7c8afad7ed193f89a292e1999608170dcc9751a7462a87dfd5bc0", size = 1743131, upload-time = "2026-01-03T17:30:58.256Z" },
+ { url = "https://files.pythonhosted.org/packages/04/98/3d21dde21889b17ca2eea54fdcff21b27b93f45b7bb94ca029c31ab59dc3/aiohttp-3.13.3-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:fc290605db2a917f6e81b0e1e0796469871f5af381ce15c604a3c5c7e51cb730", size = 1556863, upload-time = "2026-01-03T17:31:00.445Z" },
+ { url = "https://files.pythonhosted.org/packages/9e/84/da0c3ab1192eaf64782b03971ab4055b475d0db07b17eff925e8c93b3aa5/aiohttp-3.13.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:4021b51936308aeea0367b8f006dc999ca02bc118a0cc78c303f50a2ff6afb91", size = 1682793, upload-time = "2026-01-03T17:31:03.024Z" },
+ { url = "https://files.pythonhosted.org/packages/ff/0f/5802ada182f575afa02cbd0ec5180d7e13a402afb7c2c03a9aa5e5d49060/aiohttp-3.13.3-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:49a03727c1bba9a97d3e93c9f93ca03a57300f484b6e935463099841261195d3", size = 1716676, upload-time = "2026-01-03T17:31:04.842Z" },
+ { url = "https://files.pythonhosted.org/packages/3f/8c/714d53bd8b5a4560667f7bbbb06b20c2382f9c7847d198370ec6526af39c/aiohttp-3.13.3-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:3d9908a48eb7416dc1f4524e69f1d32e5d90e3981e4e37eb0aa1cd18f9cfa2a4", size = 1733217, upload-time = "2026-01-03T17:31:06.868Z" },
+ { url = "https://files.pythonhosted.org/packages/7d/79/e2176f46d2e963facea939f5be2d26368ce543622be6f00a12844d3c991f/aiohttp-3.13.3-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:2712039939ec963c237286113c68dbad80a82a4281543f3abf766d9d73228998", size = 1552303, upload-time = "2026-01-03T17:31:08.958Z" },
+ { url = "https://files.pythonhosted.org/packages/ab/6a/28ed4dea1759916090587d1fe57087b03e6c784a642b85ef48217b0277ae/aiohttp-3.13.3-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:7bfdc049127717581866fa4708791220970ce291c23e28ccf3922c700740fdc0", size = 1763673, upload-time = "2026-01-03T17:31:10.676Z" },
+ { url = "https://files.pythonhosted.org/packages/e8/35/4a3daeb8b9fab49240d21c04d50732313295e4bd813a465d840236dd0ce1/aiohttp-3.13.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:8057c98e0c8472d8846b9c79f56766bcc57e3e8ac7bfd510482332366c56c591", size = 1721120, upload-time = "2026-01-03T17:31:12.575Z" },
+ { url = "https://files.pythonhosted.org/packages/bc/9f/d643bb3c5fb99547323e635e251c609fbbc660d983144cfebec529e09264/aiohttp-3.13.3-cp313-cp313-win32.whl", hash = "sha256:1449ceddcdbcf2e0446957863af03ebaaa03f94c090f945411b61269e2cb5daf", size = 427383, upload-time = "2026-01-03T17:31:14.382Z" },
+ { url = "https://files.pythonhosted.org/packages/4e/f1/ab0395f8a79933577cdd996dd2f9aa6014af9535f65dddcf88204682fe62/aiohttp-3.13.3-cp313-cp313-win_amd64.whl", hash = "sha256:693781c45a4033d31d4187d2436f5ac701e7bbfe5df40d917736108c1cc7436e", size = 453899, upload-time = "2026-01-03T17:31:15.958Z" },
+ { url = "https://files.pythonhosted.org/packages/99/36/5b6514a9f5d66f4e2597e40dea2e3db271e023eb7a5d22defe96ba560996/aiohttp-3.13.3-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:ea37047c6b367fd4bd632bff8077449b8fa034b69e812a18e0132a00fae6e808", size = 737238, upload-time = "2026-01-03T17:31:17.909Z" },
+ { url = "https://files.pythonhosted.org/packages/f7/49/459327f0d5bcd8c6c9ca69e60fdeebc3622861e696490d8674a6d0cb90a6/aiohttp-3.13.3-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:6fc0e2337d1a4c3e6acafda6a78a39d4c14caea625124817420abceed36e2415", size = 492292, upload-time = "2026-01-03T17:31:19.919Z" },
+ { url = "https://files.pythonhosted.org/packages/e8/0b/b97660c5fd05d3495b4eb27f2d0ef18dc1dc4eff7511a9bf371397ff0264/aiohttp-3.13.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:c685f2d80bb67ca8c3837823ad76196b3694b0159d232206d1e461d3d434666f", size = 493021, upload-time = "2026-01-03T17:31:21.636Z" },
+ { url = "https://files.pythonhosted.org/packages/54/d4/438efabdf74e30aeceb890c3290bbaa449780583b1270b00661126b8aae4/aiohttp-3.13.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:48e377758516d262bde50c2584fc6c578af272559c409eecbdd2bae1601184d6", size = 1717263, upload-time = "2026-01-03T17:31:23.296Z" },
+ { url = "https://files.pythonhosted.org/packages/71/f2/7bddc7fd612367d1459c5bcf598a9e8f7092d6580d98de0e057eb42697ad/aiohttp-3.13.3-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:34749271508078b261c4abb1767d42b8d0c0cc9449c73a4df494777dc55f0687", size = 1669107, upload-time = "2026-01-03T17:31:25.334Z" },
+ { url = "https://files.pythonhosted.org/packages/00/5a/1aeaecca40e22560f97610a329e0e5efef5e0b5afdf9f857f0d93839ab2e/aiohttp-3.13.3-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:82611aeec80eb144416956ec85b6ca45a64d76429c1ed46ae1b5f86c6e0c9a26", size = 1760196, upload-time = "2026-01-03T17:31:27.394Z" },
+ { url = "https://files.pythonhosted.org/packages/f8/f8/0ff6992bea7bd560fc510ea1c815f87eedd745fe035589c71ce05612a19a/aiohttp-3.13.3-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:2fff83cfc93f18f215896e3a190e8e5cb413ce01553901aca925176e7568963a", size = 1843591, upload-time = "2026-01-03T17:31:29.238Z" },
+ { url = "https://files.pythonhosted.org/packages/e3/d1/e30e537a15f53485b61f5be525f2157da719819e8377298502aebac45536/aiohttp-3.13.3-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bbe7d4cecacb439e2e2a8a1a7b935c25b812af7a5fd26503a66dadf428e79ec1", size = 1720277, upload-time = "2026-01-03T17:31:31.053Z" },
+ { url = "https://files.pythonhosted.org/packages/84/45/23f4c451d8192f553d38d838831ebbc156907ea6e05557f39563101b7717/aiohttp-3.13.3-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:b928f30fe49574253644b1ca44b1b8adbd903aa0da4b9054a6c20fc7f4092a25", size = 1548575, upload-time = "2026-01-03T17:31:32.87Z" },
+ { url = "https://files.pythonhosted.org/packages/6a/ed/0a42b127a43712eda7807e7892c083eadfaf8429ca8fb619662a530a3aab/aiohttp-3.13.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:7b5e8fe4de30df199155baaf64f2fcd604f4c678ed20910db8e2c66dc4b11603", size = 1679455, upload-time = "2026-01-03T17:31:34.76Z" },
+ { url = "https://files.pythonhosted.org/packages/2e/b5/c05f0c2b4b4fe2c9d55e73b6d3ed4fd6c9dc2684b1d81cbdf77e7fad9adb/aiohttp-3.13.3-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:8542f41a62bcc58fc7f11cf7c90e0ec324ce44950003feb70640fc2a9092c32a", size = 1687417, upload-time = "2026-01-03T17:31:36.699Z" },
+ { url = "https://files.pythonhosted.org/packages/c9/6b/915bc5dad66aef602b9e459b5a973529304d4e89ca86999d9d75d80cbd0b/aiohttp-3.13.3-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:5e1d8c8b8f1d91cd08d8f4a3c2b067bfca6ec043d3ff36de0f3a715feeedf926", size = 1729968, upload-time = "2026-01-03T17:31:38.622Z" },
+ { url = "https://files.pythonhosted.org/packages/11/3b/e84581290a9520024a08640b63d07673057aec5ca548177a82026187ba73/aiohttp-3.13.3-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:90455115e5da1c3c51ab619ac57f877da8fd6d73c05aacd125c5ae9819582aba", size = 1545690, upload-time = "2026-01-03T17:31:40.57Z" },
+ { url = "https://files.pythonhosted.org/packages/f5/04/0c3655a566c43fd647c81b895dfe361b9f9ad6d58c19309d45cff52d6c3b/aiohttp-3.13.3-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:042e9e0bcb5fba81886c8b4fbb9a09d6b8a00245fd8d88e4d989c1f96c74164c", size = 1746390, upload-time = "2026-01-03T17:31:42.857Z" },
+ { url = "https://files.pythonhosted.org/packages/1f/53/71165b26978f719c3419381514c9690bd5980e764a09440a10bb816ea4ab/aiohttp-3.13.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:2eb752b102b12a76ca02dff751a801f028b4ffbbc478840b473597fc91a9ed43", size = 1702188, upload-time = "2026-01-03T17:31:44.984Z" },
+ { url = "https://files.pythonhosted.org/packages/29/a7/cbe6c9e8e136314fa1980da388a59d2f35f35395948a08b6747baebb6aa6/aiohttp-3.13.3-cp314-cp314-win32.whl", hash = "sha256:b556c85915d8efaed322bf1bdae9486aa0f3f764195a0fb6ee962e5c71ef5ce1", size = 433126, upload-time = "2026-01-03T17:31:47.463Z" },
+ { url = "https://files.pythonhosted.org/packages/de/56/982704adea7d3b16614fc5936014e9af85c0e34b58f9046655817f04306e/aiohttp-3.13.3-cp314-cp314-win_amd64.whl", hash = "sha256:9bf9f7a65e7aa20dd764151fb3d616c81088f91f8df39c3893a536e279b4b984", size = 459128, upload-time = "2026-01-03T17:31:49.2Z" },
+ { url = "https://files.pythonhosted.org/packages/6c/2a/3c79b638a9c3d4658d345339d22070241ea341ed4e07b5ac60fb0f418003/aiohttp-3.13.3-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:05861afbbec40650d8a07ea324367cb93e9e8cc7762e04dd4405df99fa65159c", size = 769512, upload-time = "2026-01-03T17:31:51.134Z" },
+ { url = "https://files.pythonhosted.org/packages/29/b9/3e5014d46c0ab0db8707e0ac2711ed28c4da0218c358a4e7c17bae0d8722/aiohttp-3.13.3-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:2fc82186fadc4a8316768d61f3722c230e2c1dcab4200d52d2ebdf2482e47592", size = 506444, upload-time = "2026-01-03T17:31:52.85Z" },
+ { url = "https://files.pythonhosted.org/packages/90/03/c1d4ef9a054e151cd7839cdc497f2638f00b93cbe8043983986630d7a80c/aiohttp-3.13.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:0add0900ff220d1d5c5ebbf99ed88b0c1bbf87aa7e4262300ed1376a6b13414f", size = 510798, upload-time = "2026-01-03T17:31:54.91Z" },
+ { url = "https://files.pythonhosted.org/packages/ea/76/8c1e5abbfe8e127c893fe7ead569148a4d5a799f7cf958d8c09f3eedf097/aiohttp-3.13.3-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:568f416a4072fbfae453dcf9a99194bbb8bdeab718e08ee13dfa2ba0e4bebf29", size = 1868835, upload-time = "2026-01-03T17:31:56.733Z" },
+ { url = "https://files.pythonhosted.org/packages/8e/ac/984c5a6f74c363b01ff97adc96a3976d9c98940b8969a1881575b279ac5d/aiohttp-3.13.3-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:add1da70de90a2569c5e15249ff76a631ccacfe198375eead4aadf3b8dc849dc", size = 1720486, upload-time = "2026-01-03T17:31:58.65Z" },
+ { url = "https://files.pythonhosted.org/packages/b2/9a/b7039c5f099c4eb632138728828b33428585031a1e658d693d41d07d89d1/aiohttp-3.13.3-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:10b47b7ba335d2e9b1239fa571131a87e2d8ec96b333e68b2a305e7a98b0bae2", size = 1847951, upload-time = "2026-01-03T17:32:00.989Z" },
+ { url = "https://files.pythonhosted.org/packages/3c/02/3bec2b9a1ba3c19ff89a43a19324202b8eb187ca1e928d8bdac9bbdddebd/aiohttp-3.13.3-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:3dd4dce1c718e38081c8f35f323209d4c1df7d4db4bab1b5c88a6b4d12b74587", size = 1941001, upload-time = "2026-01-03T17:32:03.122Z" },
+ { url = "https://files.pythonhosted.org/packages/37/df/d879401cedeef27ac4717f6426c8c36c3091c6e9f08a9178cc87549c537f/aiohttp-3.13.3-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:34bac00a67a812570d4a460447e1e9e06fae622946955f939051e7cc895cfab8", size = 1797246, upload-time = "2026-01-03T17:32:05.255Z" },
+ { url = "https://files.pythonhosted.org/packages/8d/15/be122de1f67e6953add23335c8ece6d314ab67c8bebb3f181063010795a7/aiohttp-3.13.3-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a19884d2ee70b06d9204b2727a7b9f983d0c684c650254679e716b0b77920632", size = 1627131, upload-time = "2026-01-03T17:32:07.607Z" },
+ { url = "https://files.pythonhosted.org/packages/12/12/70eedcac9134cfa3219ab7af31ea56bc877395b1ac30d65b1bc4b27d0438/aiohttp-3.13.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:5f8ca7f2bb6ba8348a3614c7918cc4bb73268c5ac2a207576b7afea19d3d9f64", size = 1795196, upload-time = "2026-01-03T17:32:09.59Z" },
+ { url = "https://files.pythonhosted.org/packages/32/11/b30e1b1cd1f3054af86ebe60df96989c6a414dd87e27ad16950eee420bea/aiohttp-3.13.3-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:b0d95340658b9d2f11d9697f59b3814a9d3bb4b7a7c20b131df4bcef464037c0", size = 1782841, upload-time = "2026-01-03T17:32:11.445Z" },
+ { url = "https://files.pythonhosted.org/packages/88/0d/d98a9367b38912384a17e287850f5695c528cff0f14f791ce8ee2e4f7796/aiohttp-3.13.3-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:a1e53262fd202e4b40b70c3aff944a8155059beedc8a89bba9dc1f9ef06a1b56", size = 1795193, upload-time = "2026-01-03T17:32:13.705Z" },
+ { url = "https://files.pythonhosted.org/packages/43/a5/a2dfd1f5ff5581632c7f6a30e1744deda03808974f94f6534241ef60c751/aiohttp-3.13.3-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:d60ac9663f44168038586cab2157e122e46bdef09e9368b37f2d82d354c23f72", size = 1621979, upload-time = "2026-01-03T17:32:15.965Z" },
+ { url = "https://files.pythonhosted.org/packages/fa/f0/12973c382ae7c1cccbc4417e129c5bf54c374dfb85af70893646e1f0e749/aiohttp-3.13.3-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:90751b8eed69435bac9ff4e3d2f6b3af1f57e37ecb0fbeee59c0174c9e2d41df", size = 1822193, upload-time = "2026-01-03T17:32:18.219Z" },
+ { url = "https://files.pythonhosted.org/packages/3c/5f/24155e30ba7f8c96918af1350eb0663e2430aad9e001c0489d89cd708ab1/aiohttp-3.13.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:fc353029f176fd2b3ec6cfc71be166aba1936fe5d73dd1992ce289ca6647a9aa", size = 1769801, upload-time = "2026-01-03T17:32:20.25Z" },
+ { url = "https://files.pythonhosted.org/packages/eb/f8/7314031ff5c10e6ece114da79b338ec17eeff3a079e53151f7e9f43c4723/aiohttp-3.13.3-cp314-cp314t-win32.whl", hash = "sha256:2e41b18a58da1e474a057b3d35248d8320029f61d70a37629535b16a0c8f3767", size = 466523, upload-time = "2026-01-03T17:32:22.215Z" },
+ { url = "https://files.pythonhosted.org/packages/b4/63/278a98c715ae467624eafe375542d8ba9b4383a016df8fdefe0ae28382a7/aiohttp-3.13.3-cp314-cp314t-win_amd64.whl", hash = "sha256:44531a36aa2264a1860089ffd4dce7baf875ee5a6079d5fb42e261c704ef7344", size = 499694, upload-time = "2026-01-03T17:32:24.546Z" },
]
[[package]]
@@ -274,37 +274,37 @@ wheels = [
[[package]]
name = "apache-tvm-ffi"
-version = "0.1.4"
+version = "0.1.7"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "typing-extensions" },
]
-sdist = { url = "https://files.pythonhosted.org/packages/3c/60/3282f2ba28fbd1e60dabfa244e61b40a237ccf76986772879d17d9cc5102/apache_tvm_ffi-0.1.4.tar.gz", hash = "sha256:1a6e635b671e962bbc3bf1bc97bfd82e4c0f1bedf27c8d183bb282664974d0d3", size = 1325200, upload-time = "2025-11-30T07:21:43.769Z" }
-wheels = [
- { url = "https://files.pythonhosted.org/packages/f5/cc/07d0866a8fa0e0d5318e3dea9e91ecd0edc48037c06457429a374c82f60d/apache_tvm_ffi-0.1.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:a476833cae2cd645ccd8baa6ec2b078508b1b3fff7a6147d8944b47aea0ca819", size = 1778693, upload-time = "2025-11-30T07:20:50.888Z" },
- { url = "https://files.pythonhosted.org/packages/af/20/312131abbc2e461f20f98f283de6385ea936f31a7b780e8f04c3b233c52a/apache_tvm_ffi-0.1.4-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:644af2c407a5ee489617a43717933c80a4fd72657bf510151c8ca964eb465ebc", size = 1937693, upload-time = "2025-11-30T07:20:52.644Z" },
- { url = "https://files.pythonhosted.org/packages/33/f8/36793837f70f1416e19a8cab2d18ba85e9e20aff9788b9c98d8b189b8385/apache_tvm_ffi-0.1.4-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:977a893e748f90e7a93f68deba8bba53091bce27a6c40b23ea67719789da026f", size = 2008256, upload-time = "2025-11-30T07:20:54.145Z" },
- { url = "https://files.pythonhosted.org/packages/fe/b3/4eb57df470985fd7448858246a75bae10d90c804cc5b520be8312b8cc381/apache_tvm_ffi-0.1.4-cp310-cp310-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f2cbf41984f21489c786f7ffc406abd3bd0e8b5f2eb7fd8e2dbe601b525ddcb8", size = 1884868, upload-time = "2025-11-30T07:20:55.734Z" },
- { url = "https://files.pythonhosted.org/packages/72/e7/6a8374bc8250071872ec5d896d5d5bc4b10aee8c3ff6e81e8df64cca648b/apache_tvm_ffi-0.1.4-cp310-cp310-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:99fe507b5458afc0607134fe7139ed012546fe13c8640a19732f233946d250d0", size = 1992614, upload-time = "2025-11-30T07:20:57.062Z" },
- { url = "https://files.pythonhosted.org/packages/61/ed/0ecee1013d0c19b4cccb6a04b598c1cb0474da1815b3499567f5d8c378dd/apache_tvm_ffi-0.1.4-cp310-cp310-win_amd64.whl", hash = "sha256:ee075b5ad79d0b19fdd47a0ce2560655e831f41305fa7c7ac5eba1e2224b0991", size = 1753999, upload-time = "2025-11-30T07:20:59.033Z" },
- { url = "https://files.pythonhosted.org/packages/cc/45/a161d1df747703f0b73bc68f8e0238aae356597d892b934237c8dad6ed31/apache_tvm_ffi-0.1.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:aa79114b54bfd904d895dd47c656f7983a67df8aa5527220d266f55b5559040e", size = 1778166, upload-time = "2025-11-30T07:21:00.583Z" },
- { url = "https://files.pythonhosted.org/packages/8c/df/aeed7255b979433da5e54b4e63efe5f44b8b8a6b9bf8067d9ef2013ab715/apache_tvm_ffi-0.1.4-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:2f095e2ef5c8fd4f0e1648e926a22b31b3295f24adb2ad96b7bf40af7b85f16b", size = 1937601, upload-time = "2025-11-30T07:21:01.848Z" },
- { url = "https://files.pythonhosted.org/packages/3b/a5/c35b9bc746d8d1bbdae94efc05ad69c677d7576c890bd90ce0f6bdbfd715/apache_tvm_ffi-0.1.4-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:636e50e2189d5f3b2e459d69b83375d1ac5736baa483c095110402e9f20bd877", size = 2007762, upload-time = "2025-11-30T07:21:03.435Z" },
- { url = "https://files.pythonhosted.org/packages/f2/7d/2c05ab09659c5f3b6d7017c6f533ea44a33686d8c32a350a24c8432a611a/apache_tvm_ffi-0.1.4-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:98863410a256bcaa157de44269f542eff14307be44c860c63cb6bd9f685ec050", size = 1885016, upload-time = "2025-11-30T07:21:04.715Z" },
- { url = "https://files.pythonhosted.org/packages/5e/7e/78463597793725ef6b238c6914062537b0087bbcb752622b932e2d82f6c7/apache_tvm_ffi-0.1.4-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5066e9405e6443b1e48cc791c0f17231cc13d857fef5ddd43fae0affe2bd0079", size = 1992920, upload-time = "2025-11-30T07:21:06.119Z" },
- { url = "https://files.pythonhosted.org/packages/82/aa/0f4ecf10d53e079d2a87a3f14bb94dc6df66e8fcc231c08b73007b8399b5/apache_tvm_ffi-0.1.4-cp311-cp311-win_amd64.whl", hash = "sha256:238798f1fa438d3396f68c767bc7f126d3054a323e7e304e6a4f3c7fac96cc70", size = 1753910, upload-time = "2025-11-30T07:21:07.7Z" },
- { url = "https://files.pythonhosted.org/packages/95/05/2cfdeac3c5c8468f313d4262c9253e5661904447d9801a95d2e71cb666c2/apache_tvm_ffi-0.1.4-cp312-abi3-macosx_11_0_arm64.whl", hash = "sha256:28e182e21010c7554b90fa4a43e101dffd3962de6ff0ebf71eee09a6e909034f", size = 1754878, upload-time = "2025-11-30T07:21:09.096Z" },
- { url = "https://files.pythonhosted.org/packages/f8/b4/27b932ebb2ccfa6f26048a472de87c9b9860d3fd36cc4ec627900c8c42dd/apache_tvm_ffi-0.1.4-cp312-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:a5842c53c1618e2266e6fbbb723fe823136fa0510448682e6990bb119042dee8", size = 1910199, upload-time = "2025-11-30T07:21:10.819Z" },
- { url = "https://files.pythonhosted.org/packages/6f/5f/deba40bc0d799b3282ad2b70843cade9da45242002ffef945409ca50063f/apache_tvm_ffi-0.1.4-cp312-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:0ef5b108285aa80e37e2510bf60c4fd6a2a53a382fb4b1fc0da9a2fa8d93c14b", size = 1984480, upload-time = "2025-11-30T07:21:12.463Z" },
- { url = "https://files.pythonhosted.org/packages/7f/f6/4c200bae5136497710bc912414a4661d5a0316ae87cd796255cfd47c1bc5/apache_tvm_ffi-0.1.4-cp312-abi3-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ce70e2763fc4ba160230cecd5e16d0cfde8d95330fd7f0ef82e2fa88aee28103", size = 1857622, upload-time = "2025-11-30T07:21:13.927Z" },
- { url = "https://files.pythonhosted.org/packages/1e/b7/eb2371395e41c1a60418affbecaea2fc9324e908eea1abcac1cce60097cd/apache_tvm_ffi-0.1.4-cp312-abi3-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:92ed50530973203b6860178927943689a7189af4c8a054ef6c4197678aa698ef", size = 1966156, upload-time = "2025-11-30T07:21:15.416Z" },
- { url = "https://files.pythonhosted.org/packages/29/71/12f7f25bb1ac90d91e09d6d825a5816846b1ae70ff9c3aae5721de92c37f/apache_tvm_ffi-0.1.4-cp312-abi3-win_amd64.whl", hash = "sha256:442bd6d80eafefbf455cd845684c454f472598e20afcc9e42c3ddfbe7eb6b136", size = 1734522, upload-time = "2025-11-30T07:21:16.711Z" },
- { url = "https://files.pythonhosted.org/packages/64/ad/26f0e1d5229a406c41bb4ad9dd941d19ab681ff30c65efb8e08538f0f232/apache_tvm_ffi-0.1.4-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:12d92917748ee437ec3f6a67a602c4f60149b1b4f40558950b7a04477d623b5a", size = 1785538, upload-time = "2025-11-30T07:21:18.135Z" },
- { url = "https://files.pythonhosted.org/packages/d8/b9/642b1ae04af630e70e2ac9b84ed2f33cbdd5e2661eb63aebb32782664f4b/apache_tvm_ffi-0.1.4-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:5b95979ecf950a9147786c936c99fa119ea2493e2fd642ce10ac6ef22e857d47", size = 1923008, upload-time = "2025-11-30T07:21:19.44Z" },
- { url = "https://files.pythonhosted.org/packages/32/d3/f27d99099d0977306f39e79607b69072d4806ce075da43d173a751e0ff19/apache_tvm_ffi-0.1.4-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:08d4accc0d1aa4bde4f81d55e37c4cfa449d23b4f2871c87c24a0ac0eb93fe9e", size = 1991960, upload-time = "2025-11-30T07:21:21.022Z" },
- { url = "https://files.pythonhosted.org/packages/bc/e3/befab77a55638394246bc0804cdb2d36222d78cf1ed9d4486b23a06ad2e3/apache_tvm_ffi-0.1.4-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e8f09299294b702c6cd381d098330db83590c2faf981342954711fdd02659c34", size = 1869760, upload-time = "2025-11-30T07:21:22.34Z" },
- { url = "https://files.pythonhosted.org/packages/1b/b3/2920bf4fee6db0086a6bad438371fc3d68fb625891f0130fe205063f3c6d/apache_tvm_ffi-0.1.4-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d3a2c45c74ddc85c8d55dd927deae10fbaadaebcb1943ce6052cdb2223b056f5", size = 1976158, upload-time = "2025-11-30T07:21:23.949Z" },
- { url = "https://files.pythonhosted.org/packages/ae/69/fe387b0f70ed608a363a90036e08ef8c1e844e5c98145502160661012dc0/apache_tvm_ffi-0.1.4-cp314-cp314t-win_amd64.whl", hash = "sha256:1bceda57240d03a3cf026334521c0595d097ab92b6d0df7485cbb37b2c056c27", size = 1794928, upload-time = "2025-11-30T07:21:25.234Z" },
+sdist = { url = "https://files.pythonhosted.org/packages/3d/07/6fbc8fbef1d04bd290f2dcdb3091ae784ac526b62649ec52993a41c65f72/apache_tvm_ffi-0.1.7.tar.gz", hash = "sha256:737cd4a067d6c6c7ad7dd909a0708eb3dc28540299039ea636f8ff5766b122be", size = 2397940, upload-time = "2025-12-28T09:13:25.52Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/3b/00/e6c7e0710344ccfb2a42be68e04dfd1920864c25bab4a7411a48a4809a1a/apache_tvm_ffi-0.1.7-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:cc6334f55ad8b4cb3c084dcdf33720b47665d0ea488c36a1b4f1b99445ae5a12", size = 1816700, upload-time = "2025-12-28T09:12:22.223Z" },
+ { url = "https://files.pythonhosted.org/packages/84/68/82799768095fe83640f0def07eda01891c9d713a9db8770316ca460a6114/apache_tvm_ffi-0.1.7-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:f69f1195ad7701b0a024a84914b934487a30d5975a9e5d5044c57eb9f9b0fcf7", size = 1976292, upload-time = "2025-12-28T09:12:24.623Z" },
+ { url = "https://files.pythonhosted.org/packages/8a/ab/0c01ac5c3d545c04d1adf03a154f8167dc5884c0fdcbb519714107426028/apache_tvm_ffi-0.1.7-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:7b6444a322279cc33ada0bb2a0482e3433c31028becda106dcb0d48c30fb2de0", size = 2048671, upload-time = "2025-12-28T09:12:26.457Z" },
+ { url = "https://files.pythonhosted.org/packages/0a/e3/449fcdbe7ebd8df4b830399171fb325e7f77b2babe958c6fa6c537281e26/apache_tvm_ffi-0.1.7-cp310-cp310-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d5e9e668620ba3b78b1c1f393dee67a63850882b0713dba31972c5f854f02860", size = 1920010, upload-time = "2025-12-28T09:12:27.81Z" },
+ { url = "https://files.pythonhosted.org/packages/a2/98/737ffc4576af7d4da97f3c73bf347f69d269497cfe9ac089517af5900919/apache_tvm_ffi-0.1.7-cp310-cp310-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5f7deaa48cfd720949dd1638dfbd4cc7d5285008c7f3f342887e2bf33cf1f5be", size = 2030727, upload-time = "2025-12-28T09:12:29.38Z" },
+ { url = "https://files.pythonhosted.org/packages/f1/36/8ea373c1758c812a504a856a06fc08d8761df1c0e2515e6867c22168fea7/apache_tvm_ffi-0.1.7-cp310-cp310-win_amd64.whl", hash = "sha256:c1fd70f6e7578eeec5e5d8ed0fb814b12280b724531487ff4d899edddd188d97", size = 1787864, upload-time = "2025-12-28T09:12:31.194Z" },
+ { url = "https://files.pythonhosted.org/packages/0a/e7/33ece51ba1670fa77a1897745720b9c8bdac854acb0e09d45e64340948f4/apache_tvm_ffi-0.1.7-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:20a8847f4609f1fe61015b7547bced99eba38072ed422799fc7bd15371d6d83c", size = 1818328, upload-time = "2025-12-28T09:12:32.784Z" },
+ { url = "https://files.pythonhosted.org/packages/8f/b9/3bb4099a82b4c7198823b67067a3d206ec8a0b32204a559c5cca1bee54bd/apache_tvm_ffi-0.1.7-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:f0e010e61d1f220ec4ce3d15053db3f8c8d9c79230ea763343fc5e4acf53ef17", size = 1975412, upload-time = "2025-12-28T09:12:34.737Z" },
+ { url = "https://files.pythonhosted.org/packages/48/53/423788fb9b26460b3d7ceb8588d172dfe7ae4abcc335931fcbf08a859904/apache_tvm_ffi-0.1.7-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:9b05155b4b60ebd3642213d0489b6ef24aff17b268960dbb5f106a39899bb8b1", size = 2047974, upload-time = "2025-12-28T09:12:36.296Z" },
+ { url = "https://files.pythonhosted.org/packages/a6/30/45d4acf7f99e1fc79a8663f2111901b8031e1f9b316860af7acf4859c964/apache_tvm_ffi-0.1.7-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:cceaddc7636060231aca4ada2632814189b1169224b2b451f41984145ef615fc", size = 1919697, upload-time = "2025-12-28T09:12:38.15Z" },
+ { url = "https://files.pythonhosted.org/packages/dd/bb/fa5042076bf6e7daaf9774389f99149c1851434fc0d8e4cb34aa0c4a3810/apache_tvm_ffi-0.1.7-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5769cadc42e70522e2a523f1dfe24f48dbe3bf384e63f95df251f9d572ffcf23", size = 2030760, upload-time = "2025-12-28T09:12:39.813Z" },
+ { url = "https://files.pythonhosted.org/packages/fe/74/fd06e97699e9cbf36d887c5fbbc56b14e896e2652bbe1781ab84cef82a40/apache_tvm_ffi-0.1.7-cp311-cp311-win_amd64.whl", hash = "sha256:b5c7716429ce2beb0a5b00c5a3bdd90b8a5891838afb782491c576ade42ba7c4", size = 1788026, upload-time = "2025-12-28T09:12:42.142Z" },
+ { url = "https://files.pythonhosted.org/packages/26/4e/43a41ac023a5989803952d527dfea6e63da71fe223f6e010d4ec71ca0526/apache_tvm_ffi-0.1.7-cp312-abi3-macosx_11_0_arm64.whl", hash = "sha256:12950ca9f9f4f4436869afe17845a6bfc85cbcd8a15dfa2b16095f7e6f49d06f", size = 1790152, upload-time = "2025-12-28T09:12:43.975Z" },
+ { url = "https://files.pythonhosted.org/packages/b9/d3/05ba0a63baba1e3aec0f6303c4bc567493fb1c070d9f298f929a7703c0fb/apache_tvm_ffi-0.1.7-cp312-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:d0e579234ce6fb2899377335a881ecf15d0197d833e2d370c9269ea6ca578f6f", size = 1947362, upload-time = "2025-12-28T09:12:45.921Z" },
+ { url = "https://files.pythonhosted.org/packages/f1/11/b69df7685d75144fd9f57e5155cdf4ff91d6617a9f8b89b1415204863da0/apache_tvm_ffi-0.1.7-cp312-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:258a4aecc16e963def8ba0ab07f585147c7e7f586156b9496bfdf34af229443d", size = 2024240, upload-time = "2025-12-28T09:12:47.337Z" },
+ { url = "https://files.pythonhosted.org/packages/cf/b6/31459f4141ea8621377fecac7c29e1568d494cbf95c5aa1ddf2cbc12a8ff/apache_tvm_ffi-0.1.7-cp312-abi3-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:363701589349e11a945dabce026578203bd83cb8de71af9a066beadd77af085a", size = 1891485, upload-time = "2025-12-28T09:12:49.171Z" },
+ { url = "https://files.pythonhosted.org/packages/a5/4d/d21874eda6e3ea59c5a84aa010b24b84617e3b286ad759ac5eadccb1a88c/apache_tvm_ffi-0.1.7-cp312-abi3-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:fbbf87df625930bafbd979c2c510d5bd989e9171098e5bb65320d0e7336d0095", size = 2003196, upload-time = "2025-12-28T09:12:50.891Z" },
+ { url = "https://files.pythonhosted.org/packages/3f/d4/37102d96e359386107f5ce3751c4e2a8c1b8df3d34f65b701810ba59465c/apache_tvm_ffi-0.1.7-cp312-abi3-win_amd64.whl", hash = "sha256:d2fb56f53e33c7ddf7d6d340d44cbc440d205f7dab4bc5ed1ad20c8fc779250f", size = 1768697, upload-time = "2025-12-28T09:12:52.394Z" },
+ { url = "https://files.pythonhosted.org/packages/92/c3/aa4b950032251c24b9db7d725b86d7d683b62d9919f8a32f478c28951dc3/apache_tvm_ffi-0.1.7-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:dc4a02e0252599d0c4eb2d2fa91b7756f0446b3bc42479b05c140e9d336b9b8b", size = 1820520, upload-time = "2025-12-28T09:12:54.29Z" },
+ { url = "https://files.pythonhosted.org/packages/19/70/55ee17b8a340ef8ffc0d6c0587ff5a0c7e7c85a94e6cb202e682838a42c7/apache_tvm_ffi-0.1.7-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:41e50f2c8d98d706923c70ac19fd5f605bf71b8ffa43c0c2e9e1e22c2d60d4e0", size = 1960686, upload-time = "2025-12-28T09:12:56.206Z" },
+ { url = "https://files.pythonhosted.org/packages/b6/0f/ca4f7b4836e1e03386b6e486a0ba88812644723a96965a01e2072f551f2e/apache_tvm_ffi-0.1.7-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:835bd391c6f3388e84e36f0ea2347761992241a3953be6ebb319bf1c2ac855d8", size = 2032237, upload-time = "2025-12-28T09:12:58.113Z" },
+ { url = "https://files.pythonhosted.org/packages/89/b6/35be0035f8ed9e10ae6d9ffb7e91397ba381eb734f85ff852efe56eb3012/apache_tvm_ffi-0.1.7-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d7d8b53e94c2bc28e961934e8291a9763d7868f84f9759cbae462b77ca801e5b", size = 1904414, upload-time = "2025-12-28T09:12:59.624Z" },
+ { url = "https://files.pythonhosted.org/packages/5a/5f/1f57863c2c68389d1453fe147d89da22910a0e4f645a8be29cc8f461850f/apache_tvm_ffi-0.1.7-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e135b70c7be8627661c5ec4a466e17e1aba260ffd7c6bccfe231c9ea975875e7", size = 2013039, upload-time = "2025-12-28T09:13:01.37Z" },
+ { url = "https://files.pythonhosted.org/packages/bb/3f/08d1931c6ebca557051176d400e15c1d7f6cf9096fc02f8c90ac7ee309ac/apache_tvm_ffi-0.1.7-cp314-cp314t-win_amd64.whl", hash = "sha256:408bb2c1fa585260afd556e53d65e2735f201f358202fda2b07d08a6cbfaf91f", size = 1828344, upload-time = "2025-12-28T09:13:03.359Z" },
]
[[package]]
@@ -686,11 +686,11 @@ sdist = { url = "https://files.pythonhosted.org/packages/64/cb/104778c728dc3d5ea
[[package]]
name = "certifi"
-version = "2025.11.12"
+version = "2026.1.4"
source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/a2/8c/58f469717fa48465e4a50c014a0400602d3c437d7c0c468e17ada824da3a/certifi-2025.11.12.tar.gz", hash = "sha256:d8ab5478f2ecd78af242878415affce761ca6bc54a22a27e026d7c25357c3316", size = 160538, upload-time = "2025-11-12T02:54:51.517Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/e0/2d/a891ca51311197f6ad14a7ef42e2399f36cf2f9bd44752b3dc4eab60fdc5/certifi-2026.1.4.tar.gz", hash = "sha256:ac726dd470482006e014ad384921ed6438c457018f4b3d204aea4281258b2120", size = 154268, upload-time = "2026-01-04T02:42:41.825Z" }
wheels = [
- { url = "https://files.pythonhosted.org/packages/70/7d/9bc192684cea499815ff478dfcdc13835ddf401365057044fb721ec6bddb/certifi-2025.11.12-py3-none-any.whl", hash = "sha256:97de8790030bbd5c2d96b7ec782fc2f7820ef8dba6db909ccf95449f2d062d4b", size = 159438, upload-time = "2025-11-12T02:54:49.735Z" },
+ { url = "https://files.pythonhosted.org/packages/e6/ad/3cc14f097111b4de0040c83a525973216457bbeeb63739ef1ed275c1c021/certifi-2026.1.4-py3-none-any.whl", hash = "sha256:9943707519e4add1115f44c2bc244f782c0249876bf51b6599fee1ffbedd685c", size = 152900, upload-time = "2026-01-04T02:42:40.15Z" },
]
[[package]]
@@ -905,101 +905,101 @@ wheels = [
[[package]]
name = "coverage"
-version = "7.12.0"
-source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/89/26/4a96807b193b011588099c3b5c89fbb05294e5b90e71018e065465f34eb6/coverage-7.12.0.tar.gz", hash = "sha256:fc11e0a4e372cb5f282f16ef90d4a585034050ccda536451901abfb19a57f40c", size = 819341, upload-time = "2025-11-18T13:34:20.766Z" }
-wheels = [
- { url = "https://files.pythonhosted.org/packages/26/4a/0dc3de1c172d35abe512332cfdcc43211b6ebce629e4cc42e6cd25ed8f4d/coverage-7.12.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:32b75c2ba3f324ee37af3ccee5b30458038c50b349ad9b88cee85096132a575b", size = 217409, upload-time = "2025-11-18T13:31:53.122Z" },
- { url = "https://files.pythonhosted.org/packages/01/c3/086198b98db0109ad4f84241e8e9ea7e5fb2db8c8ffb787162d40c26cc76/coverage-7.12.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:cb2a1b6ab9fe833714a483a915de350abc624a37149649297624c8d57add089c", size = 217927, upload-time = "2025-11-18T13:31:54.458Z" },
- { url = "https://files.pythonhosted.org/packages/5d/5f/34614dbf5ce0420828fc6c6f915126a0fcb01e25d16cf141bf5361e6aea6/coverage-7.12.0-cp310-cp310-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:5734b5d913c3755e72f70bf6cc37a0518d4f4745cde760c5d8e12005e62f9832", size = 244678, upload-time = "2025-11-18T13:31:55.805Z" },
- { url = "https://files.pythonhosted.org/packages/55/7b/6b26fb32e8e4a6989ac1d40c4e132b14556131493b1d06bc0f2be169c357/coverage-7.12.0-cp310-cp310-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:b527a08cdf15753279b7afb2339a12073620b761d79b81cbe2cdebdb43d90daa", size = 246507, upload-time = "2025-11-18T13:31:57.05Z" },
- { url = "https://files.pythonhosted.org/packages/06/42/7d70e6603d3260199b90fb48b537ca29ac183d524a65cc31366b2e905fad/coverage-7.12.0-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9bb44c889fb68004e94cab71f6a021ec83eac9aeabdbb5a5a88821ec46e1da73", size = 248366, upload-time = "2025-11-18T13:31:58.362Z" },
- { url = "https://files.pythonhosted.org/packages/2d/4a/d86b837923878424c72458c5b25e899a3c5ca73e663082a915f5b3c4d749/coverage-7.12.0-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:4b59b501455535e2e5dde5881739897967b272ba25988c89145c12d772810ccb", size = 245366, upload-time = "2025-11-18T13:31:59.572Z" },
- { url = "https://files.pythonhosted.org/packages/e6/c2/2adec557e0aa9721875f06ced19730fdb7fc58e31b02b5aa56f2ebe4944d/coverage-7.12.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:d8842f17095b9868a05837b7b1b73495293091bed870e099521ada176aa3e00e", size = 246408, upload-time = "2025-11-18T13:32:00.784Z" },
- { url = "https://files.pythonhosted.org/packages/5a/4b/8bd1f1148260df11c618e535fdccd1e5aaf646e55b50759006a4f41d8a26/coverage-7.12.0-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:c5a6f20bf48b8866095c6820641e7ffbe23f2ac84a2efc218d91235e404c7777", size = 244416, upload-time = "2025-11-18T13:32:01.963Z" },
- { url = "https://files.pythonhosted.org/packages/0e/13/3a248dd6a83df90414c54a4e121fd081fb20602ca43955fbe1d60e2312a9/coverage-7.12.0-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:5f3738279524e988d9da2893f307c2093815c623f8d05a8f79e3eff3a7a9e553", size = 244681, upload-time = "2025-11-18T13:32:03.408Z" },
- { url = "https://files.pythonhosted.org/packages/76/30/aa833827465a5e8c938935f5d91ba055f70516941078a703740aaf1aa41f/coverage-7.12.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:e0d68c1f7eabbc8abe582d11fa393ea483caf4f44b0af86881174769f185c94d", size = 245300, upload-time = "2025-11-18T13:32:04.686Z" },
- { url = "https://files.pythonhosted.org/packages/38/24/f85b3843af1370fb3739fa7571819b71243daa311289b31214fe3e8c9d68/coverage-7.12.0-cp310-cp310-win32.whl", hash = "sha256:7670d860e18b1e3ee5930b17a7d55ae6287ec6e55d9799982aa103a2cc1fa2ef", size = 220008, upload-time = "2025-11-18T13:32:05.806Z" },
- { url = "https://files.pythonhosted.org/packages/3a/a2/c7da5b9566f7164db9eefa133d17761ecb2c2fde9385d754e5b5c80f710d/coverage-7.12.0-cp310-cp310-win_amd64.whl", hash = "sha256:f999813dddeb2a56aab5841e687b68169da0d3f6fc78ccf50952fa2463746022", size = 220943, upload-time = "2025-11-18T13:32:07.166Z" },
- { url = "https://files.pythonhosted.org/packages/5a/0c/0dfe7f0487477d96432e4815537263363fb6dd7289743a796e8e51eabdf2/coverage-7.12.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:aa124a3683d2af98bd9d9c2bfa7a5076ca7e5ab09fdb96b81fa7d89376ae928f", size = 217535, upload-time = "2025-11-18T13:32:08.812Z" },
- { url = "https://files.pythonhosted.org/packages/9b/f5/f9a4a053a5bbff023d3bec259faac8f11a1e5a6479c2ccf586f910d8dac7/coverage-7.12.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:d93fbf446c31c0140208dcd07c5d882029832e8ed7891a39d6d44bd65f2316c3", size = 218044, upload-time = "2025-11-18T13:32:10.329Z" },
- { url = "https://files.pythonhosted.org/packages/95/c5/84fc3697c1fa10cd8571919bf9693f693b7373278daaf3b73e328d502bc8/coverage-7.12.0-cp311-cp311-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:52ca620260bd8cd6027317bdd8b8ba929be1d741764ee765b42c4d79a408601e", size = 248440, upload-time = "2025-11-18T13:32:12.536Z" },
- { url = "https://files.pythonhosted.org/packages/f4/36/2d93fbf6a04670f3874aed397d5a5371948a076e3249244a9e84fb0e02d6/coverage-7.12.0-cp311-cp311-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:f3433ffd541380f3a0e423cff0f4926d55b0cc8c1d160fdc3be24a4c03aa65f7", size = 250361, upload-time = "2025-11-18T13:32:13.852Z" },
- { url = "https://files.pythonhosted.org/packages/5d/49/66dc65cc456a6bfc41ea3d0758c4afeaa4068a2b2931bf83be6894cf1058/coverage-7.12.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f7bbb321d4adc9f65e402c677cd1c8e4c2d0105d3ce285b51b4d87f1d5db5245", size = 252472, upload-time = "2025-11-18T13:32:15.068Z" },
- { url = "https://files.pythonhosted.org/packages/35/1f/ebb8a18dffd406db9fcd4b3ae42254aedcaf612470e8712f12041325930f/coverage-7.12.0-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:22a7aade354a72dff3b59c577bfd18d6945c61f97393bc5fb7bd293a4237024b", size = 248592, upload-time = "2025-11-18T13:32:16.328Z" },
- { url = "https://files.pythonhosted.org/packages/da/a8/67f213c06e5ea3b3d4980df7dc344d7fea88240b5fe878a5dcbdfe0e2315/coverage-7.12.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:3ff651dcd36d2fea66877cd4a82de478004c59b849945446acb5baf9379a1b64", size = 250167, upload-time = "2025-11-18T13:32:17.687Z" },
- { url = "https://files.pythonhosted.org/packages/f0/00/e52aef68154164ea40cc8389c120c314c747fe63a04b013a5782e989b77f/coverage-7.12.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:31b8b2e38391a56e3cea39d22a23faaa7c3fc911751756ef6d2621d2a9daf742", size = 248238, upload-time = "2025-11-18T13:32:19.2Z" },
- { url = "https://files.pythonhosted.org/packages/1f/a4/4d88750bcf9d6d66f77865e5a05a20e14db44074c25fd22519777cb69025/coverage-7.12.0-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:297bc2da28440f5ae51c845a47c8175a4db0553a53827886e4fb25c66633000c", size = 247964, upload-time = "2025-11-18T13:32:21.027Z" },
- { url = "https://files.pythonhosted.org/packages/a7/6b/b74693158899d5b47b0bf6238d2c6722e20ba749f86b74454fac0696bb00/coverage-7.12.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:6ff7651cc01a246908eac162a6a86fc0dbab6de1ad165dfb9a1e2ec660b44984", size = 248862, upload-time = "2025-11-18T13:32:22.304Z" },
- { url = "https://files.pythonhosted.org/packages/18/de/6af6730227ce0e8ade307b1cc4a08e7f51b419a78d02083a86c04ccceb29/coverage-7.12.0-cp311-cp311-win32.whl", hash = "sha256:313672140638b6ddb2c6455ddeda41c6a0b208298034544cfca138978c6baed6", size = 220033, upload-time = "2025-11-18T13:32:23.714Z" },
- { url = "https://files.pythonhosted.org/packages/e2/a1/e7f63021a7c4fe20994359fcdeae43cbef4a4d0ca36a5a1639feeea5d9e1/coverage-7.12.0-cp311-cp311-win_amd64.whl", hash = "sha256:a1783ed5bd0d5938d4435014626568dc7f93e3cb99bc59188cc18857c47aa3c4", size = 220966, upload-time = "2025-11-18T13:32:25.599Z" },
- { url = "https://files.pythonhosted.org/packages/77/e8/deae26453f37c20c3aa0c4433a1e32cdc169bf415cce223a693117aa3ddd/coverage-7.12.0-cp311-cp311-win_arm64.whl", hash = "sha256:4648158fd8dd9381b5847622df1c90ff314efbfc1df4550092ab6013c238a5fc", size = 219637, upload-time = "2025-11-18T13:32:27.265Z" },
- { url = "https://files.pythonhosted.org/packages/02/bf/638c0427c0f0d47638242e2438127f3c8ee3cfc06c7fdeb16778ed47f836/coverage-7.12.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:29644c928772c78512b48e14156b81255000dcfd4817574ff69def189bcb3647", size = 217704, upload-time = "2025-11-18T13:32:28.906Z" },
- { url = "https://files.pythonhosted.org/packages/08/e1/706fae6692a66c2d6b871a608bbde0da6281903fa0e9f53a39ed441da36a/coverage-7.12.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:8638cbb002eaa5d7c8d04da667813ce1067080b9a91099801a0053086e52b736", size = 218064, upload-time = "2025-11-18T13:32:30.161Z" },
- { url = "https://files.pythonhosted.org/packages/a9/8b/eb0231d0540f8af3ffda39720ff43cb91926489d01524e68f60e961366e4/coverage-7.12.0-cp312-cp312-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:083631eeff5eb9992c923e14b810a179798bb598e6a0dd60586819fc23be6e60", size = 249560, upload-time = "2025-11-18T13:32:31.835Z" },
- { url = "https://files.pythonhosted.org/packages/e9/a1/67fb52af642e974d159b5b379e4d4c59d0ebe1288677fbd04bbffe665a82/coverage-7.12.0-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:99d5415c73ca12d558e07776bd957c4222c687b9f1d26fa0e1b57e3598bdcde8", size = 252318, upload-time = "2025-11-18T13:32:33.178Z" },
- { url = "https://files.pythonhosted.org/packages/41/e5/38228f31b2c7665ebf9bdfdddd7a184d56450755c7e43ac721c11a4b8dab/coverage-7.12.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e949ebf60c717c3df63adb4a1a366c096c8d7fd8472608cd09359e1bd48ef59f", size = 253403, upload-time = "2025-11-18T13:32:34.45Z" },
- { url = "https://files.pythonhosted.org/packages/ec/4b/df78e4c8188f9960684267c5a4897836f3f0f20a20c51606ee778a1d9749/coverage-7.12.0-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:6d907ddccbca819afa2cd014bc69983b146cca2735a0b1e6259b2a6c10be1e70", size = 249984, upload-time = "2025-11-18T13:32:35.747Z" },
- { url = "https://files.pythonhosted.org/packages/ba/51/bb163933d195a345c6f63eab9e55743413d064c291b6220df754075c2769/coverage-7.12.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:b1518ecbad4e6173f4c6e6c4a46e49555ea5679bf3feda5edb1b935c7c44e8a0", size = 251339, upload-time = "2025-11-18T13:32:37.352Z" },
- { url = "https://files.pythonhosted.org/packages/15/40/c9b29cdb8412c837cdcbc2cfa054547dd83affe6cbbd4ce4fdb92b6ba7d1/coverage-7.12.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:51777647a749abdf6f6fd8c7cffab12de68ab93aab15efc72fbbb83036c2a068", size = 249489, upload-time = "2025-11-18T13:32:39.212Z" },
- { url = "https://files.pythonhosted.org/packages/c8/da/b3131e20ba07a0de4437a50ef3b47840dfabf9293675b0cd5c2c7f66dd61/coverage-7.12.0-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:42435d46d6461a3b305cdfcad7cdd3248787771f53fe18305548cba474e6523b", size = 249070, upload-time = "2025-11-18T13:32:40.598Z" },
- { url = "https://files.pythonhosted.org/packages/70/81/b653329b5f6302c08d683ceff6785bc60a34be9ae92a5c7b63ee7ee7acec/coverage-7.12.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:5bcead88c8423e1855e64b8057d0544e33e4080b95b240c2a355334bb7ced937", size = 250929, upload-time = "2025-11-18T13:32:42.915Z" },
- { url = "https://files.pythonhosted.org/packages/a3/00/250ac3bca9f252a5fb1338b5ad01331ebb7b40223f72bef5b1b2cb03aa64/coverage-7.12.0-cp312-cp312-win32.whl", hash = "sha256:dcbb630ab034e86d2a0f79aefd2be07e583202f41e037602d438c80044957baa", size = 220241, upload-time = "2025-11-18T13:32:44.665Z" },
- { url = "https://files.pythonhosted.org/packages/64/1c/77e79e76d37ce83302f6c21980b45e09f8aa4551965213a10e62d71ce0ab/coverage-7.12.0-cp312-cp312-win_amd64.whl", hash = "sha256:2fd8354ed5d69775ac42986a691fbf68b4084278710cee9d7c3eaa0c28fa982a", size = 221051, upload-time = "2025-11-18T13:32:46.008Z" },
- { url = "https://files.pythonhosted.org/packages/31/f5/641b8a25baae564f9e52cac0e2667b123de961985709a004e287ee7663cc/coverage-7.12.0-cp312-cp312-win_arm64.whl", hash = "sha256:737c3814903be30695b2de20d22bcc5428fdae305c61ba44cdc8b3252984c49c", size = 219692, upload-time = "2025-11-18T13:32:47.372Z" },
- { url = "https://files.pythonhosted.org/packages/b8/14/771700b4048774e48d2c54ed0c674273702713c9ee7acdfede40c2666747/coverage-7.12.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:47324fffca8d8eae7e185b5bb20c14645f23350f870c1649003618ea91a78941", size = 217725, upload-time = "2025-11-18T13:32:49.22Z" },
- { url = "https://files.pythonhosted.org/packages/17/a7/3aa4144d3bcb719bf67b22d2d51c2d577bf801498c13cb08f64173e80497/coverage-7.12.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:ccf3b2ede91decd2fb53ec73c1f949c3e034129d1e0b07798ff1d02ea0c8fa4a", size = 218098, upload-time = "2025-11-18T13:32:50.78Z" },
- { url = "https://files.pythonhosted.org/packages/fc/9c/b846bbc774ff81091a12a10203e70562c91ae71badda00c5ae5b613527b1/coverage-7.12.0-cp313-cp313-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:b365adc70a6936c6b0582dc38746b33b2454148c02349345412c6e743efb646d", size = 249093, upload-time = "2025-11-18T13:32:52.554Z" },
- { url = "https://files.pythonhosted.org/packages/76/b6/67d7c0e1f400b32c883e9342de4a8c2ae7c1a0b57c5de87622b7262e2309/coverage-7.12.0-cp313-cp313-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:bc13baf85cd8a4cfcf4a35c7bc9d795837ad809775f782f697bf630b7e200211", size = 251686, upload-time = "2025-11-18T13:32:54.862Z" },
- { url = "https://files.pythonhosted.org/packages/cc/75/b095bd4b39d49c3be4bffbb3135fea18a99a431c52dd7513637c0762fecb/coverage-7.12.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:099d11698385d572ceafb3288a5b80fe1fc58bf665b3f9d362389de488361d3d", size = 252930, upload-time = "2025-11-18T13:32:56.417Z" },
- { url = "https://files.pythonhosted.org/packages/6e/f3/466f63015c7c80550bead3093aacabf5380c1220a2a93c35d374cae8f762/coverage-7.12.0-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:473dc45d69694069adb7680c405fb1e81f60b2aff42c81e2f2c3feaf544d878c", size = 249296, upload-time = "2025-11-18T13:32:58.074Z" },
- { url = "https://files.pythonhosted.org/packages/27/86/eba2209bf2b7e28c68698fc13437519a295b2d228ba9e0ec91673e09fa92/coverage-7.12.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:583f9adbefd278e9de33c33d6846aa8f5d164fa49b47144180a0e037f0688bb9", size = 251068, upload-time = "2025-11-18T13:32:59.646Z" },
- { url = "https://files.pythonhosted.org/packages/ec/55/ca8ae7dbba962a3351f18940b359b94c6bafdd7757945fdc79ec9e452dc7/coverage-7.12.0-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:b2089cc445f2dc0af6f801f0d1355c025b76c24481935303cf1af28f636688f0", size = 249034, upload-time = "2025-11-18T13:33:01.481Z" },
- { url = "https://files.pythonhosted.org/packages/7a/d7/39136149325cad92d420b023b5fd900dabdd1c3a0d1d5f148ef4a8cedef5/coverage-7.12.0-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:950411f1eb5d579999c5f66c62a40961f126fc71e5e14419f004471957b51508", size = 248853, upload-time = "2025-11-18T13:33:02.935Z" },
- { url = "https://files.pythonhosted.org/packages/fe/b6/76e1add8b87ef60e00643b0b7f8f7bb73d4bf5249a3be19ebefc5793dd25/coverage-7.12.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:b1aab7302a87bafebfe76b12af681b56ff446dc6f32ed178ff9c092ca776e6bc", size = 250619, upload-time = "2025-11-18T13:33:04.336Z" },
- { url = "https://files.pythonhosted.org/packages/95/87/924c6dc64f9203f7a3c1832a6a0eee5a8335dbe5f1bdadcc278d6f1b4d74/coverage-7.12.0-cp313-cp313-win32.whl", hash = "sha256:d7e0d0303c13b54db495eb636bc2465b2fb8475d4c8bcec8fe4b5ca454dfbae8", size = 220261, upload-time = "2025-11-18T13:33:06.493Z" },
- { url = "https://files.pythonhosted.org/packages/91/77/dd4aff9af16ff776bf355a24d87eeb48fc6acde54c907cc1ea89b14a8804/coverage-7.12.0-cp313-cp313-win_amd64.whl", hash = "sha256:ce61969812d6a98a981d147d9ac583a36ac7db7766f2e64a9d4d059c2fe29d07", size = 221072, upload-time = "2025-11-18T13:33:07.926Z" },
- { url = "https://files.pythonhosted.org/packages/70/49/5c9dc46205fef31b1b226a6e16513193715290584317fd4df91cdaf28b22/coverage-7.12.0-cp313-cp313-win_arm64.whl", hash = "sha256:bcec6f47e4cb8a4c2dc91ce507f6eefc6a1b10f58df32cdc61dff65455031dfc", size = 219702, upload-time = "2025-11-18T13:33:09.631Z" },
- { url = "https://files.pythonhosted.org/packages/9b/62/f87922641c7198667994dd472a91e1d9b829c95d6c29529ceb52132436ad/coverage-7.12.0-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:459443346509476170d553035e4a3eed7b860f4fe5242f02de1010501956ce87", size = 218420, upload-time = "2025-11-18T13:33:11.153Z" },
- { url = "https://files.pythonhosted.org/packages/85/dd/1cc13b2395ef15dbb27d7370a2509b4aee77890a464fb35d72d428f84871/coverage-7.12.0-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:04a79245ab2b7a61688958f7a855275997134bc84f4a03bc240cf64ff132abf6", size = 218773, upload-time = "2025-11-18T13:33:12.569Z" },
- { url = "https://files.pythonhosted.org/packages/74/40/35773cc4bb1e9d4658d4fb669eb4195b3151bef3bbd6f866aba5cd5dac82/coverage-7.12.0-cp313-cp313t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:09a86acaaa8455f13d6a99221d9654df249b33937b4e212b4e5a822065f12aa7", size = 260078, upload-time = "2025-11-18T13:33:14.037Z" },
- { url = "https://files.pythonhosted.org/packages/ec/ee/231bb1a6ffc2905e396557585ebc6bdc559e7c66708376d245a1f1d330fc/coverage-7.12.0-cp313-cp313t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:907e0df1b71ba77463687a74149c6122c3f6aac56c2510a5d906b2f368208560", size = 262144, upload-time = "2025-11-18T13:33:15.601Z" },
- { url = "https://files.pythonhosted.org/packages/28/be/32f4aa9f3bf0b56f3971001b56508352c7753915345d45fab4296a986f01/coverage-7.12.0-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9b57e2d0ddd5f0582bae5437c04ee71c46cd908e7bc5d4d0391f9a41e812dd12", size = 264574, upload-time = "2025-11-18T13:33:17.354Z" },
- { url = "https://files.pythonhosted.org/packages/68/7c/00489fcbc2245d13ab12189b977e0cf06ff3351cb98bc6beba8bd68c5902/coverage-7.12.0-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:58c1c6aa677f3a1411fe6fb28ec3a942e4f665df036a3608816e0847fad23296", size = 259298, upload-time = "2025-11-18T13:33:18.958Z" },
- { url = "https://files.pythonhosted.org/packages/96/b4/f0760d65d56c3bea95b449e02570d4abd2549dc784bf39a2d4721a2d8ceb/coverage-7.12.0-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:4c589361263ab2953e3c4cd2a94db94c4ad4a8e572776ecfbad2389c626e4507", size = 262150, upload-time = "2025-11-18T13:33:20.644Z" },
- { url = "https://files.pythonhosted.org/packages/c5/71/9a9314df00f9326d78c1e5a910f520d599205907432d90d1c1b7a97aa4b1/coverage-7.12.0-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:91b810a163ccad2e43b1faa11d70d3cf4b6f3d83f9fd5f2df82a32d47b648e0d", size = 259763, upload-time = "2025-11-18T13:33:22.189Z" },
- { url = "https://files.pythonhosted.org/packages/10/34/01a0aceed13fbdf925876b9a15d50862eb8845454301fe3cdd1df08b2182/coverage-7.12.0-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:40c867af715f22592e0d0fb533a33a71ec9e0f73a6945f722a0c85c8c1cbe3a2", size = 258653, upload-time = "2025-11-18T13:33:24.239Z" },
- { url = "https://files.pythonhosted.org/packages/8d/04/81d8fd64928acf1574bbb0181f66901c6c1c6279c8ccf5f84259d2c68ae9/coverage-7.12.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:68b0d0a2d84f333de875666259dadf28cc67858bc8fd8b3f1eae84d3c2bec455", size = 260856, upload-time = "2025-11-18T13:33:26.365Z" },
- { url = "https://files.pythonhosted.org/packages/f2/76/fa2a37bfaeaf1f766a2d2360a25a5297d4fb567098112f6517475eee120b/coverage-7.12.0-cp313-cp313t-win32.whl", hash = "sha256:73f9e7fbd51a221818fd11b7090eaa835a353ddd59c236c57b2199486b116c6d", size = 220936, upload-time = "2025-11-18T13:33:28.165Z" },
- { url = "https://files.pythonhosted.org/packages/f9/52/60f64d932d555102611c366afb0eb434b34266b1d9266fc2fe18ab641c47/coverage-7.12.0-cp313-cp313t-win_amd64.whl", hash = "sha256:24cff9d1f5743f67db7ba46ff284018a6e9aeb649b67aa1e70c396aa1b7cb23c", size = 222001, upload-time = "2025-11-18T13:33:29.656Z" },
- { url = "https://files.pythonhosted.org/packages/77/df/c303164154a5a3aea7472bf323b7c857fed93b26618ed9fc5c2955566bb0/coverage-7.12.0-cp313-cp313t-win_arm64.whl", hash = "sha256:c87395744f5c77c866d0f5a43d97cc39e17c7f1cb0115e54a2fe67ca75c5d14d", size = 220273, upload-time = "2025-11-18T13:33:31.415Z" },
- { url = "https://files.pythonhosted.org/packages/bf/2e/fc12db0883478d6e12bbd62d481210f0c8daf036102aa11434a0c5755825/coverage-7.12.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:a1c59b7dc169809a88b21a936eccf71c3895a78f5592051b1af8f4d59c2b4f92", size = 217777, upload-time = "2025-11-18T13:33:32.86Z" },
- { url = "https://files.pythonhosted.org/packages/1f/c1/ce3e525d223350c6ec16b9be8a057623f54226ef7f4c2fee361ebb6a02b8/coverage-7.12.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:8787b0f982e020adb732b9f051f3e49dd5054cebbc3f3432061278512a2b1360", size = 218100, upload-time = "2025-11-18T13:33:34.532Z" },
- { url = "https://files.pythonhosted.org/packages/15/87/113757441504aee3808cb422990ed7c8bcc2d53a6779c66c5adef0942939/coverage-7.12.0-cp314-cp314-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:5ea5a9f7dc8877455b13dd1effd3202e0bca72f6f3ab09f9036b1bcf728f69ac", size = 249151, upload-time = "2025-11-18T13:33:36.135Z" },
- { url = "https://files.pythonhosted.org/packages/d9/1d/9529d9bd44049b6b05bb319c03a3a7e4b0a8a802d28fa348ad407e10706d/coverage-7.12.0-cp314-cp314-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:fdba9f15849534594f60b47c9a30bc70409b54947319a7c4fd0e8e3d8d2f355d", size = 251667, upload-time = "2025-11-18T13:33:37.996Z" },
- { url = "https://files.pythonhosted.org/packages/11/bb/567e751c41e9c03dc29d3ce74b8c89a1e3396313e34f255a2a2e8b9ebb56/coverage-7.12.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a00594770eb715854fb1c57e0dea08cce6720cfbc531accdb9850d7c7770396c", size = 253003, upload-time = "2025-11-18T13:33:39.553Z" },
- { url = "https://files.pythonhosted.org/packages/e4/b3/c2cce2d8526a02fb9e9ca14a263ca6fc074449b33a6afa4892838c903528/coverage-7.12.0-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:5560c7e0d82b42eb1951e4f68f071f8017c824ebfd5a6ebe42c60ac16c6c2434", size = 249185, upload-time = "2025-11-18T13:33:42.086Z" },
- { url = "https://files.pythonhosted.org/packages/0e/a7/967f93bb66e82c9113c66a8d0b65ecf72fc865adfba5a145f50c7af7e58d/coverage-7.12.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:d6c2e26b481c9159c2773a37947a9718cfdc58893029cdfb177531793e375cfc", size = 251025, upload-time = "2025-11-18T13:33:43.634Z" },
- { url = "https://files.pythonhosted.org/packages/b9/b2/f2f6f56337bc1af465d5b2dc1ee7ee2141b8b9272f3bf6213fcbc309a836/coverage-7.12.0-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:6e1a8c066dabcde56d5d9fed6a66bc19a2883a3fe051f0c397a41fc42aedd4cc", size = 248979, upload-time = "2025-11-18T13:33:46.04Z" },
- { url = "https://files.pythonhosted.org/packages/f4/7a/bf4209f45a4aec09d10a01a57313a46c0e0e8f4c55ff2965467d41a92036/coverage-7.12.0-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:f7ba9da4726e446d8dd8aae5a6cd872511184a5d861de80a86ef970b5dacce3e", size = 248800, upload-time = "2025-11-18T13:33:47.546Z" },
- { url = "https://files.pythonhosted.org/packages/b8/b7/1e01b8696fb0521810f60c5bbebf699100d6754183e6cc0679bf2ed76531/coverage-7.12.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:e0f483ab4f749039894abaf80c2f9e7ed77bbf3c737517fb88c8e8e305896a17", size = 250460, upload-time = "2025-11-18T13:33:49.537Z" },
- { url = "https://files.pythonhosted.org/packages/71/ae/84324fb9cb46c024760e706353d9b771a81b398d117d8c1fe010391c186f/coverage-7.12.0-cp314-cp314-win32.whl", hash = "sha256:76336c19a9ef4a94b2f8dc79f8ac2da3f193f625bb5d6f51a328cd19bfc19933", size = 220533, upload-time = "2025-11-18T13:33:51.16Z" },
- { url = "https://files.pythonhosted.org/packages/e2/71/1033629deb8460a8f97f83e6ac4ca3b93952e2b6f826056684df8275e015/coverage-7.12.0-cp314-cp314-win_amd64.whl", hash = "sha256:7c1059b600aec6ef090721f8f633f60ed70afaffe8ecab85b59df748f24b31fe", size = 221348, upload-time = "2025-11-18T13:33:52.776Z" },
- { url = "https://files.pythonhosted.org/packages/0a/5f/ac8107a902f623b0c251abdb749be282dc2ab61854a8a4fcf49e276fce2f/coverage-7.12.0-cp314-cp314-win_arm64.whl", hash = "sha256:172cf3a34bfef42611963e2b661302a8931f44df31629e5b1050567d6b90287d", size = 219922, upload-time = "2025-11-18T13:33:54.316Z" },
- { url = "https://files.pythonhosted.org/packages/79/6e/f27af2d4da367f16077d21ef6fe796c874408219fa6dd3f3efe7751bd910/coverage-7.12.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:aa7d48520a32cb21c7a9b31f81799e8eaec7239db36c3b670be0fa2403828d1d", size = 218511, upload-time = "2025-11-18T13:33:56.343Z" },
- { url = "https://files.pythonhosted.org/packages/67/dd/65fd874aa460c30da78f9d259400d8e6a4ef457d61ab052fd248f0050558/coverage-7.12.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:90d58ac63bc85e0fb919f14d09d6caa63f35a5512a2205284b7816cafd21bb03", size = 218771, upload-time = "2025-11-18T13:33:57.966Z" },
- { url = "https://files.pythonhosted.org/packages/55/e0/7c6b71d327d8068cb79c05f8f45bf1b6145f7a0de23bbebe63578fe5240a/coverage-7.12.0-cp314-cp314t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:ca8ecfa283764fdda3eae1bdb6afe58bf78c2c3ec2b2edcb05a671f0bba7b3f9", size = 260151, upload-time = "2025-11-18T13:33:59.597Z" },
- { url = "https://files.pythonhosted.org/packages/49/ce/4697457d58285b7200de6b46d606ea71066c6e674571a946a6ea908fb588/coverage-7.12.0-cp314-cp314t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:874fe69a0785d96bd066059cd4368022cebbec1a8958f224f0016979183916e6", size = 262257, upload-time = "2025-11-18T13:34:01.166Z" },
- { url = "https://files.pythonhosted.org/packages/2f/33/acbc6e447aee4ceba88c15528dbe04a35fb4d67b59d393d2e0d6f1e242c1/coverage-7.12.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5b3c889c0b8b283a24d721a9eabc8ccafcfc3aebf167e4cd0d0e23bf8ec4e339", size = 264671, upload-time = "2025-11-18T13:34:02.795Z" },
- { url = "https://files.pythonhosted.org/packages/87/ec/e2822a795c1ed44d569980097be839c5e734d4c0c1119ef8e0a073496a30/coverage-7.12.0-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:8bb5b894b3ec09dcd6d3743229dc7f2c42ef7787dc40596ae04c0edda487371e", size = 259231, upload-time = "2025-11-18T13:34:04.397Z" },
- { url = "https://files.pythonhosted.org/packages/72/c5/a7ec5395bb4a49c9b7ad97e63f0c92f6bf4a9e006b1393555a02dae75f16/coverage-7.12.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:79a44421cd5fba96aa57b5e3b5a4d3274c449d4c622e8f76882d76635501fd13", size = 262137, upload-time = "2025-11-18T13:34:06.068Z" },
- { url = "https://files.pythonhosted.org/packages/67/0c/02c08858b764129f4ecb8e316684272972e60777ae986f3865b10940bdd6/coverage-7.12.0-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:33baadc0efd5c7294f436a632566ccc1f72c867f82833eb59820ee37dc811c6f", size = 259745, upload-time = "2025-11-18T13:34:08.04Z" },
- { url = "https://files.pythonhosted.org/packages/5a/04/4fd32b7084505f3829a8fe45c1a74a7a728cb251aaadbe3bec04abcef06d/coverage-7.12.0-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:c406a71f544800ef7e9e0000af706b88465f3573ae8b8de37e5f96c59f689ad1", size = 258570, upload-time = "2025-11-18T13:34:09.676Z" },
- { url = "https://files.pythonhosted.org/packages/48/35/2365e37c90df4f5342c4fa202223744119fe31264ee2924f09f074ea9b6d/coverage-7.12.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:e71bba6a40883b00c6d571599b4627f50c360b3d0d02bfc658168936be74027b", size = 260899, upload-time = "2025-11-18T13:34:11.259Z" },
- { url = "https://files.pythonhosted.org/packages/05/56/26ab0464ca733fa325e8e71455c58c1c374ce30f7c04cebb88eabb037b18/coverage-7.12.0-cp314-cp314t-win32.whl", hash = "sha256:9157a5e233c40ce6613dead4c131a006adfda70e557b6856b97aceed01b0e27a", size = 221313, upload-time = "2025-11-18T13:34:12.863Z" },
- { url = "https://files.pythonhosted.org/packages/da/1c/017a3e1113ed34d998b27d2c6dba08a9e7cb97d362f0ec988fcd873dcf81/coverage-7.12.0-cp314-cp314t-win_amd64.whl", hash = "sha256:e84da3a0fd233aeec797b981c51af1cabac74f9bd67be42458365b30d11b5291", size = 222423, upload-time = "2025-11-18T13:34:15.14Z" },
- { url = "https://files.pythonhosted.org/packages/4c/36/bcc504fdd5169301b52568802bb1b9cdde2e27a01d39fbb3b4b508ab7c2c/coverage-7.12.0-cp314-cp314t-win_arm64.whl", hash = "sha256:01d24af36fedda51c2b1aca56e4330a3710f83b02a5ff3743a6b015ffa7c9384", size = 220459, upload-time = "2025-11-18T13:34:17.222Z" },
- { url = "https://files.pythonhosted.org/packages/ce/a3/43b749004e3c09452e39bb56347a008f0a0668aad37324a99b5c8ca91d9e/coverage-7.12.0-py3-none-any.whl", hash = "sha256:159d50c0b12e060b15ed3d39f87ed43d4f7f7ad40b8a534f4dd331adbb51104a", size = 209503, upload-time = "2025-11-18T13:34:18.892Z" },
+version = "7.13.1"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/23/f9/e92df5e07f3fc8d4c7f9a0f146ef75446bf870351cd37b788cf5897f8079/coverage-7.13.1.tar.gz", hash = "sha256:b7593fe7eb5feaa3fbb461ac79aac9f9fc0387a5ca8080b0c6fe2ca27b091afd", size = 825862, upload-time = "2025-12-28T15:42:56.969Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/2d/9a/3742e58fd04b233df95c012ee9f3dfe04708a5e1d32613bd2d47d4e1be0d/coverage-7.13.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:e1fa280b3ad78eea5be86f94f461c04943d942697e0dac889fa18fff8f5f9147", size = 218633, upload-time = "2025-12-28T15:40:10.165Z" },
+ { url = "https://files.pythonhosted.org/packages/7e/45/7e6bdc94d89cd7c8017ce735cf50478ddfe765d4fbf0c24d71d30ea33d7a/coverage-7.13.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:c3d8c679607220979434f494b139dfb00131ebf70bb406553d69c1ff01a5c33d", size = 219147, upload-time = "2025-12-28T15:40:12.069Z" },
+ { url = "https://files.pythonhosted.org/packages/f7/38/0d6a258625fd7f10773fe94097dc16937a5f0e3e0cdf3adef67d3ac6baef/coverage-7.13.1-cp310-cp310-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:339dc63b3eba969067b00f41f15ad161bf2946613156fb131266d8debc8e44d0", size = 245894, upload-time = "2025-12-28T15:40:13.556Z" },
+ { url = "https://files.pythonhosted.org/packages/27/58/409d15ea487986994cbd4d06376e9860e9b157cfbfd402b1236770ab8dd2/coverage-7.13.1-cp310-cp310-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:db622b999ffe49cb891f2fff3b340cdc2f9797d01a0a202a0973ba2562501d90", size = 247721, upload-time = "2025-12-28T15:40:15.37Z" },
+ { url = "https://files.pythonhosted.org/packages/da/bf/6e8056a83fd7a96c93341f1ffe10df636dd89f26d5e7b9ca511ce3bcf0df/coverage-7.13.1-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d1443ba9acbb593fa7c1c29e011d7c9761545fe35e7652e85ce7f51a16f7e08d", size = 249585, upload-time = "2025-12-28T15:40:17.226Z" },
+ { url = "https://files.pythonhosted.org/packages/f4/15/e1daff723f9f5959acb63cbe35b11203a9df77ee4b95b45fffd38b318390/coverage-7.13.1-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:c832ec92c4499ac463186af72f9ed4d8daec15499b16f0a879b0d1c8e5cf4a3b", size = 246597, upload-time = "2025-12-28T15:40:19.028Z" },
+ { url = "https://files.pythonhosted.org/packages/74/a6/1efd31c5433743a6ddbc9d37ac30c196bb07c7eab3d74fbb99b924c93174/coverage-7.13.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:562ec27dfa3f311e0db1ba243ec6e5f6ab96b1edfcfc6cf86f28038bc4961ce6", size = 247626, upload-time = "2025-12-28T15:40:20.846Z" },
+ { url = "https://files.pythonhosted.org/packages/6d/9f/1609267dd3e749f57fdd66ca6752567d1c13b58a20a809dc409b263d0b5f/coverage-7.13.1-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:4de84e71173d4dada2897e5a0e1b7877e5eefbfe0d6a44edee6ce31d9b8ec09e", size = 245629, upload-time = "2025-12-28T15:40:22.397Z" },
+ { url = "https://files.pythonhosted.org/packages/e2/f6/6815a220d5ec2466383d7cc36131b9fa6ecbe95c50ec52a631ba733f306a/coverage-7.13.1-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:a5a68357f686f8c4d527a2dc04f52e669c2fc1cbde38f6f7eb6a0e58cbd17cae", size = 245901, upload-time = "2025-12-28T15:40:23.836Z" },
+ { url = "https://files.pythonhosted.org/packages/ac/58/40576554cd12e0872faf6d2c0eb3bc85f71d78427946ddd19ad65201e2c0/coverage-7.13.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:77cc258aeb29a3417062758975521eae60af6f79e930d6993555eeac6a8eac29", size = 246505, upload-time = "2025-12-28T15:40:25.421Z" },
+ { url = "https://files.pythonhosted.org/packages/3b/77/9233a90253fba576b0eee81707b5781d0e21d97478e5377b226c5b096c0f/coverage-7.13.1-cp310-cp310-win32.whl", hash = "sha256:bb4f8c3c9a9f34423dba193f241f617b08ffc63e27f67159f60ae6baf2dcfe0f", size = 221257, upload-time = "2025-12-28T15:40:27.217Z" },
+ { url = "https://files.pythonhosted.org/packages/e0/43/e842ff30c1a0a623ec80db89befb84a3a7aad7bfe44a6ea77d5a3e61fedd/coverage-7.13.1-cp310-cp310-win_amd64.whl", hash = "sha256:c8e2706ceb622bc63bac98ebb10ef5da80ed70fbd8a7999a5076de3afaef0fb1", size = 222191, upload-time = "2025-12-28T15:40:28.916Z" },
+ { url = "https://files.pythonhosted.org/packages/b4/9b/77baf488516e9ced25fc215a6f75d803493fc3f6a1a1227ac35697910c2a/coverage-7.13.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1a55d509a1dc5a5b708b5dad3b5334e07a16ad4c2185e27b40e4dba796ab7f88", size = 218755, upload-time = "2025-12-28T15:40:30.812Z" },
+ { url = "https://files.pythonhosted.org/packages/d7/cd/7ab01154e6eb79ee2fab76bf4d89e94c6648116557307ee4ebbb85e5c1bf/coverage-7.13.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:4d010d080c4888371033baab27e47c9df7d6fb28d0b7b7adf85a4a49be9298b3", size = 219257, upload-time = "2025-12-28T15:40:32.333Z" },
+ { url = "https://files.pythonhosted.org/packages/01/d5/b11ef7863ffbbdb509da0023fad1e9eda1c0eaea61a6d2ea5b17d4ac706e/coverage-7.13.1-cp311-cp311-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:d938b4a840fb1523b9dfbbb454f652967f18e197569c32266d4d13f37244c3d9", size = 249657, upload-time = "2025-12-28T15:40:34.1Z" },
+ { url = "https://files.pythonhosted.org/packages/f7/7c/347280982982383621d29b8c544cf497ae07ac41e44b1ca4903024131f55/coverage-7.13.1-cp311-cp311-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:bf100a3288f9bb7f919b87eb84f87101e197535b9bd0e2c2b5b3179633324fee", size = 251581, upload-time = "2025-12-28T15:40:36.131Z" },
+ { url = "https://files.pythonhosted.org/packages/82/f6/ebcfed11036ade4c0d75fa4453a6282bdd225bc073862766eec184a4c643/coverage-7.13.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ef6688db9bf91ba111ae734ba6ef1a063304a881749726e0d3575f5c10a9facf", size = 253691, upload-time = "2025-12-28T15:40:37.626Z" },
+ { url = "https://files.pythonhosted.org/packages/02/92/af8f5582787f5d1a8b130b2dcba785fa5e9a7a8e121a0bb2220a6fdbdb8a/coverage-7.13.1-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:0b609fc9cdbd1f02e51f67f51e5aee60a841ef58a68d00d5ee2c0faf357481a3", size = 249799, upload-time = "2025-12-28T15:40:39.47Z" },
+ { url = "https://files.pythonhosted.org/packages/24/aa/0e39a2a3b16eebf7f193863323edbff38b6daba711abaaf807d4290cf61a/coverage-7.13.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:c43257717611ff5e9a1d79dce8e47566235ebda63328718d9b65dd640bc832ef", size = 251389, upload-time = "2025-12-28T15:40:40.954Z" },
+ { url = "https://files.pythonhosted.org/packages/73/46/7f0c13111154dc5b978900c0ccee2e2ca239b910890e674a77f1363d483e/coverage-7.13.1-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:e09fbecc007f7b6afdfb3b07ce5bd9f8494b6856dd4f577d26c66c391b829851", size = 249450, upload-time = "2025-12-28T15:40:42.489Z" },
+ { url = "https://files.pythonhosted.org/packages/ac/ca/e80da6769e8b669ec3695598c58eef7ad98b0e26e66333996aee6316db23/coverage-7.13.1-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:a03a4f3a19a189919c7055098790285cc5c5b0b3976f8d227aea39dbf9f8bfdb", size = 249170, upload-time = "2025-12-28T15:40:44.279Z" },
+ { url = "https://files.pythonhosted.org/packages/af/18/9e29baabdec1a8644157f572541079b4658199cfd372a578f84228e860de/coverage-7.13.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:3820778ea1387c2b6a818caec01c63adc5b3750211af6447e8dcfb9b6f08dbba", size = 250081, upload-time = "2025-12-28T15:40:45.748Z" },
+ { url = "https://files.pythonhosted.org/packages/00/f8/c3021625a71c3b2f516464d322e41636aea381018319050a8114105872ee/coverage-7.13.1-cp311-cp311-win32.whl", hash = "sha256:ff10896fa55167371960c5908150b434b71c876dfab97b69478f22c8b445ea19", size = 221281, upload-time = "2025-12-28T15:40:47.232Z" },
+ { url = "https://files.pythonhosted.org/packages/27/56/c216625f453df6e0559ed666d246fcbaaa93f3aa99eaa5080cea1229aa3d/coverage-7.13.1-cp311-cp311-win_amd64.whl", hash = "sha256:a998cc0aeeea4c6d5622a3754da5a493055d2d95186bad877b0a34ea6e6dbe0a", size = 222215, upload-time = "2025-12-28T15:40:49.19Z" },
+ { url = "https://files.pythonhosted.org/packages/5c/9a/be342e76f6e531cae6406dc46af0d350586f24d9b67fdfa6daee02df71af/coverage-7.13.1-cp311-cp311-win_arm64.whl", hash = "sha256:fea07c1a39a22614acb762e3fbbb4011f65eedafcb2948feeef641ac78b4ee5c", size = 220886, upload-time = "2025-12-28T15:40:51.067Z" },
+ { url = "https://files.pythonhosted.org/packages/ce/8a/87af46cccdfa78f53db747b09f5f9a21d5fc38d796834adac09b30a8ce74/coverage-7.13.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:6f34591000f06e62085b1865c9bc5f7858df748834662a51edadfd2c3bfe0dd3", size = 218927, upload-time = "2025-12-28T15:40:52.814Z" },
+ { url = "https://files.pythonhosted.org/packages/82/a8/6e22fdc67242a4a5a153f9438d05944553121c8f4ba70cb072af4c41362e/coverage-7.13.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:b67e47c5595b9224599016e333f5ec25392597a89d5744658f837d204e16c63e", size = 219288, upload-time = "2025-12-28T15:40:54.262Z" },
+ { url = "https://files.pythonhosted.org/packages/d0/0a/853a76e03b0f7c4375e2ca025df45c918beb367f3e20a0a8e91967f6e96c/coverage-7.13.1-cp312-cp312-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:3e7b8bd70c48ffb28461ebe092c2345536fb18bbbf19d287c8913699735f505c", size = 250786, upload-time = "2025-12-28T15:40:56.059Z" },
+ { url = "https://files.pythonhosted.org/packages/ea/b4/694159c15c52b9f7ec7adf49d50e5f8ee71d3e9ef38adb4445d13dd56c20/coverage-7.13.1-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:c223d078112e90dc0e5c4e35b98b9584164bea9fbbd221c0b21c5241f6d51b62", size = 253543, upload-time = "2025-12-28T15:40:57.585Z" },
+ { url = "https://files.pythonhosted.org/packages/96/b2/7f1f0437a5c855f87e17cf5d0dc35920b6440ff2b58b1ba9788c059c26c8/coverage-7.13.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:794f7c05af0763b1bbd1b9e6eff0e52ad068be3b12cd96c87de037b01390c968", size = 254635, upload-time = "2025-12-28T15:40:59.443Z" },
+ { url = "https://files.pythonhosted.org/packages/e9/d1/73c3fdb8d7d3bddd9473c9c6a2e0682f09fc3dfbcb9c3f36412a7368bcab/coverage-7.13.1-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:0642eae483cc8c2902e4af7298bf886d605e80f26382124cddc3967c2a3df09e", size = 251202, upload-time = "2025-12-28T15:41:01.328Z" },
+ { url = "https://files.pythonhosted.org/packages/66/3c/f0edf75dcc152f145d5598329e864bbbe04ab78660fe3e8e395f9fff010f/coverage-7.13.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:9f5e772ed5fef25b3de9f2008fe67b92d46831bd2bc5bdc5dd6bfd06b83b316f", size = 252566, upload-time = "2025-12-28T15:41:03.319Z" },
+ { url = "https://files.pythonhosted.org/packages/17/b3/e64206d3c5f7dcbceafd14941345a754d3dbc78a823a6ed526e23b9cdaab/coverage-7.13.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:45980ea19277dc0a579e432aef6a504fe098ef3a9032ead15e446eb0f1191aee", size = 250711, upload-time = "2025-12-28T15:41:06.411Z" },
+ { url = "https://files.pythonhosted.org/packages/dc/ad/28a3eb970a8ef5b479ee7f0c484a19c34e277479a5b70269dc652b730733/coverage-7.13.1-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:e4f18eca6028ffa62adbd185a8f1e1dd242f2e68164dba5c2b74a5204850b4cf", size = 250278, upload-time = "2025-12-28T15:41:08.285Z" },
+ { url = "https://files.pythonhosted.org/packages/54/e3/c8f0f1a93133e3e1291ca76cbb63565bd4b5c5df63b141f539d747fff348/coverage-7.13.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:f8dca5590fec7a89ed6826fce625595279e586ead52e9e958d3237821fbc750c", size = 252154, upload-time = "2025-12-28T15:41:09.969Z" },
+ { url = "https://files.pythonhosted.org/packages/d0/bf/9939c5d6859c380e405b19e736321f1c7d402728792f4c752ad1adcce005/coverage-7.13.1-cp312-cp312-win32.whl", hash = "sha256:ff86d4e85188bba72cfb876df3e11fa243439882c55957184af44a35bd5880b7", size = 221487, upload-time = "2025-12-28T15:41:11.468Z" },
+ { url = "https://files.pythonhosted.org/packages/fa/dc/7282856a407c621c2aad74021680a01b23010bb8ebf427cf5eacda2e876f/coverage-7.13.1-cp312-cp312-win_amd64.whl", hash = "sha256:16cc1da46c04fb0fb128b4dc430b78fa2aba8a6c0c9f8eb391fd5103409a6ac6", size = 222299, upload-time = "2025-12-28T15:41:13.386Z" },
+ { url = "https://files.pythonhosted.org/packages/10/79/176a11203412c350b3e9578620013af35bcdb79b651eb976f4a4b32044fa/coverage-7.13.1-cp312-cp312-win_arm64.whl", hash = "sha256:8d9bc218650022a768f3775dd7fdac1886437325d8d295d923ebcfef4892ad5c", size = 220941, upload-time = "2025-12-28T15:41:14.975Z" },
+ { url = "https://files.pythonhosted.org/packages/a3/a4/e98e689347a1ff1a7f67932ab535cef82eb5e78f32a9e4132e114bbb3a0a/coverage-7.13.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:cb237bfd0ef4d5eb6a19e29f9e528ac67ac3be932ea6b44fb6cc09b9f3ecff78", size = 218951, upload-time = "2025-12-28T15:41:16.653Z" },
+ { url = "https://files.pythonhosted.org/packages/32/33/7cbfe2bdc6e2f03d6b240d23dc45fdaf3fd270aaf2d640be77b7f16989ab/coverage-7.13.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:1dcb645d7e34dcbcc96cd7c132b1fc55c39263ca62eb961c064eb3928997363b", size = 219325, upload-time = "2025-12-28T15:41:18.609Z" },
+ { url = "https://files.pythonhosted.org/packages/59/f6/efdabdb4929487baeb7cb2a9f7dac457d9356f6ad1b255be283d58b16316/coverage-7.13.1-cp313-cp313-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:3d42df8201e00384736f0df9be2ced39324c3907607d17d50d50116c989d84cd", size = 250309, upload-time = "2025-12-28T15:41:20.629Z" },
+ { url = "https://files.pythonhosted.org/packages/12/da/91a52516e9d5aea87d32d1523f9cdcf7a35a3b298e6be05d6509ba3cfab2/coverage-7.13.1-cp313-cp313-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:fa3edde1aa8807de1d05934982416cb3ec46d1d4d91e280bcce7cca01c507992", size = 252907, upload-time = "2025-12-28T15:41:22.257Z" },
+ { url = "https://files.pythonhosted.org/packages/75/38/f1ea837e3dc1231e086db1638947e00d264e7e8c41aa8ecacf6e1e0c05f4/coverage-7.13.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9edd0e01a343766add6817bc448408858ba6b489039eaaa2018474e4001651a4", size = 254148, upload-time = "2025-12-28T15:41:23.87Z" },
+ { url = "https://files.pythonhosted.org/packages/7f/43/f4f16b881aaa34954ba446318dea6b9ed5405dd725dd8daac2358eda869a/coverage-7.13.1-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:985b7836931d033570b94c94713c6dba5f9d3ff26045f72c3e5dbc5fe3361e5a", size = 250515, upload-time = "2025-12-28T15:41:25.437Z" },
+ { url = "https://files.pythonhosted.org/packages/84/34/8cba7f00078bd468ea914134e0144263194ce849ec3baad187ffb6203d1c/coverage-7.13.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:ffed1e4980889765c84a5d1a566159e363b71d6b6fbaf0bebc9d3c30bc016766", size = 252292, upload-time = "2025-12-28T15:41:28.459Z" },
+ { url = "https://files.pythonhosted.org/packages/8c/a4/cffac66c7652d84ee4ac52d3ccb94c015687d3b513f9db04bfcac2ac800d/coverage-7.13.1-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:8842af7f175078456b8b17f1b73a0d16a65dcbdc653ecefeb00a56b3c8c298c4", size = 250242, upload-time = "2025-12-28T15:41:30.02Z" },
+ { url = "https://files.pythonhosted.org/packages/f4/78/9a64d462263dde416f3c0067efade7b52b52796f489b1037a95b0dc389c9/coverage-7.13.1-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:ccd7a6fca48ca9c131d9b0a2972a581e28b13416fc313fb98b6d24a03ce9a398", size = 250068, upload-time = "2025-12-28T15:41:32.007Z" },
+ { url = "https://files.pythonhosted.org/packages/69/c8/a8994f5fece06db7c4a97c8fc1973684e178599b42e66280dded0524ef00/coverage-7.13.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:0403f647055de2609be776965108447deb8e384fe4a553c119e3ff6bfbab4784", size = 251846, upload-time = "2025-12-28T15:41:33.946Z" },
+ { url = "https://files.pythonhosted.org/packages/cc/f7/91fa73c4b80305c86598a2d4e54ba22df6bf7d0d97500944af7ef155d9f7/coverage-7.13.1-cp313-cp313-win32.whl", hash = "sha256:549d195116a1ba1e1ae2f5ca143f9777800f6636eab917d4f02b5310d6d73461", size = 221512, upload-time = "2025-12-28T15:41:35.519Z" },
+ { url = "https://files.pythonhosted.org/packages/45/0b/0768b4231d5a044da8f75e097a8714ae1041246bb765d6b5563bab456735/coverage-7.13.1-cp313-cp313-win_amd64.whl", hash = "sha256:5899d28b5276f536fcf840b18b61a9fce23cc3aec1d114c44c07fe94ebeaa500", size = 222321, upload-time = "2025-12-28T15:41:37.371Z" },
+ { url = "https://files.pythonhosted.org/packages/9b/b8/bdcb7253b7e85157282450262008f1366aa04663f3e3e4c30436f596c3e2/coverage-7.13.1-cp313-cp313-win_arm64.whl", hash = "sha256:868a2fae76dfb06e87291bcbd4dcbcc778a8500510b618d50496e520bd94d9b9", size = 220949, upload-time = "2025-12-28T15:41:39.553Z" },
+ { url = "https://files.pythonhosted.org/packages/70/52/f2be52cc445ff75ea8397948c96c1b4ee14f7f9086ea62fc929c5ae7b717/coverage-7.13.1-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:67170979de0dacac3f3097d02b0ad188d8edcea44ccc44aaa0550af49150c7dc", size = 219643, upload-time = "2025-12-28T15:41:41.567Z" },
+ { url = "https://files.pythonhosted.org/packages/47/79/c85e378eaa239e2edec0c5523f71542c7793fe3340954eafb0bc3904d32d/coverage-7.13.1-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:f80e2bb21bfab56ed7405c2d79d34b5dc0bc96c2c1d2a067b643a09fb756c43a", size = 219997, upload-time = "2025-12-28T15:41:43.418Z" },
+ { url = "https://files.pythonhosted.org/packages/fe/9b/b1ade8bfb653c0bbce2d6d6e90cc6c254cbb99b7248531cc76253cb4da6d/coverage-7.13.1-cp313-cp313t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:f83351e0f7dcdb14d7326c3d8d8c4e915fa685cbfdc6281f9470d97a04e9dfe4", size = 261296, upload-time = "2025-12-28T15:41:45.207Z" },
+ { url = "https://files.pythonhosted.org/packages/1f/af/ebf91e3e1a2473d523e87e87fd8581e0aa08741b96265730e2d79ce78d8d/coverage-7.13.1-cp313-cp313t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:bb3f6562e89bad0110afbe64e485aac2462efdce6232cdec7862a095dc3412f6", size = 263363, upload-time = "2025-12-28T15:41:47.163Z" },
+ { url = "https://files.pythonhosted.org/packages/c4/8b/fb2423526d446596624ac7fde12ea4262e66f86f5120114c3cfd0bb2befa/coverage-7.13.1-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:77545b5dcda13b70f872c3b5974ac64c21d05e65b1590b441c8560115dc3a0d1", size = 265783, upload-time = "2025-12-28T15:41:49.03Z" },
+ { url = "https://files.pythonhosted.org/packages/9b/26/ef2adb1e22674913b89f0fe7490ecadcef4a71fa96f5ced90c60ec358789/coverage-7.13.1-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a4d240d260a1aed814790bbe1f10a5ff31ce6c21bc78f0da4a1e8268d6c80dbd", size = 260508, upload-time = "2025-12-28T15:41:51.035Z" },
+ { url = "https://files.pythonhosted.org/packages/ce/7d/f0f59b3404caf662e7b5346247883887687c074ce67ba453ea08c612b1d5/coverage-7.13.1-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:d2287ac9360dec3837bfdad969963a5d073a09a85d898bd86bea82aa8876ef3c", size = 263357, upload-time = "2025-12-28T15:41:52.631Z" },
+ { url = "https://files.pythonhosted.org/packages/1a/b1/29896492b0b1a047604d35d6fa804f12818fa30cdad660763a5f3159e158/coverage-7.13.1-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:0d2c11f3ea4db66b5cbded23b20185c35066892c67d80ec4be4bab257b9ad1e0", size = 260978, upload-time = "2025-12-28T15:41:54.589Z" },
+ { url = "https://files.pythonhosted.org/packages/48/f2/971de1238a62e6f0a4128d37adadc8bb882ee96afbe03ff1570291754629/coverage-7.13.1-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:3fc6a169517ca0d7ca6846c3c5392ef2b9e38896f61d615cb75b9e7134d4ee1e", size = 259877, upload-time = "2025-12-28T15:41:56.263Z" },
+ { url = "https://files.pythonhosted.org/packages/6a/fc/0474efcbb590ff8628830e9aaec5f1831594874360e3251f1fdec31d07a3/coverage-7.13.1-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:d10a2ed46386e850bb3de503a54f9fe8192e5917fcbb143bfef653a9355e9a53", size = 262069, upload-time = "2025-12-28T15:41:58.093Z" },
+ { url = "https://files.pythonhosted.org/packages/88/4f/3c159b7953db37a7b44c0eab8a95c37d1aa4257c47b4602c04022d5cb975/coverage-7.13.1-cp313-cp313t-win32.whl", hash = "sha256:75a6f4aa904301dab8022397a22c0039edc1f51e90b83dbd4464b8a38dc87842", size = 222184, upload-time = "2025-12-28T15:41:59.763Z" },
+ { url = "https://files.pythonhosted.org/packages/58/a5/6b57d28f81417f9335774f20679d9d13b9a8fb90cd6160957aa3b54a2379/coverage-7.13.1-cp313-cp313t-win_amd64.whl", hash = "sha256:309ef5706e95e62578cda256b97f5e097916a2c26247c287bbe74794e7150df2", size = 223250, upload-time = "2025-12-28T15:42:01.52Z" },
+ { url = "https://files.pythonhosted.org/packages/81/7c/160796f3b035acfbb58be80e02e484548595aa67e16a6345e7910ace0a38/coverage-7.13.1-cp313-cp313t-win_arm64.whl", hash = "sha256:92f980729e79b5d16d221038dbf2e8f9a9136afa072f9d5d6ed4cb984b126a09", size = 221521, upload-time = "2025-12-28T15:42:03.275Z" },
+ { url = "https://files.pythonhosted.org/packages/aa/8e/ba0e597560c6563fc0adb902fda6526df5d4aa73bb10adf0574d03bd2206/coverage-7.13.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:97ab3647280d458a1f9adb85244e81587505a43c0c7cff851f5116cd2814b894", size = 218996, upload-time = "2025-12-28T15:42:04.978Z" },
+ { url = "https://files.pythonhosted.org/packages/6b/8e/764c6e116f4221dc7aa26c4061181ff92edb9c799adae6433d18eeba7a14/coverage-7.13.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:8f572d989142e0908e6acf57ad1b9b86989ff057c006d13b76c146ec6a20216a", size = 219326, upload-time = "2025-12-28T15:42:06.691Z" },
+ { url = "https://files.pythonhosted.org/packages/4f/a6/6130dc6d8da28cdcbb0f2bf8865aeca9b157622f7c0031e48c6cf9a0e591/coverage-7.13.1-cp314-cp314-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:d72140ccf8a147e94274024ff6fd8fb7811354cf7ef88b1f0a988ebaa5bc774f", size = 250374, upload-time = "2025-12-28T15:42:08.786Z" },
+ { url = "https://files.pythonhosted.org/packages/82/2b/783ded568f7cd6b677762f780ad338bf4b4750205860c17c25f7c708995e/coverage-7.13.1-cp314-cp314-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:d3c9f051b028810f5a87c88e5d6e9af3c0ff32ef62763bf15d29f740453ca909", size = 252882, upload-time = "2025-12-28T15:42:10.515Z" },
+ { url = "https://files.pythonhosted.org/packages/cd/b2/9808766d082e6a4d59eb0cc881a57fc1600eb2c5882813eefff8254f71b5/coverage-7.13.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f398ba4df52d30b1763f62eed9de5620dcde96e6f491f4c62686736b155aa6e4", size = 254218, upload-time = "2025-12-28T15:42:12.208Z" },
+ { url = "https://files.pythonhosted.org/packages/44/ea/52a985bb447c871cb4d2e376e401116520991b597c85afdde1ea9ef54f2c/coverage-7.13.1-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:132718176cc723026d201e347f800cd1a9e4b62ccd3f82476950834dad501c75", size = 250391, upload-time = "2025-12-28T15:42:14.21Z" },
+ { url = "https://files.pythonhosted.org/packages/7f/1d/125b36cc12310718873cfc8209ecfbc1008f14f4f5fa0662aa608e579353/coverage-7.13.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:9e549d642426e3579b3f4b92d0431543b012dcb6e825c91619d4e93b7363c3f9", size = 252239, upload-time = "2025-12-28T15:42:16.292Z" },
+ { url = "https://files.pythonhosted.org/packages/6a/16/10c1c164950cade470107f9f14bbac8485f8fb8515f515fca53d337e4a7f/coverage-7.13.1-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:90480b2134999301eea795b3a9dbf606c6fbab1b489150c501da84a959442465", size = 250196, upload-time = "2025-12-28T15:42:18.54Z" },
+ { url = "https://files.pythonhosted.org/packages/2a/c6/cd860fac08780c6fd659732f6ced1b40b79c35977c1356344e44d72ba6c4/coverage-7.13.1-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:e825dbb7f84dfa24663dd75835e7257f8882629fc11f03ecf77d84a75134b864", size = 250008, upload-time = "2025-12-28T15:42:20.365Z" },
+ { url = "https://files.pythonhosted.org/packages/f0/3a/a8c58d3d38f82a5711e1e0a67268362af48e1a03df27c03072ac30feefcf/coverage-7.13.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:623dcc6d7a7ba450bbdbeedbaa0c42b329bdae16491af2282f12a7e809be7eb9", size = 251671, upload-time = "2025-12-28T15:42:22.114Z" },
+ { url = "https://files.pythonhosted.org/packages/f0/bc/fd4c1da651d037a1e3d53e8cb3f8182f4b53271ffa9a95a2e211bacc0349/coverage-7.13.1-cp314-cp314-win32.whl", hash = "sha256:6e73ebb44dca5f708dc871fe0b90cf4cff1a13f9956f747cc87b535a840386f5", size = 221777, upload-time = "2025-12-28T15:42:23.919Z" },
+ { url = "https://files.pythonhosted.org/packages/4b/50/71acabdc8948464c17e90b5ffd92358579bd0910732c2a1c9537d7536aa6/coverage-7.13.1-cp314-cp314-win_amd64.whl", hash = "sha256:be753b225d159feb397bd0bf91ae86f689bad0da09d3b301478cd39b878ab31a", size = 222592, upload-time = "2025-12-28T15:42:25.619Z" },
+ { url = "https://files.pythonhosted.org/packages/f7/c8/a6fb943081bb0cc926499c7907731a6dc9efc2cbdc76d738c0ab752f1a32/coverage-7.13.1-cp314-cp314-win_arm64.whl", hash = "sha256:228b90f613b25ba0019361e4ab81520b343b622fc657daf7e501c4ed6a2366c0", size = 221169, upload-time = "2025-12-28T15:42:27.629Z" },
+ { url = "https://files.pythonhosted.org/packages/16/61/d5b7a0a0e0e40d62e59bc8c7aa1afbd86280d82728ba97f0673b746b78e2/coverage-7.13.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:60cfb538fe9ef86e5b2ab0ca8fc8d62524777f6c611dcaf76dc16fbe9b8e698a", size = 219730, upload-time = "2025-12-28T15:42:29.306Z" },
+ { url = "https://files.pythonhosted.org/packages/a3/2c/8881326445fd071bb49514d1ce97d18a46a980712b51fee84f9ab42845b4/coverage-7.13.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:57dfc8048c72ba48a8c45e188d811e5efd7e49b387effc8fb17e97936dde5bf6", size = 220001, upload-time = "2025-12-28T15:42:31.319Z" },
+ { url = "https://files.pythonhosted.org/packages/b5/d7/50de63af51dfa3a7f91cc37ad8fcc1e244b734232fbc8b9ab0f3c834a5cd/coverage-7.13.1-cp314-cp314t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:3f2f725aa3e909b3c5fdb8192490bdd8e1495e85906af74fe6e34a2a77ba0673", size = 261370, upload-time = "2025-12-28T15:42:32.992Z" },
+ { url = "https://files.pythonhosted.org/packages/e1/2c/d31722f0ec918fd7453b2758312729f645978d212b410cd0f7c2aed88a94/coverage-7.13.1-cp314-cp314t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:9ee68b21909686eeb21dfcba2c3b81fee70dcf38b140dcd5aa70680995fa3aa5", size = 263485, upload-time = "2025-12-28T15:42:34.759Z" },
+ { url = "https://files.pythonhosted.org/packages/fa/7a/2c114fa5c5fc08ba0777e4aec4c97e0b4a1afcb69c75f1f54cff78b073ab/coverage-7.13.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:724b1b270cb13ea2e6503476e34541a0b1f62280bc997eab443f87790202033d", size = 265890, upload-time = "2025-12-28T15:42:36.517Z" },
+ { url = "https://files.pythonhosted.org/packages/65/d9/f0794aa1c74ceabc780fe17f6c338456bbc4e96bd950f2e969f48ac6fb20/coverage-7.13.1-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:916abf1ac5cf7eb16bc540a5bf75c71c43a676f5c52fcb9fe75a2bd75fb944e8", size = 260445, upload-time = "2025-12-28T15:42:38.646Z" },
+ { url = "https://files.pythonhosted.org/packages/49/23/184b22a00d9bb97488863ced9454068c79e413cb23f472da6cbddc6cfc52/coverage-7.13.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:776483fd35b58d8afe3acbd9988d5de592ab6da2d2a865edfdbc9fdb43e7c486", size = 263357, upload-time = "2025-12-28T15:42:40.788Z" },
+ { url = "https://files.pythonhosted.org/packages/7d/bd/58af54c0c9199ea4190284f389005779d7daf7bf3ce40dcd2d2b2f96da69/coverage-7.13.1-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:b6f3b96617e9852703f5b633ea01315ca45c77e879584f283c44127f0f1ec564", size = 260959, upload-time = "2025-12-28T15:42:42.808Z" },
+ { url = "https://files.pythonhosted.org/packages/4b/2a/6839294e8f78a4891bf1df79d69c536880ba2f970d0ff09e7513d6e352e9/coverage-7.13.1-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:bd63e7b74661fed317212fab774e2a648bc4bb09b35f25474f8e3325d2945cd7", size = 259792, upload-time = "2025-12-28T15:42:44.818Z" },
+ { url = "https://files.pythonhosted.org/packages/ba/c3/528674d4623283310ad676c5af7414b9850ab6d55c2300e8aa4b945ec554/coverage-7.13.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:933082f161bbb3e9f90d00990dc956120f608cdbcaeea15c4d897f56ef4fe416", size = 262123, upload-time = "2025-12-28T15:42:47.108Z" },
+ { url = "https://files.pythonhosted.org/packages/06/c5/8c0515692fb4c73ac379d8dc09b18eaf0214ecb76ea6e62467ba7a1556ff/coverage-7.13.1-cp314-cp314t-win32.whl", hash = "sha256:18be793c4c87de2965e1c0f060f03d9e5aff66cfeae8e1dbe6e5b88056ec153f", size = 222562, upload-time = "2025-12-28T15:42:49.144Z" },
+ { url = "https://files.pythonhosted.org/packages/05/0e/c0a0c4678cb30dac735811db529b321d7e1c9120b79bd728d4f4d6b010e9/coverage-7.13.1-cp314-cp314t-win_amd64.whl", hash = "sha256:0e42e0ec0cd3e0d851cb3c91f770c9301f48647cb2877cb78f74bdaa07639a79", size = 223670, upload-time = "2025-12-28T15:42:51.218Z" },
+ { url = "https://files.pythonhosted.org/packages/f5/5f/b177aa0011f354abf03a8f30a85032686d290fdeed4222b27d36b4372a50/coverage-7.13.1-cp314-cp314t-win_arm64.whl", hash = "sha256:eaecf47ef10c72ece9a2a92118257da87e460e113b83cc0d2905cbbe931792b4", size = 221707, upload-time = "2025-12-28T15:42:53.034Z" },
+ { url = "https://files.pythonhosted.org/packages/cc/48/d9f421cb8da5afaa1a64570d9989e00fb7955e6acddc5a12979f7666ef60/coverage-7.13.1-py3-none-any.whl", hash = "sha256:2016745cb3ba554469d02819d78958b571792bb68e31302610e898f80dd3a573", size = 210722, upload-time = "2025-12-28T15:42:54.901Z" },
]
[package.optional-dependencies]
@@ -1047,30 +1047,30 @@ wheels = [
[[package]]
name = "cuda-bindings"
-version = "13.1.0"
+version = "13.1.1"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "cuda-pathfinder" },
]
wheels = [
- { url = "https://files.pythonhosted.org/packages/e0/13/b0e2f65a2dc1e7ae9a0662e2de2382af92f944f896fa981e6fd4d865ddec/cuda_bindings-13.1.0-cp310-cp310-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9dba661b641c4559ae28b3bd17620bdee469aec706feafcf666853c1b2df35eb", size = 12115816, upload-time = "2025-12-04T22:56:13.537Z" },
- { url = "https://files.pythonhosted.org/packages/b4/bf/ad971786911f91b73b82c56d17679d9068ffbedc55fcb485c5791f83801d/cuda_bindings-13.1.0-cp310-cp310-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:122d3f0dde611e84bb0448cb2a1760b3022da5a09b24cc27ec6403fe3d3ebcb5", size = 12515163, upload-time = "2025-12-04T22:56:15.657Z" },
- { url = "https://files.pythonhosted.org/packages/2e/2c/835ca2ca66328e2d8adde888dbc94b20e947d315f8a7b1aeb181aa956cf7/cuda_bindings-13.1.0-cp310-cp310-win_amd64.whl", hash = "sha256:7949148190c0611b3eae87698a8778ab4d1c1f220a066c83de01251a6ce577ae", size = 11314923, upload-time = "2025-12-04T22:56:18.149Z" },
- { url = "https://files.pythonhosted.org/packages/75/69/a7ee82619cfaf5ed8409c9628825063a9d5b8a1589e2ddb4d164b065e818/cuda_bindings-13.1.0-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4f099d5e7e952ddafc62a1abad1ae30f46fcd3ebe683bac628c927d8050779e3", size = 12118903, upload-time = "2025-12-04T22:56:20.973Z" },
- { url = "https://files.pythonhosted.org/packages/78/9f/6e563566a55990dc52f42968ab3a2b17897959459ad95f9df5aef170e2a1/cuda_bindings-13.1.0-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e4e4e396ebb051bd268ec3f97c7cf350fbf2eee8ff5cf0b572218ea52fde0960", size = 12541331, upload-time = "2025-12-04T22:56:22.991Z" },
- { url = "https://files.pythonhosted.org/packages/e5/50/996454cddab45ed670d150f9cce4f4dbfaffdc2e833f27b0d29be066fe8b/cuda_bindings-13.1.0-cp311-cp311-win_amd64.whl", hash = "sha256:556867e171ada3bd7716133f9b689a89c5486110757d44f0884e0f1c1cf5cb98", size = 11344306, upload-time = "2025-12-04T22:56:26.462Z" },
- { url = "https://files.pythonhosted.org/packages/11/8b/c30de01881ade1651ca71c6a111483e256b2064b53e9e29e04cd42a47ee6/cuda_bindings-13.1.0-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f7a12d696987854b04a184c4e5e61f043b56bb94469d27c546a8aec55d9aa6be", size = 11833630, upload-time = "2025-12-04T22:56:28.401Z" },
- { url = "https://files.pythonhosted.org/packages/fa/9a/ca6cf7fb8a287c55596393eee05bd205f79a0270987447402d80f22db382/cuda_bindings-13.1.0-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6f63214291d316e18152e277edc1f159083a931f9936e2e3d854da8155a5f0f7", size = 12272962, upload-time = "2025-12-04T22:56:30.465Z" },
- { url = "https://files.pythonhosted.org/packages/c2/b6/686b16cf2dddc3403b88a0396b2e411bfdd730a29ca24e1f17f5e982bc1f/cuda_bindings-13.1.0-cp312-cp312-win_amd64.whl", hash = "sha256:61510bfda2d4cd6afb589d39806f82e184beced26a5d8a12db10770ccbc99754", size = 11449081, upload-time = "2025-12-04T22:56:32.762Z" },
- { url = "https://files.pythonhosted.org/packages/c8/a0/ca376832b8727f9b449f307755517c839d036695b330b821b1b6193e327c/cuda_bindings-13.1.0-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9d3bdbb1b860f37450b6280d6a4d27716b952f480ef0be6daa27d04d9c4825ac", size = 11836584, upload-time = "2025-12-04T22:56:34.573Z" },
- { url = "https://files.pythonhosted.org/packages/71/1a/97c4246984ea6c717b4eeb139b9abcf4831b7c3557ae28eccc9bce7e20a3/cuda_bindings-13.1.0-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:72b4236637ece577b0dc8528a6a60beecb19e64ca9c6818386f982200e461057", size = 12221797, upload-time = "2025-12-04T22:56:36.65Z" },
- { url = "https://files.pythonhosted.org/packages/dc/f8/50d0bed242ca79edf3fbf20f95aa0a11225b4b814fe56f67ccfa5511832e/cuda_bindings-13.1.0-cp313-cp313-win_amd64.whl", hash = "sha256:34181796a768cd7bb6a2e7407db76eed42a0d7e48a4b24aed502e9b485fcb0d5", size = 11425811, upload-time = "2025-12-04T22:56:38.7Z" },
- { url = "https://files.pythonhosted.org/packages/56/a0/95bd9447d39dcfe61ef831471aef21eeb4685b34ca914cdc60baccaa8029/cuda_bindings-13.1.0-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:12106d5bb73fff4c25740ae1bd56af9630306a7fff6a700de54015083ba3831b", size = 11900651, upload-time = "2025-12-04T22:56:40.649Z" },
- { url = "https://files.pythonhosted.org/packages/75/27/440110cce146715d862c05e414e249bc9f57e450ba0d3f5052c7856ca0bc/cuda_bindings-13.1.0-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:80cccf2ce884f70d7d41d34fe622b4a8e6cadcc11ce943c7c5bedfb9285ac61c", size = 12264562, upload-time = "2025-12-04T22:56:42.638Z" },
- { url = "https://files.pythonhosted.org/packages/57/52/619e37982f83c0c7515d8c2c0708d48ce1de16879ca3c63dc6f05a142913/cuda_bindings-13.1.0-cp314-cp314-win_amd64.whl", hash = "sha256:2fa92fad8c21cdeaa2e58a733e5013a7377840e2fbc0239757409141b90704c2", size = 11547446, upload-time = "2025-12-04T22:56:44.795Z" },
- { url = "https://files.pythonhosted.org/packages/f0/3e/23a7180f5eec1c914ce2a8f29a968de203047b31594db0828daf1f25fb03/cuda_bindings-13.1.0-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a1a73affa8b33e677b876cd570120023497a43a1045303a3ebf152749d0c5a93", size = 11799779, upload-time = "2025-12-04T22:56:47.055Z" },
- { url = "https://files.pythonhosted.org/packages/a7/1f/1c08cd0db90d90c5b65acae80ac719b2c6aa145e260e6abece985978ab0a/cuda_bindings-13.1.0-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7803f0a2b9085eb44805c8226907b7091d3afba5453c1c1ef6710f0151521ddb", size = 12188089, upload-time = "2025-12-04T22:56:48.998Z" },
- { url = "https://files.pythonhosted.org/packages/d0/c6/b8345151fef3bb8dc48d09487c48c68c8aea7d68704cd6452e1cbc9c83c7/cuda_bindings-13.1.0-cp314-cp314t-win_amd64.whl", hash = "sha256:2ac82549be8c9b4490f876caaeff5be95dd29147832982d2a4cccb906dab8373", size = 12036656, upload-time = "2025-12-04T22:56:51.444Z" },
+ { url = "https://files.pythonhosted.org/packages/60/63/579402b642f5b9b8ceb79e456b39b5771f27e132a8af3b140e54d69790fc/cuda_bindings-13.1.1-cp310-cp310-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4400370a83f1538e25ed4c18c34a0e9d5fad39741e282e69ce24d1479a11017d", size = 15777291, upload-time = "2025-12-09T22:05:41.109Z" },
+ { url = "https://files.pythonhosted.org/packages/df/6a/3a293cfb01cd4964444a0f75917b6edb1c31ea69d0230e329975da6991ba/cuda_bindings-13.1.1-cp310-cp310-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:81f92500e2f6aec2dac00a5a1ce77d5aa77ea77b606dc484d951f1f2cc3eaa13", size = 16311623, upload-time = "2025-12-09T22:05:43.897Z" },
+ { url = "https://files.pythonhosted.org/packages/72/b8/a5860b9e70faa53658236dc61efc3ecc51846beff4a0b73de9151130ff98/cuda_bindings-13.1.1-cp310-cp310-win_amd64.whl", hash = "sha256:3f5bb8190267216f96597235252087accac4cbccefd1b60756cced114b2d6754", size = 15185932, upload-time = "2025-12-09T22:05:46.089Z" },
+ { url = "https://files.pythonhosted.org/packages/b0/58/b8d4c7c5fb29ba46088a7e78d1065484219f8fe41a08adc4a85b1ee56149/cuda_bindings-13.1.1-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a5f5a6ade0ad45096568bc4dd1eb3377b65884d29124338fe9a4353130ef6631", size = 15771605, upload-time = "2025-12-09T22:05:48.266Z" },
+ { url = "https://files.pythonhosted.org/packages/17/af/710403f76f2d608d483d87089465e1f666351641dbd73d19bd025e652bad/cuda_bindings-13.1.1-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9348f69b03b257f07159dd4c869615e139722c2bd81e96c66f6b8f77615efd82", size = 16338970, upload-time = "2025-12-09T22:05:50.598Z" },
+ { url = "https://files.pythonhosted.org/packages/64/1c/e7ea27d4cb7d07331c88e3bbed3cacc947d2237471801086c7447b3e195d/cuda_bindings-13.1.1-cp311-cp311-win_amd64.whl", hash = "sha256:ec33b84f4bd65a86a734427f2b9cb8f221bedab2c4cfb681488cabc82f1d64ab", size = 15210672, upload-time = "2025-12-09T22:05:53.369Z" },
+ { url = "https://files.pythonhosted.org/packages/53/3d/c8ed9d169843091f3f0d6b8218e826fd59520a37e0434c204feada597988/cuda_bindings-13.1.1-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1e75ad0cb863330df784236d289612d71ca855c013d19ae00e5693574abd6915", size = 15530160, upload-time = "2025-12-09T22:05:55.386Z" },
+ { url = "https://files.pythonhosted.org/packages/4a/8e/368295623ee43fba622909d780fbb6863efc1638dff55f67a0f04eac6470/cuda_bindings-13.1.1-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:25785d1a3cdcd98f151240fd5efd025609319a6720a217dee2a929241749d488", size = 16110386, upload-time = "2025-12-09T22:05:57.71Z" },
+ { url = "https://files.pythonhosted.org/packages/60/1f/ecc4701ade3e85f091c625a920574527b9daf7fb354189fbfbc5516af6cd/cuda_bindings-13.1.1-cp312-cp312-win_amd64.whl", hash = "sha256:ccde9c95c0e953b31fe7731bb08da9d0a34b1770498df9a3c156fdfdbe3951ad", size = 15250028, upload-time = "2025-12-09T22:06:00.346Z" },
+ { url = "https://files.pythonhosted.org/packages/fe/c1/0ee8fd94bab7e23116e0e3da8c0902e299f3d9edc95f1d7d8ef894c897ed/cuda_bindings-13.1.1-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0c9822a57c8f952dc367aacd7c32fe4cb17371104383606f455ea74635bff4c7", size = 15421116, upload-time = "2025-12-09T22:06:02.994Z" },
+ { url = "https://files.pythonhosted.org/packages/f3/c2/f272fad414b96299e010dcbe510cf17fc25deaf3443e0fdb55020a8298a3/cuda_bindings-13.1.1-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5837f5ea422c5653626dcfe22e9ab68142cd19af9e67a226100f224cc25a1b99", size = 15940152, upload-time = "2025-12-09T22:06:05.079Z" },
+ { url = "https://files.pythonhosted.org/packages/2a/56/433093bec0121f031edb582ea3a72f71031e8fbebecaaf329809344da4c7/cuda_bindings-13.1.1-cp313-cp313-win_amd64.whl", hash = "sha256:9e4f348cd7a779657d51e6f71aac3965fb1738f40ff3bbe75265a3242fd6f29f", size = 15216463, upload-time = "2025-12-09T22:06:07.296Z" },
+ { url = "https://files.pythonhosted.org/packages/de/38/40416d037ed25db68f1dbd50e0232775a62d90c9f25af22b196c0a13b88c/cuda_bindings-13.1.1-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:86258fe1b0d3998bea7f57dc891569e4996705b8dd00366e44c722d0a29b2090", size = 15498927, upload-time = "2025-12-09T22:06:09.476Z" },
+ { url = "https://files.pythonhosted.org/packages/ac/3f/f1f88b6cdb7d41ba076f8ff10edf6d3bd17e740da9a163544b43d6349653/cuda_bindings-13.1.1-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:daf8468fd603b2724c2d16cbd499348c64916ed72b1d04643f1660ce13cd12ae", size = 15984539, upload-time = "2025-12-09T22:06:11.882Z" },
+ { url = "https://files.pythonhosted.org/packages/f6/33/7739cc5e9a3373df8e7dea9060528bee5f70cf6e28b9c14f765502816c71/cuda_bindings-13.1.1-cp314-cp314-win_amd64.whl", hash = "sha256:f2e079182014dbc162562b46467815272c14c7afe5b988978fa968728b0ac726", size = 15373212, upload-time = "2025-12-09T22:06:13.989Z" },
+ { url = "https://files.pythonhosted.org/packages/9e/0a/5c6d514e566ff86c4054bbbb6554bf49b9c55fefbc934eb456faecab53c9/cuda_bindings-13.1.1-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d0cd96a6ec00a78235947bff9462b2139bc5b83ce8e297d865802f0b52d1e23d", size = 15403944, upload-time = "2025-12-09T22:06:16.315Z" },
+ { url = "https://files.pythonhosted.org/packages/0b/5b/319cfa491a685d4d4757aa24223b6dbc0976954afac42f49fc47290ba6a3/cuda_bindings-13.1.1-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9ff465829c6c394c2b4047250324a19925cf8c44633345b2746a4741e07bf827", size = 15911462, upload-time = "2025-12-09T22:06:18.403Z" },
+ { url = "https://files.pythonhosted.org/packages/e3/5c/38b92080c5b6c4ddb09f0be2536123f81c7e9e1a89e4573f20cb00347ee3/cuda_bindings-13.1.1-cp314-cp314t-win_amd64.whl", hash = "sha256:8205eee6b8b458a2110c0384923ace206855d0f1b436fc1b145fcbaa1653b501", size = 16044390, upload-time = "2025-12-09T22:06:20.945Z" },
]
[[package]]
@@ -1083,57 +1083,57 @@ wheels = [
[[package]]
name = "cuda-python"
-version = "13.1.0"
+version = "13.1.1"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "cuda-bindings" },
{ name = "cuda-pathfinder" },
]
wheels = [
- { url = "https://files.pythonhosted.org/packages/3d/4d/876c2f87d34ccde0f11688f07e98a43cb3498cc115ee85fc7ae79711b7ae/cuda_python-13.1.0-py3-none-any.whl", hash = "sha256:19ce93ab3c8b2116ebe23c87fe023d82df0766af4f956582a42d3482a2787e33", size = 7613, upload-time = "2025-12-04T23:03:37.916Z" },
+ { url = "https://files.pythonhosted.org/packages/cd/08/b5e3b9822662d72d540d830531e3ab6a7cabbda3dd56175696aabccfeb76/cuda_python-13.1.1-py3-none-any.whl", hash = "sha256:944cc4fe6482673d28dd545797a28840945a1668739328fa2ad1e9be4f7050d9", size = 8038, upload-time = "2025-12-09T22:13:10.719Z" },
]
[[package]]
name = "cython"
-version = "3.2.2"
+version = "3.2.4"
source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/29/17/55fc687ba986f2210298fa2f60fec265fa3004c3f9a1e958ea1fe2d4e061/cython-3.2.2.tar.gz", hash = "sha256:c3add3d483acc73129a61d105389344d792c17e7c1cee24863f16416bd071634", size = 3275797, upload-time = "2025-11-30T12:48:20.942Z" }
-wheels = [
- { url = "https://files.pythonhosted.org/packages/39/1f/98673761cc20b05376e559fbcc201268d5ee9b0234bcf582ca547f1088ed/cython-3.2.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:b5afac4e77e71a9010dc7fd3191ced00f9b12b494dd7525c140781054ce63a73", size = 2958838, upload-time = "2025-11-30T12:48:30.259Z" },
- { url = "https://files.pythonhosted.org/packages/37/4c/bff2cef3a1873d5f265f200c6a31ea070b914c13ed8775bfe700760bb0a2/cython-3.2.2-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9cd2ede6af225499ad22888dbfb13b92d71fc1016f401ee637559a5831b177c2", size = 3367932, upload-time = "2025-11-30T12:48:33.522Z" },
- { url = "https://files.pythonhosted.org/packages/2c/58/5cdb26758b5c76e47388da0f9a6930e1a58639bf69f4827e91e77308092b/cython-3.2.2-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8c9265b3e84ae2d999b7c3165c683e366bbbbbe4346468055ca2366fe013f2df", size = 3536550, upload-time = "2025-11-30T12:48:35.714Z" },
- { url = "https://files.pythonhosted.org/packages/32/5e/2e5ff6a51c47adb2b755ca9a1d9fa0aef3b408cb1a0274a3d0cbeec301b5/cython-3.2.2-cp310-cp310-win_amd64.whl", hash = "sha256:d7b3447b2005dffc5f276d420a480d2b57d15091242652d410b6a46fb00ed251", size = 2765189, upload-time = "2025-11-30T12:48:37.58Z" },
- { url = "https://files.pythonhosted.org/packages/e0/ba/d785f60564a43bddbb7316134252a55d67ff6f164f0be90c4bf31482da82/cython-3.2.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:d140c2701cbb8cf960300cf1b67f3b4fa9d294d32e51b85f329bff56936a82fd", size = 2951181, upload-time = "2025-11-30T12:48:39.723Z" },
- { url = "https://files.pythonhosted.org/packages/8c/36/277ea908bd7a326adcce5f67581926783ec11a4402ead0719e72555da2e7/cython-3.2.2-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:50bbaabee733fd2780985e459fc20f655e02def83e8eff10220ad88455a34622", size = 3243194, upload-time = "2025-11-30T12:48:41.862Z" },
- { url = "https://files.pythonhosted.org/packages/74/87/7776495bbff80bdd7754f4849bfde126b1d97741df5bc58a3a00af641747/cython-3.2.2-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a9509f1e9c41c86b790cff745bb31927bbc861662a3b462596d71d3d2a578abb", size = 3378096, upload-time = "2025-11-30T12:48:43.584Z" },
- { url = "https://files.pythonhosted.org/packages/a5/d0/fabb9f4767f7101f1c0d6b23116d6d250148f656dac152afd792756661a8/cython-3.2.2-cp311-cp311-win_amd64.whl", hash = "sha256:034ab96cb8bc8e7432bc27491f8d66f51e435b1eb21ddc03aa844be8f21ad847", size = 2768932, upload-time = "2025-11-30T12:48:45.193Z" },
- { url = "https://files.pythonhosted.org/packages/57/0f/6fd78dc581373722bb9dedfc90c35b59ba88af988756315af227a877c7a2/cython-3.2.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:692a41c8fe06fb2dc55ca2c8d71c80c469fd16fe69486ed99f3b3cbb2d3af83f", size = 2968037, upload-time = "2025-11-30T12:48:47.279Z" },
- { url = "https://files.pythonhosted.org/packages/b0/52/50b6263c2fbad73aae8911ce54641ee1739d430a0592d3b3510591d7842b/cython-3.2.2-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:098590c1dc309f8a0406ade031963a95a87714296b425539f9920aebf924560d", size = 3223137, upload-time = "2025-11-30T12:48:48.951Z" },
- { url = "https://files.pythonhosted.org/packages/d6/44/4e34d161674c9162c6eb9ddef0cd69d41d92ae7e6dee3945fed3a6871ebe/cython-3.2.2-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a3898c076e9c458bcb3e4936187919fda5f5365fe4c567d35d2b003444b6f3fe", size = 3390943, upload-time = "2025-11-30T12:48:51.125Z" },
- { url = "https://files.pythonhosted.org/packages/62/8a/ffc2df024c1341737008fbaf0fbea51ef983a7146b43b84a239f197cf005/cython-3.2.2-cp312-cp312-win_amd64.whl", hash = "sha256:2b910b89a2a71004064c5e890b9512a251eda63fae252caa0feb9835057035f9", size = 2756403, upload-time = "2025-11-30T12:48:52.929Z" },
- { url = "https://files.pythonhosted.org/packages/a2/4f/b5355918962ec28b376eb8e357c718d58baf32d6df7017be8d147dd4ba29/cython-3.2.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:aa24cd0bdab27ca099b2467806c684404add597c1108e07ddf7b6471653c85d7", size = 2958578, upload-time = "2025-11-30T12:48:55.354Z" },
- { url = "https://files.pythonhosted.org/packages/46/21/a8038c8253e7a5241ed1db6d031bac586f7a502d92f487124abbc3f3e94f/cython-3.2.2-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:60f4aa425e1ff98abf8d965ae7020f06dd2cbc01dbd945137d2f9cca4ff0524a", size = 3212479, upload-time = "2025-11-30T12:48:57.567Z" },
- { url = "https://files.pythonhosted.org/packages/57/c1/76928c07176a4402c74d5b304936ad8ee167dd04a07cf7dca545e8c25f9b/cython-3.2.2-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a473df474ba89e9fee81ee82b31062a267f9e598096b222783477e56d02ad12c", size = 3374773, upload-time = "2025-11-30T12:48:59.318Z" },
- { url = "https://files.pythonhosted.org/packages/fa/cb/ce641e07ba9c0cde8468e83e0214fb87020b74ba34dbb9dfe8d250a327f5/cython-3.2.2-cp313-cp313-win_amd64.whl", hash = "sha256:b4df52101209817fde7284cf779156f79142fb639b1d7840f11680ff4bb30604", size = 2754492, upload-time = "2025-11-30T12:49:01.029Z" },
- { url = "https://files.pythonhosted.org/packages/c0/f2/cd60f639f0fde38b71319d7b6808e1ff17a6fd7f3feaff475b866a5c0aef/cython-3.2.2-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:177faf4d61e9f2d4d2db61194ac9ec16d3fe3041c1b6830f871a01935319eeb3", size = 2969023, upload-time = "2025-11-30T12:49:02.734Z" },
- { url = "https://files.pythonhosted.org/packages/5d/45/6f155a9ad125536d8f30716c4d7571caae73ec811039d3ae33f9b535090d/cython-3.2.2-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8db28aef793c81dc69383b619ca508668998aaf099cd839d3cbae85184cce744", size = 3258270, upload-time = "2025-11-30T12:49:04.878Z" },
- { url = "https://files.pythonhosted.org/packages/af/7e/022c25886fdc3ff6a005b6ae4a1c3d8522006bb738367aa5bd6c2590130b/cython-3.2.2-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3de43a5786033a27fae1c882feb5ff0d023c38b83356e6800c1be0bcd6cf9f11", size = 3384504, upload-time = "2025-11-30T12:49:07.078Z" },
- { url = "https://files.pythonhosted.org/packages/b6/07/1e3e4faf6f785d5ba053e9d6320b3f338162dc122c27a7c540b49615fc39/cython-3.2.2-cp314-cp314-win_amd64.whl", hash = "sha256:fed44d0ab2d36f1b0301c770b0dafec23bcb9700d58e7769cd6d9136b3304c11", size = 2791504, upload-time = "2025-11-30T12:49:08.729Z" },
- { url = "https://files.pythonhosted.org/packages/f4/69/5430879d35235ec3d5ffd778862173b6419390509ae4e37a72bdd45d9e86/cython-3.2.2-cp39-abi3-macosx_10_9_x86_64.whl", hash = "sha256:a6387e3ad31342443916db9a419509935fddd8d4cbac34aab9c895ae55326a56", size = 2874031, upload-time = "2025-11-30T12:49:18.34Z" },
- { url = "https://files.pythonhosted.org/packages/51/fa/584f4b56b35b3e7a43dc16603dd722cb5528484da67c27136534b782827b/cython-3.2.2-cp39-abi3-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:436eb562d0affbc0b959f62f3f9c1ed251b9499e4f29c1d19514ae859894b6bf", size = 3210813, upload-time = "2025-11-30T12:49:20.55Z" },
- { url = "https://files.pythonhosted.org/packages/d1/d4/063c34a34d9ef54836a5dafb100b8f4fdbdaa63942913fe93f9eb93a11a2/cython-3.2.2-cp39-abi3-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:f560ff3aea5b5df93853ec7bf1a1e9623d6d511f4192f197559aca18fca43392", size = 2855611, upload-time = "2025-11-30T12:49:22.303Z" },
- { url = "https://files.pythonhosted.org/packages/b9/44/c0b8854e0bf6d444c88cc2050f550d964596daea20eaf1bc592fcfde2782/cython-3.2.2-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:d8c93fe128b58942832b1fcac96e48f93c2c69b569eff0d38d30fb5995fecfa0", size = 2992824, upload-time = "2025-11-30T12:49:24.02Z" },
- { url = "https://files.pythonhosted.org/packages/90/6f/741186935c52de99acf4d7fad5c3dcf28d980b4c95d171d9618f9c399316/cython-3.2.2-cp39-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:b4fe499eed7cd70b2aa4e096b9ce2588f5e6fdf049b46d40a5e55efcde6e4904", size = 2890389, upload-time = "2025-11-30T12:49:25.783Z" },
- { url = "https://files.pythonhosted.org/packages/5c/79/3e487876addd0d69c148a529f3973c1942498ad39cede1e63565676064ed/cython-3.2.2-cp39-abi3-musllinux_1_2_i686.whl", hash = "sha256:14432d7f207245a3c35556155873f494784169297b28978a6204f1c60d31553e", size = 3224881, upload-time = "2025-11-30T12:49:27.484Z" },
- { url = "https://files.pythonhosted.org/packages/15/b9/d9a103feb74d04579c6bde7b0cad6d5f45c002d843ca70788a5758707b68/cython-3.2.2-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:820c4a99dbf6b3e6c0300be42b4040b501eff0e1feeb80cfa52c48a346fb0df2", size = 3114308, upload-time = "2025-11-30T12:49:29.292Z" },
- { url = "https://files.pythonhosted.org/packages/18/56/90445707cff62ab72136857a0134c5e50f9c73920c1a3af5218dfdae1c19/cython-3.2.2-cp39-abi3-win32.whl", hash = "sha256:826cad0ad43ab05a26e873b5d625f64d458dc739ec6fdeecab848b60a91c4252", size = 2435212, upload-time = "2025-11-30T12:49:32.179Z" },
- { url = "https://files.pythonhosted.org/packages/44/54/25a98c2731521ac9fc18e17d79a0e7d58164d4db398f09e1bd24cdd27ed1/cython-3.2.2-cp39-abi3-win_arm64.whl", hash = "sha256:5f818d40bbcf17e2089e2de7840f0de1c0ca527acf9b044aba79d5f5d8a5bdba", size = 2440536, upload-time = "2025-11-30T12:49:34.109Z" },
- { url = "https://files.pythonhosted.org/packages/76/f2/98fd8d0b514622a789fd2824b59bd6041b799aaeeba14a8d92d52f6654dd/cython-3.2.2-py3-none-any.whl", hash = "sha256:13b99ecb9482aff6a6c12d1ca6feef6940c507af909914b49f568de74fa965fb", size = 1255106, upload-time = "2025-11-30T12:48:18.454Z" },
+sdist = { url = "https://files.pythonhosted.org/packages/91/85/7574c9cd44b69a27210444b6650f6477f56c75fee1b70d7672d3e4166167/cython-3.2.4.tar.gz", hash = "sha256:84226ecd313b233da27dc2eb3601b4f222b8209c3a7216d8733b031da1dc64e6", size = 3280291, upload-time = "2026-01-04T14:14:14.473Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/a1/10/720e0fb84eab4c927c4dd6b61eb7993f7732dd83d29ba6d73083874eade9/cython-3.2.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:02cb0cc0f23b9874ad262d7d2b9560aed9c7e2df07b49b920bda6f2cc9cb505e", size = 2960836, upload-time = "2026-01-04T14:14:51.103Z" },
+ { url = "https://files.pythonhosted.org/packages/7d/3d/b26f29092c71c36e0462752885bdfb18c23c176af4de953fdae2772a8941/cython-3.2.4-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f136f379a4a54246facd0eb6f1ee15c3837cb314ce87b677582ec014db4c6845", size = 3370134, upload-time = "2026-01-04T14:14:53.627Z" },
+ { url = "https://files.pythonhosted.org/packages/56/9e/539fb0d09e4f5251b5b14f8daf77e71fee021527f1013791038234618b6b/cython-3.2.4-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:35ab0632186057406ec729374c737c37051d2eacad9d515d94e5a3b3e58a9b02", size = 3537552, upload-time = "2026-01-04T14:14:56.852Z" },
+ { url = "https://files.pythonhosted.org/packages/10/c6/82d19a451c050d1be0f05b1a3302267463d391db548f013ee88b5348a8e9/cython-3.2.4-cp310-cp310-win_amd64.whl", hash = "sha256:ca2399dc75796b785f74fb85c938254fa10c80272004d573c455f9123eceed86", size = 2766191, upload-time = "2026-01-04T14:14:58.709Z" },
+ { url = "https://files.pythonhosted.org/packages/85/cc/8f06145ec3efa121c8b1b67f06a640386ddacd77ee3e574da582a21b14ee/cython-3.2.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:ff9af2134c05e3734064808db95b4dd7341a39af06e8945d05ea358e1741aaed", size = 2953769, upload-time = "2026-01-04T14:15:00.361Z" },
+ { url = "https://files.pythonhosted.org/packages/55/b0/706cf830eddd831666208af1b3058c2e0758ae157590909c1f634b53bed9/cython-3.2.4-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:67922c9de058a0bfb72d2e75222c52d09395614108c68a76d9800f150296ddb3", size = 3243841, upload-time = "2026-01-04T14:15:02.066Z" },
+ { url = "https://files.pythonhosted.org/packages/ac/25/58893afd4ef45f79e3d4db82742fa4ff874b936d67a83c92939053920ccd/cython-3.2.4-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b362819d155fff1482575e804e43e3a8825332d32baa15245f4642022664a3f4", size = 3378083, upload-time = "2026-01-04T14:15:04.248Z" },
+ { url = "https://files.pythonhosted.org/packages/32/e4/424a004d7c0d8a4050c81846ebbd22272ececfa9a498cb340aa44fccbec2/cython-3.2.4-cp311-cp311-win_amd64.whl", hash = "sha256:1a64a112a34ec719b47c01395647e54fb4cf088a511613f9a3a5196694e8e382", size = 2769990, upload-time = "2026-01-04T14:15:06.53Z" },
+ { url = "https://files.pythonhosted.org/packages/91/4d/1eb0c7c196a136b1926f4d7f0492a96c6fabd604d77e6cd43b56a3a16d83/cython-3.2.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:64d7f71be3dd6d6d4a4c575bb3a4674ea06d1e1e5e4cd1b9882a2bc40ed3c4c9", size = 2970064, upload-time = "2026-01-04T14:15:08.567Z" },
+ { url = "https://files.pythonhosted.org/packages/03/1c/46e34b08bea19a1cdd1e938a4c123e6299241074642db9d81983cef95e9f/cython-3.2.4-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:869487ea41d004f8b92171f42271fbfadb1ec03bede3158705d16cd570d6b891", size = 3226757, upload-time = "2026-01-04T14:15:10.812Z" },
+ { url = "https://files.pythonhosted.org/packages/12/33/3298a44d201c45bcf0d769659725ae70e9c6c42adf8032f6d89c8241098d/cython-3.2.4-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:55b6c44cd30821f0b25220ceba6fe636ede48981d2a41b9bbfe3c7902ce44ea7", size = 3388969, upload-time = "2026-01-04T14:15:12.45Z" },
+ { url = "https://files.pythonhosted.org/packages/bb/f3/4275cd3ea0a4cf4606f9b92e7f8766478192010b95a7f516d1b7cf22cb10/cython-3.2.4-cp312-cp312-win_amd64.whl", hash = "sha256:767b143704bdd08a563153448955935844e53b852e54afdc552b43902ed1e235", size = 2756457, upload-time = "2026-01-04T14:15:14.67Z" },
+ { url = "https://files.pythonhosted.org/packages/18/b5/1cfca43b7d20a0fdb1eac67313d6bb6b18d18897f82dd0f17436bdd2ba7f/cython-3.2.4-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:28e8075087a59756f2d059273184b8b639fe0f16cf17470bd91c39921bc154e0", size = 2960506, upload-time = "2026-01-04T14:15:16.733Z" },
+ { url = "https://files.pythonhosted.org/packages/71/bb/8f28c39c342621047fea349a82fac712a5e2b37546d2f737bbde48d5143d/cython-3.2.4-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:03893c88299a2c868bb741ba6513357acd104e7c42265809fd58dce1456a36fc", size = 3213148, upload-time = "2026-01-04T14:15:18.804Z" },
+ { url = "https://files.pythonhosted.org/packages/7a/d2/16fa02f129ed2b627e88d9d9ebd5ade3eeb66392ae5ba85b259d2d52b047/cython-3.2.4-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f81eda419b5ada7b197bbc3c5f4494090e3884521ffd75a3876c93fbf66c9ca8", size = 3375764, upload-time = "2026-01-04T14:15:20.817Z" },
+ { url = "https://files.pythonhosted.org/packages/91/3f/deb8f023a5c10c0649eb81332a58c180fad27c7533bb4aae138b5bc34d92/cython-3.2.4-cp313-cp313-win_amd64.whl", hash = "sha256:83266c356c13c68ffe658b4905279c993d8a5337bb0160fa90c8a3e297ea9a2e", size = 2754238, upload-time = "2026-01-04T14:15:23.001Z" },
+ { url = "https://files.pythonhosted.org/packages/ee/d7/3bda3efce0c5c6ce79cc21285dbe6f60369c20364e112f5a506ee8a1b067/cython-3.2.4-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:d4b4fd5332ab093131fa6172e8362f16adef3eac3179fd24bbdc392531cb82fa", size = 2971496, upload-time = "2026-01-04T14:15:25.038Z" },
+ { url = "https://files.pythonhosted.org/packages/89/ed/1021ffc80b9c4720b7ba869aea8422c82c84245ef117ebe47a556bdc00c3/cython-3.2.4-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e3b5ac54e95f034bc7fb07313996d27cbf71abc17b229b186c1540942d2dc28e", size = 3256146, upload-time = "2026-01-04T14:15:26.741Z" },
+ { url = "https://files.pythonhosted.org/packages/0c/51/ca221ec7e94b3c5dc4138dcdcbd41178df1729c1e88c5dfb25f9d30ba3da/cython-3.2.4-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:90f43be4eaa6afd58ce20d970bb1657a3627c44e1760630b82aa256ba74b4acb", size = 3383458, upload-time = "2026-01-04T14:15:28.425Z" },
+ { url = "https://files.pythonhosted.org/packages/79/2e/1388fc0243240cd54994bb74f26aaaf3b2e22f89d3a2cf8da06d75d46ca2/cython-3.2.4-cp314-cp314-win_amd64.whl", hash = "sha256:983f9d2bb8a896e16fa68f2b37866ded35fa980195eefe62f764ddc5f9f5ef8e", size = 2791241, upload-time = "2026-01-04T14:15:30.448Z" },
+ { url = "https://files.pythonhosted.org/packages/0a/8b/fd393f0923c82be4ec0db712fffb2ff0a7a131707b842c99bf24b549274d/cython-3.2.4-cp39-abi3-macosx_10_9_x86_64.whl", hash = "sha256:36bf3f5eb56d5281aafabecbaa6ed288bc11db87547bba4e1e52943ae6961ccf", size = 2875622, upload-time = "2026-01-04T14:15:39.749Z" },
+ { url = "https://files.pythonhosted.org/packages/73/48/48530d9b9d64ec11dbe0dd3178a5fe1e0b27977c1054ecffb82be81e9b6a/cython-3.2.4-cp39-abi3-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:6d5267f22b6451eb1e2e1b88f6f78a2c9c8733a6ddefd4520d3968d26b824581", size = 3210669, upload-time = "2026-01-04T14:15:41.911Z" },
+ { url = "https://files.pythonhosted.org/packages/5e/91/4865fbfef1f6bb4f21d79c46104a53d1a3fa4348286237e15eafb26e0828/cython-3.2.4-cp39-abi3-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:3b6e58f73a69230218d5381817850ce6d0da5bb7e87eb7d528c7027cbba40b06", size = 2856835, upload-time = "2026-01-04T14:15:43.815Z" },
+ { url = "https://files.pythonhosted.org/packages/fa/39/60317957dbef179572398253f29d28f75f94ab82d6d39ea3237fb6c89268/cython-3.2.4-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:e71efb20048358a6b8ec604a0532961c50c067b5e63e345e2e359fff72feaee8", size = 2994408, upload-time = "2026-01-04T14:15:45.422Z" },
+ { url = "https://files.pythonhosted.org/packages/8d/30/7c24d9292650db4abebce98abc9b49c820d40fa7c87921c0a84c32f4efe7/cython-3.2.4-cp39-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:28b1e363b024c4b8dcf52ff68125e635cb9cb4b0ba997d628f25e32543a71103", size = 2891478, upload-time = "2026-01-04T14:15:47.394Z" },
+ { url = "https://files.pythonhosted.org/packages/86/70/03dc3c962cde9da37a93cca8360e576f904d5f9beecfc9d70b1f820d2e5f/cython-3.2.4-cp39-abi3-musllinux_1_2_i686.whl", hash = "sha256:31a90b4a2c47bb6d56baeb926948348ec968e932c1ae2c53239164e3e8880ccf", size = 3225663, upload-time = "2026-01-04T14:15:49.446Z" },
+ { url = "https://files.pythonhosted.org/packages/b1/97/10b50c38313c37b1300325e2e53f48ea9a2c078a85c0c9572057135e31d5/cython-3.2.4-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:e65e4773021f8dc8532010b4fbebe782c77f9a0817e93886e518c93bd6a44e9d", size = 3115628, upload-time = "2026-01-04T14:15:51.323Z" },
+ { url = "https://files.pythonhosted.org/packages/8f/b1/d6a353c9b147848122a0db370863601fdf56de2d983b5c4a6a11e6ee3cd7/cython-3.2.4-cp39-abi3-win32.whl", hash = "sha256:2b1f12c0e4798293d2754e73cd6f35fa5bbdf072bdc14bc6fc442c059ef2d290", size = 2437463, upload-time = "2026-01-04T14:15:53.787Z" },
+ { url = "https://files.pythonhosted.org/packages/2d/d8/319a1263b9c33b71343adfd407e5daffd453daef47ebc7b642820a8b68ed/cython-3.2.4-cp39-abi3-win_arm64.whl", hash = "sha256:3b8e62049afef9da931d55de82d8f46c9a147313b69d5ff6af6e9121d545ce7a", size = 2442754, upload-time = "2026-01-04T14:15:55.382Z" },
+ { url = "https://files.pythonhosted.org/packages/ff/fa/d3c15189f7c52aaefbaea76fb012119b04b9013f4bf446cb4eb4c26c4e6b/cython-3.2.4-py3-none-any.whl", hash = "sha256:732fc93bc33ae4b14f6afaca663b916c2fdd5dcbfad7114e17fb2434eeaea45c", size = 1257078, upload-time = "2026-01-04T14:14:12.373Z" },
]
[[package]]
name = "datasets"
-version = "4.4.1"
+version = "4.4.2"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "dill" },
@@ -1143,7 +1143,7 @@ dependencies = [
{ name = "huggingface-hub" },
{ name = "multiprocess" },
{ name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version < '3.11' and extra == 'extra-13-megatron-core-dev') or (python_full_version < '3.11' and extra == 'extra-13-megatron-core-lts') or (extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" },
- { name = "numpy", version = "2.3.5", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version >= '3.11' and extra == 'extra-13-megatron-core-dev') or (python_full_version >= '3.11' and extra == 'extra-13-megatron-core-lts') or (extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" },
+ { name = "numpy", version = "2.4.0", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version >= '3.11' and extra == 'extra-13-megatron-core-dev') or (python_full_version >= '3.11' and extra == 'extra-13-megatron-core-lts') or (extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" },
{ name = "packaging" },
{ name = "pandas" },
{ name = "pyarrow" },
@@ -1152,9 +1152,9 @@ dependencies = [
{ name = "tqdm" },
{ name = "xxhash" },
]
-sdist = { url = "https://files.pythonhosted.org/packages/93/bf/0dae295d6d1ba0b1a200a9dd216838464b5bbd05da01407cb1330b377445/datasets-4.4.1.tar.gz", hash = "sha256:80322699aa8c0bbbdb7caa87906da689c3c2e29523cff698775c67f28fdab1fc", size = 585341, upload-time = "2025-11-05T16:00:38.162Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/c4/54/9359803da96bc65439a28fbb014dc2c90b7d4d8034a93b72362b0d40191f/datasets-4.4.2.tar.gz", hash = "sha256:9de16e415c4ba4713eac0493f7c7dc74f3aa21599297f00cc6ddab409cb7b24b", size = 586474, upload-time = "2025-12-19T15:03:09.129Z" }
wheels = [
- { url = "https://files.pythonhosted.org/packages/3b/5e/6f8d874366788ad5d549e9ba258037d974dda6e004843be1bda794571701/datasets-4.4.1-py3-none-any.whl", hash = "sha256:c1163de5211e42546079ab355cc0250c7e6db16eb209ac5ac6252f801f596c44", size = 511591, upload-time = "2025-11-05T16:00:36.365Z" },
+ { url = "https://files.pythonhosted.org/packages/7b/b5/fefa518c809de7bced5cddb7c21c010da66fa2ae494bda96844a280cc6ce/datasets-4.4.2-py3-none-any.whl", hash = "sha256:6f5ef3417504d9cd663c71c1b90b9a494ff4c2076a2cd6a6e40ceee6ad95befc", size = 512268, upload-time = "2025-12-19T15:03:07.087Z" },
]
[[package]]
@@ -1249,10 +1249,10 @@ wheels = [
[[package]]
name = "emerging-optimizers"
version = "0.1.0"
-source = { git = "https://github.com/NVIDIA-NeMo/Emerging-Optimizers.git?rev=fb1add873e7851ec34b48581ea1b15761b73d189#fb1add873e7851ec34b48581ea1b15761b73d189" }
+source = { git = "https://github.com/NVIDIA-NeMo/Emerging-Optimizers.git?rev=v0.1.0#d5363b4a418128cd8111983b191c4b8869a9766b" }
dependencies = [
{ name = "absl-py" },
- { name = "torch", marker = "sys_platform == 'never' or (extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" },
+ { name = "torch", marker = "sys_platform == 'never'" },
{ name = "typing-extensions" },
]
@@ -1285,7 +1285,7 @@ wheels = [
[[package]]
name = "fastapi"
-version = "0.123.9"
+version = "0.128.0"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "annotated-doc" },
@@ -1293,9 +1293,9 @@ dependencies = [
{ name = "starlette" },
{ name = "typing-extensions" },
]
-sdist = { url = "https://files.pythonhosted.org/packages/2c/01/c3fb48c0135d89586a03c3e2c5bc04540dda52079a1af5cac4a63598efb9/fastapi-0.123.9.tar.gz", hash = "sha256:ab33d672d8e1cc6e0b49777eb73c32ccf20761011f5ca16755889ab406fd1de0", size = 355616, upload-time = "2025-12-04T22:24:47.598Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/52/08/8c8508db6c7b9aae8f7175046af41baad690771c9bcde676419965e338c7/fastapi-0.128.0.tar.gz", hash = "sha256:1cc179e1cef10a6be60ffe429f79b829dce99d8de32d7acb7e6c8dfdf7f2645a", size = 365682, upload-time = "2025-12-27T15:21:13.714Z" }
wheels = [
- { url = "https://files.pythonhosted.org/packages/db/15/a785e992a27620e022d0bc61b6c897ec14cff07c5ab7ff9f27651a21570b/fastapi-0.123.9-py3-none-any.whl", hash = "sha256:f54c69f23db14bd3dbcdfaf3fdce0483ca5f499512380c8e379a70cda30aa920", size = 111776, upload-time = "2025-12-04T22:24:46.042Z" },
+ { url = "https://files.pythonhosted.org/packages/5c/05/5cbb59154b093548acd0f4c7c474a118eda06da25aa75c616b72d8fcd92a/fastapi-0.128.0-py3-none-any.whl", hash = "sha256:aebd93f9716ee3b4f4fcfe13ffb7cf308d99c9f3ab5622d8877441072561582d", size = 103094, upload-time = "2025-12-27T15:21:12.154Z" },
]
[[package]]
@@ -1315,11 +1315,11 @@ wheels = [
[[package]]
name = "filelock"
-version = "3.20.0"
+version = "3.20.2"
source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/58/46/0028a82567109b5ef6e4d2a1f04a583fb513e6cf9527fcdd09afd817deeb/filelock-3.20.0.tar.gz", hash = "sha256:711e943b4ec6be42e1d4e6690b48dc175c822967466bb31c0c293f34334c13f4", size = 18922, upload-time = "2025-10-08T18:03:50.056Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/c1/e0/a75dbe4bca1e7d41307323dad5ea2efdd95408f74ab2de8bd7dba9b51a1a/filelock-3.20.2.tar.gz", hash = "sha256:a2241ff4ddde2a7cebddf78e39832509cb045d18ec1a09d7248d6bfc6bfbbe64", size = 19510, upload-time = "2026-01-02T15:33:32.582Z" }
wheels = [
- { url = "https://files.pythonhosted.org/packages/76/91/7216b27286936c16f5b4d0c530087e4a54eead683e6b0b73dd0c64844af6/filelock-3.20.0-py3-none-any.whl", hash = "sha256:339b4732ffda5cd79b13f4e2711a31b0365ce445d95d243bb996273d072546a2", size = 16054, upload-time = "2025-10-08T18:03:48.35Z" },
+ { url = "https://files.pythonhosted.org/packages/9a/30/ab407e2ec752aa541704ed8f93c11e2a5d92c168b8a755d818b74a3c5c2d/filelock-3.20.2-py3-none-any.whl", hash = "sha256:fbba7237d6ea277175a32c54bb71ef814a8546d8601269e1bfc388de333974e8", size = 16697, upload-time = "2026-01-02T15:33:31.133Z" },
]
[[package]]
@@ -1331,6 +1331,19 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/18/79/1b8fa1bb3568781e84c9200f951c735f3f157429f44be0495da55894d620/filetype-1.2.0-py2.py3-none-any.whl", hash = "sha256:7ce71b6880181241cf7ac8697a2f1eb6a8bd9b429f7ad6d27b8db9ba5f1c2d25", size = 19970, upload-time = "2022-11-02T17:34:01.425Z" },
]
+[[package]]
+name = "fla-core"
+version = "0.3.2"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+ { name = "einops" },
+ { name = "torch", marker = "sys_platform == 'never'" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/67/c6/10a1149b07e6bab45b2cb2d07f6b827716c2baf5f3404161753f25c6389b/fla_core-0.3.2.tar.gz", hash = "sha256:d38db16bc4e1c6fa8c04df442f246da1e6926a209426bc6ef703d41bfbc37c92", size = 296725, upload-time = "2025-09-10T07:43:40.155Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/7e/f5/74947b33c07682280e65adbdf17c4ee94b30232df2f728bafecf13d1d820/fla_core-0.3.2-py3-none-any.whl", hash = "sha256:e751d5a41e33eee721a6fb6588bd857f6f36e0d14719a23b1ebdbd617d307209", size = 413594, upload-time = "2025-09-10T07:43:37.786Z" },
+]
+
[[package]]
name = "flake8"
version = "7.1.0"
@@ -1345,6 +1358,21 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/dc/43/d5147aadaa52558e94e024811f2f9543b4bd7203b3a9659eeb5dff9c61b3/flake8-7.1.0-py2.py3-none-any.whl", hash = "sha256:2e416edcc62471a64cea09353f4e7bdba32aeb079b6e360554c659a122b1bc6a", size = 57569, upload-time = "2024-06-15T21:37:05.342Z" },
]
+[[package]]
+name = "flash-linear-attention"
+version = "0.3.2"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+ { name = "datasets" },
+ { name = "fla-core" },
+ { name = "pytest" },
+ { name = "transformers" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/84/f6/e62c1e562a288557eba7f06f168a7615813d1a227327b8beb8ba426da2c5/flash_linear_attention-0.3.2.tar.gz", hash = "sha256:9147747316c2951fed4ebeb4fa87977c05d807dc70c93b46250b68a6eb1183e2", size = 150880, upload-time = "2025-09-10T07:43:41.37Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/a0/d0/35ce9eac5f52c72005095aaa12a393d2656ed7ffedf925b2381a6b76d10c/flash_linear_attention-0.3.2-py3-none-any.whl", hash = "sha256:604e73361437ba786420ab195e2caa3fd19280503761e703fa353c5ce5c65376", size = 274592, upload-time = "2025-09-10T07:43:39.107Z" },
+]
+
[[package]]
name = "flash-mla"
version = "1.0.0+9edee0c"
@@ -1360,7 +1388,7 @@ dependencies = [
{ name = "einops" },
{ name = "ninja" },
{ name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version < '3.11' and extra == 'extra-13-megatron-core-dev') or (python_full_version < '3.11' and extra == 'extra-13-megatron-core-lts') or (extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" },
- { name = "numpy", version = "2.3.5", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version >= '3.11' and extra == 'extra-13-megatron-core-dev') or (python_full_version >= '3.11' and extra == 'extra-13-megatron-core-lts') or (extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" },
+ { name = "numpy", version = "2.4.0", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version >= '3.11' and extra == 'extra-13-megatron-core-dev') or (python_full_version >= '3.11' and extra == 'extra-13-megatron-core-lts') or (extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" },
{ name = "nvidia-cudnn-frontend" },
{ name = "nvidia-cutlass-dsl" },
{ name = "nvidia-ml-py" },
@@ -1556,14 +1584,14 @@ wheels = [
[[package]]
name = "gitpython"
-version = "3.1.45"
+version = "3.1.46"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "gitdb" },
]
-sdist = { url = "https://files.pythonhosted.org/packages/9a/c8/dd58967d119baab745caec2f9d853297cec1989ec1d63f677d3880632b88/gitpython-3.1.45.tar.gz", hash = "sha256:85b0ee964ceddf211c41b9f27a49086010a190fd8132a24e21f362a4b36a791c", size = 215076, upload-time = "2025-07-24T03:45:54.871Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/df/b5/59d16470a1f0dfe8c793f9ef56fd3826093fc52b3bd96d6b9d6c26c7e27b/gitpython-3.1.46.tar.gz", hash = "sha256:400124c7d0ef4ea03f7310ac2fbf7151e09ff97f2a3288d64a440c584a29c37f", size = 215371, upload-time = "2026-01-01T15:37:32.073Z" }
wheels = [
- { url = "https://files.pythonhosted.org/packages/01/61/d4b89fec821f72385526e1b9d9a3a0385dda4a72b206d28049e2c7cd39b8/gitpython-3.1.45-py3-none-any.whl", hash = "sha256:8908cb2e02fb3b93b7eb0f2827125cb699869470432cc885f019b8fd0fccff77", size = 208168, upload-time = "2025-07-24T03:45:52.517Z" },
+ { url = "https://files.pythonhosted.org/packages/6a/09/e21df6aef1e1ffc0c816f0522ddc3f6dcded766c3261813131c78a704470/gitpython-3.1.46-py3-none-any.whl", hash = "sha256:79812ed143d9d25b6d176a10bb511de0f9c67b1fa641d82097b0ab90398a2058", size = 208620, upload-time = "2026-01-01T15:37:30.574Z" },
]
[[package]]
@@ -1876,11 +1904,11 @@ wheels = [
[[package]]
name = "joblib"
-version = "1.5.2"
+version = "1.5.3"
source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/e8/5d/447af5ea094b9e4c4054f82e223ada074c552335b9b4b2d14bd9b35a67c4/joblib-1.5.2.tar.gz", hash = "sha256:3faa5c39054b2f03ca547da9b2f52fde67c06240c31853f306aea97f13647b55", size = 331077, upload-time = "2025-08-27T12:15:46.575Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/41/f2/d34e8b3a08a9cc79a50b2208a93dce981fe615b64d5a4d4abee421d898df/joblib-1.5.3.tar.gz", hash = "sha256:8561a3269e6801106863fd0d6d84bb737be9e7631e33aaed3fb9ce5953688da3", size = 331603, upload-time = "2025-12-15T08:41:46.427Z" }
wheels = [
- { url = "https://files.pythonhosted.org/packages/1e/e8/685f47e0d754320684db4425a0967f7d3fa70126bffd76110b7009a0090f/joblib-1.5.2-py3-none-any.whl", hash = "sha256:4e1f0bdbb987e6d843c70cf43714cb276623def372df3c22fe5266b2670bc241", size = 308396, upload-time = "2025-08-27T12:15:45.188Z" },
+ { url = "https://files.pythonhosted.org/packages/7b/91/984aca2ec129e2757d1e4e3c81c3fcda9d0f85b74670a094cc443d9ee949/joblib-1.5.3-py3-none-any.whl", hash = "sha256:5fc3c5039fc5ca8c0276333a188bbd59d6b7ab37fe6632daa76bc7f9ec18e713", size = 309071, upload-time = "2025-12-15T08:41:44.973Z" },
]
[[package]]
@@ -2195,7 +2223,7 @@ name = "megatron-core"
source = { editable = "." }
dependencies = [
{ name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11' or (extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" },
- { name = "numpy", version = "2.3.5", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11' or (extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" },
+ { name = "numpy", version = "2.4.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11' or (extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" },
{ name = "packaging" },
{ name = "torch", marker = "sys_platform == 'never' or (extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" },
]
@@ -2206,7 +2234,9 @@ dev = [
{ name = "causal-conv1d" },
{ name = "datasets" },
{ name = "einops" },
+ { name = "emerging-optimizers" },
{ name = "fastapi" },
+ { name = "flash-linear-attention" },
{ name = "flashinfer-python" },
{ name = "mamba-ssm" },
{ name = "megatron-energon", extra = ["av-decode"], marker = "extra == 'extra-13-megatron-core-dev'" },
@@ -2218,7 +2248,7 @@ dev = [
{ name = "onnxscript" },
{ name = "opentelemetry-api" },
{ name = "tensorstore", version = "0.1.78", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version < '3.11' and extra == 'extra-13-megatron-core-dev') or (extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" },
- { name = "tensorstore", version = "0.1.79", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version >= '3.11' and extra == 'extra-13-megatron-core-dev') or (extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" },
+ { name = "tensorstore", version = "0.1.80", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version >= '3.11' and extra == 'extra-13-megatron-core-dev') or (extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" },
{ name = "tqdm" },
{ name = "transformer-engine", marker = "extra == 'extra-13-megatron-core-dev'" },
{ name = "wget" },
@@ -2238,7 +2268,7 @@ lts = [
{ name = "onnxscript" },
{ name = "opentelemetry-api" },
{ name = "tensorstore", version = "0.1.78", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version < '3.11' and extra == 'extra-13-megatron-core-lts') or (extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" },
- { name = "tensorstore", version = "0.1.79", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version >= '3.11' and extra == 'extra-13-megatron-core-lts') or (extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" },
+ { name = "tensorstore", version = "0.1.80", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version >= '3.11' and extra == 'extra-13-megatron-core-lts') or (extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" },
{ name = "tqdm" },
{ name = "wget" },
]
@@ -2312,8 +2342,10 @@ requires-dist = [
{ name = "datasets", marker = "extra == 'lts'" },
{ name = "einops", marker = "extra == 'dev'", specifier = "~=0.8" },
{ name = "einops", marker = "extra == 'lts'", specifier = "~=0.8" },
+ { name = "emerging-optimizers", marker = "extra == 'dev'", git = "https://github.com/NVIDIA-NeMo/Emerging-Optimizers.git?rev=v0.1.0" },
{ name = "fastapi", marker = "extra == 'dev'", specifier = "~=0.50" },
{ name = "fastapi", marker = "extra == 'lts'", specifier = "~=0.50" },
+ { name = "flash-linear-attention", marker = "extra == 'dev'", specifier = "~=0.3.2" },
{ name = "flashinfer-python", marker = "extra == 'dev'" },
{ name = "flashinfer-python", marker = "extra == 'lts'" },
{ name = "flask-restful", marker = "extra == 'mlm'" },
@@ -2342,7 +2374,7 @@ requires-dist = [
{ name = "torch" },
{ name = "tqdm", marker = "extra == 'dev'" },
{ name = "tqdm", marker = "extra == 'lts'" },
- { name = "transformer-engine", extras = ["core-cu13", "pytorch"], marker = "extra == 'dev'", git = "https://github.com/NVIDIA/TransformerEngine.git?rev=release_v2.10" },
+ { name = "transformer-engine", extras = ["core-cu13", "pytorch"], marker = "extra == 'dev'", git = "https://github.com/NVIDIA/TransformerEngine.git?rev=release_v2.11" },
{ name = "transformers", marker = "extra == 'mlm'" },
{ name = "wandb", marker = "extra == 'mlm'" },
{ name = "wget", marker = "extra == 'dev'" },
@@ -2381,7 +2413,7 @@ linting = [
{ name = "ruff", specifier = "~=0.9.0" },
]
no-pypi-wheels = [
- { name = "emerging-optimizers", git = "https://github.com/NVIDIA-NeMo/Emerging-Optimizers.git?rev=fb1add873e7851ec34b48581ea1b15761b73d189" },
+ { name = "emerging-optimizers", git = "https://github.com/NVIDIA-NeMo/Emerging-Optimizers.git?rev=v0.1.0" },
{ name = "flash-mla", git = "https://github.com/deepseek-ai/FlashMLA?rev=9edee0c022cd0938148a18e334203b0aab43aa19" },
]
test = [
@@ -2409,7 +2441,7 @@ dependencies = [
{ name = "click" },
{ name = "multi-storage-client" },
{ name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version < '3.11' and extra == 'extra-13-megatron-core-dev') or (python_full_version < '3.11' and extra == 'extra-13-megatron-core-lts') or (extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" },
- { name = "numpy", version = "2.3.5", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version >= '3.11' and extra == 'extra-13-megatron-core-dev') or (python_full_version >= '3.11' and extra == 'extra-13-megatron-core-lts') or (extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" },
+ { name = "numpy", version = "2.4.0", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version >= '3.11' and extra == 'extra-13-megatron-core-dev') or (python_full_version >= '3.11' and extra == 'extra-13-megatron-core-lts') or (extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" },
{ name = "pillow" },
{ name = "pyyaml" },
{ name = "s3fs" },
@@ -2438,7 +2470,7 @@ version = "0.5.4"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version < '3.11' and extra == 'extra-13-megatron-core-dev') or (python_full_version < '3.11' and extra == 'extra-13-megatron-core-lts') or (extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" },
- { name = "numpy", version = "2.3.5", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version >= '3.11' and extra == 'extra-13-megatron-core-dev') or (python_full_version >= '3.11' and extra == 'extra-13-megatron-core-lts') or (extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" },
+ { name = "numpy", version = "2.4.0", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version >= '3.11' and extra == 'extra-13-megatron-core-dev') or (python_full_version >= '3.11' and extra == 'extra-13-megatron-core-lts') or (extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" },
]
sdist = { url = "https://files.pythonhosted.org/packages/0e/4a/c27b42ed9b1c7d13d9ba8b6905dece787d6259152f2309338aed29b2447b/ml_dtypes-0.5.4.tar.gz", hash = "sha256:8ab06a50fb9bf9666dd0fe5dfb4676fa2b0ac0f31ecff72a6c3af8e22c063453", size = 692314, upload-time = "2025-11-17T22:32:31.031Z" }
wheels = [
@@ -2550,7 +2582,7 @@ wheels = [
[[package]]
name = "multi-storage-client"
-version = "0.36.0"
+version = "0.39.1"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "filelock" },
@@ -2568,22 +2600,18 @@ dependencies = [
{ name = "xattr" },
]
wheels = [
- { url = "https://files.pythonhosted.org/packages/be/5f/8011fd041f695670b339c25f059b68207c315250ccc25a08f190bff78318/multi_storage_client-0.36.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:763cdb5e24b78adf33882b1d1c0d15021cc2c0088ffc6e7b0269259f0cd45fd2", size = 5299321, upload-time = "2025-11-26T20:03:58.147Z" },
- { url = "https://files.pythonhosted.org/packages/51/06/cfd17d307fe29fbbce9f196ec1d8dda3f93fd44711c0adb282d9c393a2b2/multi_storage_client-0.36.0-cp310-cp310-macosx_11_0_x86_64.whl", hash = "sha256:eb84ea0bdffcfddf9beb7239c6d0b1950a67a0afe36ef970da70ba4ab373c0c9", size = 5420867, upload-time = "2025-11-26T20:05:32.445Z" },
- { url = "https://files.pythonhosted.org/packages/7c/7f/bf22f9c67c70d5ec2f6a7a4798cb106f3023bf25ba6c21b0ade1a53fa5b3/multi_storage_client-0.36.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ff03a0213ce1377abee61e8deb87607f0ccd35c245fbaab2fee51d2e591e833e", size = 3188237, upload-time = "2025-11-26T20:01:51.354Z" },
- { url = "https://files.pythonhosted.org/packages/fb/20/c0c019b3dc7719f79c1826364fc9c3e1bbe9b00246b1d7414ce2b4defd0b/multi_storage_client-0.36.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f16e577ef4ee6f8ac481b3f2290e7b0525676efd82c71fb694ba4e6c65a8facd", size = 3363259, upload-time = "2025-11-26T20:00:10.679Z" },
- { url = "https://files.pythonhosted.org/packages/2b/f8/eea6be7f4258c811373dc989e8eaa23a404499c2574059f6fd876d6904e4/multi_storage_client-0.36.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:6c913b132573fbd7a5ada63086d3ce2669b913b79206f86867cc674d57b9164d", size = 5299844, upload-time = "2025-11-26T20:00:32.46Z" },
- { url = "https://files.pythonhosted.org/packages/df/aa/b73441dc17097ee92e7efac5080e2cfb8fe4515dd4dc91ca351829e6b7a9/multi_storage_client-0.36.0-cp311-cp311-macosx_11_0_x86_64.whl", hash = "sha256:4dd2ccf67deae403098a5e867ce33d35ce348d2acd1a743c9ef485b3b1eea65c", size = 5424007, upload-time = "2025-11-26T19:55:30.305Z" },
- { url = "https://files.pythonhosted.org/packages/54/d6/850550de6b0dc740ced2f8fbf83f13f757860b5fdaa652e477c567c01f34/multi_storage_client-0.36.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:04b31b6a5d6a3c90a592b23a4b90368fa1dcca8cb03f76a862d307f8b072c1d3", size = 3188451, upload-time = "2025-11-26T19:56:32.191Z" },
- { url = "https://files.pythonhosted.org/packages/a3/c5/93e038c0cce46cb9b1b8e19f7215ce3e7fa1af5e0a9662f36dfe47062f7e/multi_storage_client-0.36.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:252f84116f674962eabd066e16040f0304f6191c06ab09ef2ec02dbfd2c4d2ea", size = 3366554, upload-time = "2025-11-26T19:58:37.742Z" },
- { url = "https://files.pythonhosted.org/packages/28/a2/46320db394150a2f0547930b902e8ad045a084fb519f408e2c9b4ca673a0/multi_storage_client-0.36.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:2058e8e8f8fd9eef033171b0bf1966596e9862c7f20c2886101ad979996c453b", size = 5293778, upload-time = "2025-11-26T20:07:11.731Z" },
- { url = "https://files.pythonhosted.org/packages/00/2d/658af3b4104c4f2aa2621469482dca8270490601e98d8f7997361499adaa/multi_storage_client-0.36.0-cp312-cp312-macosx_11_0_x86_64.whl", hash = "sha256:22b69c7f3c9ffa166f38bafa7e08f6b664a5dbee8c88d5d740bed719e6f410a1", size = 5418642, upload-time = "2025-11-26T19:58:15.717Z" },
- { url = "https://files.pythonhosted.org/packages/09/2f/6441794bf8dc195d614d63ad2b7068ad7703972fd6f960d43202d29748b1/multi_storage_client-0.36.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b384fb326637e79706ff706e60f384b24fdbcc824420bb66ef615a9ef5ffb4ec", size = 3194133, upload-time = "2025-11-26T20:05:54.618Z" },
- { url = "https://files.pythonhosted.org/packages/0e/ba/b07361ff84e5bd263e299b03776382f59bd92862573c915dd705a09f3c1d/multi_storage_client-0.36.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7111567b971a68719c0eb68245d49a0a3c3bf5af2f609351446f20ac3e83c0d5", size = 3364563, upload-time = "2025-11-26T20:04:20.3Z" },
- { url = "https://files.pythonhosted.org/packages/f9/4a/cbd61589a457e2f4fbacd08b7e7dd11cdb74690857f4b40042844b1ff894/multi_storage_client-0.36.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:a8137558d5f05e4722c54540e2d6067ea61e9ce3d736fa9cb5c541c7f94d1b48", size = 5293550, upload-time = "2025-11-26T20:03:36.459Z" },
- { url = "https://files.pythonhosted.org/packages/a7/3d/7499a9d537fa950a9acf11604b1f9372ed2cadd582b55f1c7cb885ce6f40/multi_storage_client-0.36.0-cp313-cp313-macosx_11_0_x86_64.whl", hash = "sha256:5394c5e040c32433b42e902d9fcf03f8a475c5c9ff1cca80743b2cb944c8af9e", size = 5417538, upload-time = "2025-11-26T20:06:16.782Z" },
- { url = "https://files.pythonhosted.org/packages/d7/c3/1b1adc3b3b8569d258a34dbedb6a8c51fc94b947b2df276e251f0f1e23a2/multi_storage_client-0.36.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:195e8c8d57d812b73efd41b96cd60825c484d317ec86379fad3e435e9365a4a6", size = 3193426, upload-time = "2025-11-26T20:00:56.034Z" },
- { url = "https://files.pythonhosted.org/packages/60/f5/f8b97a87d928057b493733760f37de70ae5ffff84b86f6efae101cdd57a2/multi_storage_client-0.36.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8402d0e1cefedf38ad9eefe8b3c56d3a44cfec7775ef711da18e7dbf72669444", size = 3363531, upload-time = "2025-11-26T20:02:35.296Z" },
+ { url = "https://files.pythonhosted.org/packages/03/d2/6cce7b432f58bcffb394bac96d9edad2d927ffec382a53300e41186da1a5/multi_storage_client-0.39.1-cp310-cp310-macosx_14_0_arm64.whl", hash = "sha256:46aa5c7446e079dac852e7db9077e80fe69f4c7e4690f526cc61cbd15d43b07e", size = 8429120, upload-time = "2025-12-19T03:18:25.375Z" },
+ { url = "https://files.pythonhosted.org/packages/18/00/423e6fcf218a52216ad86686f4fffa4f18b605594601d621aec68ad02d33/multi_storage_client-0.39.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:31249cee7f0fa4cc536136371eb41ad48c4b86c02fdf4e3186e7b464488d1e73", size = 4784439, upload-time = "2025-12-19T03:20:50.881Z" },
+ { url = "https://files.pythonhosted.org/packages/14/73/161ebe8bb71acee7bb7a42389756cd43d07e56e155d40f54b72370c5eb64/multi_storage_client-0.39.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:22eb940daae3149efe2e8e412fdd4e3d8d10c0077b336cf2ebc90236dfe58665", size = 5048336, upload-time = "2025-12-19T03:17:35.193Z" },
+ { url = "https://files.pythonhosted.org/packages/6d/77/c20249c7887c37c0868ec8bc9ca6313fca54a232c3a50b04cd56b0b514ea/multi_storage_client-0.39.1-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:4d71b05d5ea7d09c075edae731ee6b89ed2114b5eeaf96e46e2a15b37b91de07", size = 8427425, upload-time = "2025-12-19T03:20:26.171Z" },
+ { url = "https://files.pythonhosted.org/packages/8d/ab/350acee344fe32db07ae535021e339ec4edf5e40b78a323fc11fcd6dda97/multi_storage_client-0.39.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1f0dd3bb536f8fc5992dccb1e53108a695fb4e703e9320d5292f63188269bfcd", size = 4783799, upload-time = "2025-12-19T03:19:14.054Z" },
+ { url = "https://files.pythonhosted.org/packages/83/c0/19b03d58f4d2713b3948e3bd72d5711d89f22250b966b70ccfbb914cb6fe/multi_storage_client-0.39.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:44602b32f7b708b82eab56d49ba00a05ed478193387cf4e22ca2c19da8c6877a", size = 5047335, upload-time = "2025-12-19T03:18:49.925Z" },
+ { url = "https://files.pythonhosted.org/packages/76/c5/204f3859f3cc7dde35fc74b52c5d61d7017434781c296c9640c1bbd849c7/multi_storage_client-0.39.1-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:15973fff7b351f2949d4cd3bf9f24bb8c73838f5ab29e67f018318ec3d3e3079", size = 8420253, upload-time = "2025-12-19T03:16:48.333Z" },
+ { url = "https://files.pythonhosted.org/packages/57/dd/9f2d20e83742c5dcf49719a2905157b372e6380779d8c2fdd90f3898f6b9/multi_storage_client-0.39.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0fc12834fd3fada72016b4df54f908c769d1fd6d5b9dbbc573831665def8b46c", size = 4784064, upload-time = "2025-12-19T03:17:11.978Z" },
+ { url = "https://files.pythonhosted.org/packages/7f/c8/fbc5a69eb910246bf154030aec0d9df6c204481d8a1ec3352de042499300/multi_storage_client-0.39.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:881365a17bc3886bf8f54d33c9c5d0d5a393b6000cdd12eed756b6e5eb3b2bb0", size = 5048702, upload-time = "2025-12-19T03:15:27.792Z" },
+ { url = "https://files.pythonhosted.org/packages/8d/e6/7ca7a7fd03893d03b36c225702e2a644b38bfe1b5c0fa5b266fd8f72ba1d/multi_storage_client-0.39.1-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:e6f7a9710d1e61beb3f736cc2a1bbb9916c462022be544edf604cc8d8a9ac201", size = 8418852, upload-time = "2025-12-19T03:16:23.313Z" },
+ { url = "https://files.pythonhosted.org/packages/bb/6a/7b25d15446085a103ebdf21834705020693e76ea093ca23e5647872b4165/multi_storage_client-0.39.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cfa87a3eecb09de64a77c68b622f2d2b0cb3aefb8a9d2306b1bf83c085f3bdd5", size = 4784229, upload-time = "2025-12-19T03:21:14.885Z" },
+ { url = "https://files.pythonhosted.org/packages/a1/bd/dbda0847ef2ffab6a11b60f4702edf60fc1287174009bb2e35dff205d5ba/multi_storage_client-0.39.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4f705d1d4d11b19ec9f6819674ef852d9b8fa2c95906c4c5cab2ecb7c22bc290", size = 5048266, upload-time = "2025-12-19T03:15:56.348Z" },
]
[[package]]
@@ -2787,7 +2815,7 @@ dependencies = [
{ name = "jinja2" },
{ name = "leptonai" },
{ name = "networkx", version = "3.4.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11' or (extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" },
- { name = "networkx", version = "3.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11' or (extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" },
+ { name = "networkx", version = "3.6.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11' or (extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" },
{ name = "omegaconf" },
{ name = "packaging" },
{ name = "rich" },
@@ -2811,7 +2839,7 @@ wheels = [
[[package]]
name = "networkx"
-version = "3.6"
+version = "3.6.1"
source = { registry = "https://pypi.org/simple" }
resolution-markers = [
"python_full_version >= '3.14' and sys_platform == 'linux'",
@@ -2823,9 +2851,9 @@ resolution-markers = [
"python_full_version == '3.11.*' and sys_platform == 'linux'",
"python_full_version == '3.11.*' and sys_platform != 'linux'",
]
-sdist = { url = "https://files.pythonhosted.org/packages/e8/fc/7b6fd4d22c8c4dc5704430140d8b3f520531d4fe7328b8f8d03f5a7950e8/networkx-3.6.tar.gz", hash = "sha256:285276002ad1f7f7da0f7b42f004bcba70d381e936559166363707fdad3d72ad", size = 2511464, upload-time = "2025-11-24T03:03:47.158Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/6a/51/63fe664f3908c97be9d2e4f1158eb633317598cfa6e1fc14af5383f17512/networkx-3.6.1.tar.gz", hash = "sha256:26b7c357accc0c8cde558ad486283728b65b6a95d85ee1cd66bafab4c8168509", size = 2517025, upload-time = "2025-12-08T17:02:39.908Z" }
wheels = [
- { url = "https://files.pythonhosted.org/packages/07/c7/d64168da60332c17d24c0d2f08bdf3987e8d1ae9d84b5bbd0eec2eb26a55/networkx-3.6-py3-none-any.whl", hash = "sha256:cdb395b105806062473d3be36458d8f1459a4e4b98e236a66c3a48996e07684f", size = 2063713, upload-time = "2025-11-24T03:03:45.21Z" },
+ { url = "https://files.pythonhosted.org/packages/9e/c9/b2622292ea83fbb4ec318f5b9ab867d0a28ab43c5717bb85b0a5f6b3b0a4/networkx-3.6.1-py3-none-any.whl", hash = "sha256:d47fbf302e7d9cbbb9e2555a0d267983d2aa476bac30e90dfbe5669bd57f3762", size = 2068504, upload-time = "2025-12-08T17:02:38.159Z" },
]
[[package]]
@@ -2937,7 +2965,7 @@ wheels = [
[[package]]
name = "numpy"
-version = "2.3.5"
+version = "2.4.0"
source = { registry = "https://pypi.org/simple" }
resolution-markers = [
"python_full_version >= '3.14' and sys_platform == 'linux'",
@@ -2949,94 +2977,92 @@ resolution-markers = [
"python_full_version == '3.11.*' and sys_platform == 'linux'",
"python_full_version == '3.11.*' and sys_platform != 'linux'",
]
-sdist = { url = "https://files.pythonhosted.org/packages/76/65/21b3bc86aac7b8f2862db1e808f1ea22b028e30a225a34a5ede9bf8678f2/numpy-2.3.5.tar.gz", hash = "sha256:784db1dcdab56bf0517743e746dfb0f885fc68d948aba86eeec2cba234bdf1c0", size = 20584950, upload-time = "2025-11-16T22:52:42.067Z" }
-wheels = [
- { url = "https://files.pythonhosted.org/packages/43/77/84dd1d2e34d7e2792a236ba180b5e8fcc1e3e414e761ce0253f63d7f572e/numpy-2.3.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:de5672f4a7b200c15a4127042170a694d4df43c992948f5e1af57f0174beed10", size = 17034641, upload-time = "2025-11-16T22:49:19.336Z" },
- { url = "https://files.pythonhosted.org/packages/2a/ea/25e26fa5837106cde46ae7d0b667e20f69cbbc0efd64cba8221411ab26ae/numpy-2.3.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:acfd89508504a19ed06ef963ad544ec6664518c863436306153e13e94605c218", size = 12528324, upload-time = "2025-11-16T22:49:22.582Z" },
- { url = "https://files.pythonhosted.org/packages/4d/1a/e85f0eea4cf03d6a0228f5c0256b53f2df4bc794706e7df019fc622e47f1/numpy-2.3.5-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:ffe22d2b05504f786c867c8395de703937f934272eb67586817b46188b4ded6d", size = 5356872, upload-time = "2025-11-16T22:49:25.408Z" },
- { url = "https://files.pythonhosted.org/packages/5c/bb/35ef04afd567f4c989c2060cde39211e4ac5357155c1833bcd1166055c61/numpy-2.3.5-cp311-cp311-macosx_14_0_x86_64.whl", hash = "sha256:872a5cf366aec6bb1147336480fef14c9164b154aeb6542327de4970282cd2f5", size = 6893148, upload-time = "2025-11-16T22:49:27.549Z" },
- { url = "https://files.pythonhosted.org/packages/f2/2b/05bbeb06e2dff5eab512dfc678b1cc5ee94d8ac5956a0885c64b6b26252b/numpy-2.3.5-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3095bdb8dd297e5920b010e96134ed91d852d81d490e787beca7e35ae1d89cf7", size = 14557282, upload-time = "2025-11-16T22:49:30.964Z" },
- { url = "https://files.pythonhosted.org/packages/65/fb/2b23769462b34398d9326081fad5655198fcf18966fcb1f1e49db44fbf31/numpy-2.3.5-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8cba086a43d54ca804ce711b2a940b16e452807acebe7852ff327f1ecd49b0d4", size = 16897903, upload-time = "2025-11-16T22:49:34.191Z" },
- { url = "https://files.pythonhosted.org/packages/ac/14/085f4cf05fc3f1e8aa95e85404e984ffca9b2275a5dc2b1aae18a67538b8/numpy-2.3.5-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:6cf9b429b21df6b99f4dee7a1218b8b7ffbbe7df8764dc0bd60ce8a0708fed1e", size = 16341672, upload-time = "2025-11-16T22:49:37.2Z" },
- { url = "https://files.pythonhosted.org/packages/6f/3b/1f73994904142b2aa290449b3bb99772477b5fd94d787093e4f24f5af763/numpy-2.3.5-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:396084a36abdb603546b119d96528c2f6263921c50df3c8fd7cb28873a237748", size = 18838896, upload-time = "2025-11-16T22:49:39.727Z" },
- { url = "https://files.pythonhosted.org/packages/cd/b9/cf6649b2124f288309ffc353070792caf42ad69047dcc60da85ee85fea58/numpy-2.3.5-cp311-cp311-win32.whl", hash = "sha256:b0c7088a73aef3d687c4deef8452a3ac7c1be4e29ed8bf3b366c8111128ac60c", size = 6563608, upload-time = "2025-11-16T22:49:42.079Z" },
- { url = "https://files.pythonhosted.org/packages/aa/44/9fe81ae1dcc29c531843852e2874080dc441338574ccc4306b39e2ff6e59/numpy-2.3.5-cp311-cp311-win_amd64.whl", hash = "sha256:a414504bef8945eae5f2d7cb7be2d4af77c5d1cb5e20b296c2c25b61dff2900c", size = 13078442, upload-time = "2025-11-16T22:49:43.99Z" },
- { url = "https://files.pythonhosted.org/packages/6d/a7/f99a41553d2da82a20a2f22e93c94f928e4490bb447c9ff3c4ff230581d3/numpy-2.3.5-cp311-cp311-win_arm64.whl", hash = "sha256:0cd00b7b36e35398fa2d16af7b907b65304ef8bb4817a550e06e5012929830fa", size = 10458555, upload-time = "2025-11-16T22:49:47.092Z" },
- { url = "https://files.pythonhosted.org/packages/44/37/e669fe6cbb2b96c62f6bbedc6a81c0f3b7362f6a59230b23caa673a85721/numpy-2.3.5-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:74ae7b798248fe62021dbf3c914245ad45d1a6b0cb4a29ecb4b31d0bfbc4cc3e", size = 16733873, upload-time = "2025-11-16T22:49:49.84Z" },
- { url = "https://files.pythonhosted.org/packages/c5/65/df0db6c097892c9380851ab9e44b52d4f7ba576b833996e0080181c0c439/numpy-2.3.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ee3888d9ff7c14604052b2ca5535a30216aa0a58e948cdd3eeb8d3415f638769", size = 12259838, upload-time = "2025-11-16T22:49:52.863Z" },
- { url = "https://files.pythonhosted.org/packages/5b/e1/1ee06e70eb2136797abe847d386e7c0e830b67ad1d43f364dd04fa50d338/numpy-2.3.5-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:612a95a17655e213502f60cfb9bf9408efdc9eb1d5f50535cc6eb365d11b42b5", size = 5088378, upload-time = "2025-11-16T22:49:55.055Z" },
- { url = "https://files.pythonhosted.org/packages/6d/9c/1ca85fb86708724275103b81ec4cf1ac1d08f465368acfc8da7ab545bdae/numpy-2.3.5-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:3101e5177d114a593d79dd79658650fe28b5a0d8abeb8ce6f437c0e6df5be1a4", size = 6628559, upload-time = "2025-11-16T22:49:57.371Z" },
- { url = "https://files.pythonhosted.org/packages/74/78/fcd41e5a0ce4f3f7b003da85825acddae6d7ecb60cf25194741b036ca7d6/numpy-2.3.5-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8b973c57ff8e184109db042c842423ff4f60446239bd585a5131cc47f06f789d", size = 14250702, upload-time = "2025-11-16T22:49:59.632Z" },
- { url = "https://files.pythonhosted.org/packages/b6/23/2a1b231b8ff672b4c450dac27164a8b2ca7d9b7144f9c02d2396518352eb/numpy-2.3.5-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0d8163f43acde9a73c2a33605353a4f1bc4798745a8b1d73183b28e5b435ae28", size = 16606086, upload-time = "2025-11-16T22:50:02.127Z" },
- { url = "https://files.pythonhosted.org/packages/a0/c5/5ad26fbfbe2012e190cc7d5003e4d874b88bb18861d0829edc140a713021/numpy-2.3.5-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:51c1e14eb1e154ebd80e860722f9e6ed6ec89714ad2db2d3aa33c31d7c12179b", size = 16025985, upload-time = "2025-11-16T22:50:04.536Z" },
- { url = "https://files.pythonhosted.org/packages/d2/fa/dd48e225c46c819288148d9d060b047fd2a6fb1eb37eae25112ee4cb4453/numpy-2.3.5-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:b46b4ec24f7293f23adcd2d146960559aaf8020213de8ad1909dba6c013bf89c", size = 18542976, upload-time = "2025-11-16T22:50:07.557Z" },
- { url = "https://files.pythonhosted.org/packages/05/79/ccbd23a75862d95af03d28b5c6901a1b7da4803181513d52f3b86ed9446e/numpy-2.3.5-cp312-cp312-win32.whl", hash = "sha256:3997b5b3c9a771e157f9aae01dd579ee35ad7109be18db0e85dbdbe1de06e952", size = 6285274, upload-time = "2025-11-16T22:50:10.746Z" },
- { url = "https://files.pythonhosted.org/packages/2d/57/8aeaf160312f7f489dea47ab61e430b5cb051f59a98ae68b7133ce8fa06a/numpy-2.3.5-cp312-cp312-win_amd64.whl", hash = "sha256:86945f2ee6d10cdfd67bcb4069c1662dd711f7e2a4343db5cecec06b87cf31aa", size = 12782922, upload-time = "2025-11-16T22:50:12.811Z" },
- { url = "https://files.pythonhosted.org/packages/78/a6/aae5cc2ca78c45e64b9ef22f089141d661516856cf7c8a54ba434576900d/numpy-2.3.5-cp312-cp312-win_arm64.whl", hash = "sha256:f28620fe26bee16243be2b7b874da327312240a7cdc38b769a697578d2100013", size = 10194667, upload-time = "2025-11-16T22:50:16.16Z" },
- { url = "https://files.pythonhosted.org/packages/db/69/9cde09f36da4b5a505341180a3f2e6fadc352fd4d2b7096ce9778db83f1a/numpy-2.3.5-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:d0f23b44f57077c1ede8c5f26b30f706498b4862d3ff0a7298b8411dd2f043ff", size = 16728251, upload-time = "2025-11-16T22:50:19.013Z" },
- { url = "https://files.pythonhosted.org/packages/79/fb/f505c95ceddd7027347b067689db71ca80bd5ecc926f913f1a23e65cf09b/numpy-2.3.5-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:aa5bc7c5d59d831d9773d1170acac7893ce3a5e130540605770ade83280e7188", size = 12254652, upload-time = "2025-11-16T22:50:21.487Z" },
- { url = "https://files.pythonhosted.org/packages/78/da/8c7738060ca9c31b30e9301ee0cf6c5ffdbf889d9593285a1cead337f9a5/numpy-2.3.5-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:ccc933afd4d20aad3c00bcef049cb40049f7f196e0397f1109dba6fed63267b0", size = 5083172, upload-time = "2025-11-16T22:50:24.562Z" },
- { url = "https://files.pythonhosted.org/packages/a4/b4/ee5bb2537fb9430fd2ef30a616c3672b991a4129bb1c7dcc42aa0abbe5d7/numpy-2.3.5-cp313-cp313-macosx_14_0_x86_64.whl", hash = "sha256:afaffc4393205524af9dfa400fa250143a6c3bc646c08c9f5e25a9f4b4d6a903", size = 6622990, upload-time = "2025-11-16T22:50:26.47Z" },
- { url = "https://files.pythonhosted.org/packages/95/03/dc0723a013c7d7c19de5ef29e932c3081df1c14ba582b8b86b5de9db7f0f/numpy-2.3.5-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9c75442b2209b8470d6d5d8b1c25714270686f14c749028d2199c54e29f20b4d", size = 14248902, upload-time = "2025-11-16T22:50:28.861Z" },
- { url = "https://files.pythonhosted.org/packages/f5/10/ca162f45a102738958dcec8023062dad0cbc17d1ab99d68c4e4a6c45fb2b/numpy-2.3.5-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:11e06aa0af8c0f05104d56450d6093ee639e15f24ecf62d417329d06e522e017", size = 16597430, upload-time = "2025-11-16T22:50:31.56Z" },
- { url = "https://files.pythonhosted.org/packages/2a/51/c1e29be863588db58175175f057286900b4b3327a1351e706d5e0f8dd679/numpy-2.3.5-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:ed89927b86296067b4f81f108a2271d8926467a8868e554eaf370fc27fa3ccaf", size = 16024551, upload-time = "2025-11-16T22:50:34.242Z" },
- { url = "https://files.pythonhosted.org/packages/83/68/8236589d4dbb87253d28259d04d9b814ec0ecce7cb1c7fed29729f4c3a78/numpy-2.3.5-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:51c55fe3451421f3a6ef9a9c1439e82101c57a2c9eab9feb196a62b1a10b58ce", size = 18533275, upload-time = "2025-11-16T22:50:37.651Z" },
- { url = "https://files.pythonhosted.org/packages/40/56/2932d75b6f13465239e3b7b7e511be27f1b8161ca2510854f0b6e521c395/numpy-2.3.5-cp313-cp313-win32.whl", hash = "sha256:1978155dd49972084bd6ef388d66ab70f0c323ddee6f693d539376498720fb7e", size = 6277637, upload-time = "2025-11-16T22:50:40.11Z" },
- { url = "https://files.pythonhosted.org/packages/0c/88/e2eaa6cffb115b85ed7c7c87775cb8bcf0816816bc98ca8dbfa2ee33fe6e/numpy-2.3.5-cp313-cp313-win_amd64.whl", hash = "sha256:00dc4e846108a382c5869e77c6ed514394bdeb3403461d25a829711041217d5b", size = 12779090, upload-time = "2025-11-16T22:50:42.503Z" },
- { url = "https://files.pythonhosted.org/packages/8f/88/3f41e13a44ebd4034ee17baa384acac29ba6a4fcc2aca95f6f08ca0447d1/numpy-2.3.5-cp313-cp313-win_arm64.whl", hash = "sha256:0472f11f6ec23a74a906a00b48a4dcf3849209696dff7c189714511268d103ae", size = 10194710, upload-time = "2025-11-16T22:50:44.971Z" },
- { url = "https://files.pythonhosted.org/packages/13/cb/71744144e13389d577f867f745b7df2d8489463654a918eea2eeb166dfc9/numpy-2.3.5-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:414802f3b97f3c1eef41e530aaba3b3c1620649871d8cb38c6eaff034c2e16bd", size = 16827292, upload-time = "2025-11-16T22:50:47.715Z" },
- { url = "https://files.pythonhosted.org/packages/71/80/ba9dc6f2a4398e7f42b708a7fdc841bb638d353be255655498edbf9a15a8/numpy-2.3.5-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:5ee6609ac3604fa7780e30a03e5e241a7956f8e2fcfe547d51e3afa5247ac47f", size = 12378897, upload-time = "2025-11-16T22:50:51.327Z" },
- { url = "https://files.pythonhosted.org/packages/2e/6d/db2151b9f64264bcceccd51741aa39b50150de9b602d98ecfe7e0c4bff39/numpy-2.3.5-cp313-cp313t-macosx_14_0_arm64.whl", hash = "sha256:86d835afea1eaa143012a2d7a3f45a3adce2d7adc8b4961f0b362214d800846a", size = 5207391, upload-time = "2025-11-16T22:50:54.542Z" },
- { url = "https://files.pythonhosted.org/packages/80/ae/429bacace5ccad48a14c4ae5332f6aa8ab9f69524193511d60ccdfdc65fa/numpy-2.3.5-cp313-cp313t-macosx_14_0_x86_64.whl", hash = "sha256:30bc11310e8153ca664b14c5f1b73e94bd0503681fcf136a163de856f3a50139", size = 6721275, upload-time = "2025-11-16T22:50:56.794Z" },
- { url = "https://files.pythonhosted.org/packages/74/5b/1919abf32d8722646a38cd527bc3771eb229a32724ee6ba340ead9b92249/numpy-2.3.5-cp313-cp313t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1062fde1dcf469571705945b0f221b73928f34a20c904ffb45db101907c3454e", size = 14306855, upload-time = "2025-11-16T22:50:59.208Z" },
- { url = "https://files.pythonhosted.org/packages/a5/87/6831980559434973bebc30cd9c1f21e541a0f2b0c280d43d3afd909b66d0/numpy-2.3.5-cp313-cp313t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ce581db493ea1a96c0556360ede6607496e8bf9b3a8efa66e06477267bc831e9", size = 16657359, upload-time = "2025-11-16T22:51:01.991Z" },
- { url = "https://files.pythonhosted.org/packages/dd/91/c797f544491ee99fd00495f12ebb7802c440c1915811d72ac5b4479a3356/numpy-2.3.5-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:cc8920d2ec5fa99875b670bb86ddeb21e295cb07aa331810d9e486e0b969d946", size = 16093374, upload-time = "2025-11-16T22:51:05.291Z" },
- { url = "https://files.pythonhosted.org/packages/74/a6/54da03253afcbe7a72785ec4da9c69fb7a17710141ff9ac5fcb2e32dbe64/numpy-2.3.5-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:9ee2197ef8c4f0dfe405d835f3b6a14f5fee7782b5de51ba06fb65fc9b36e9f1", size = 18594587, upload-time = "2025-11-16T22:51:08.585Z" },
- { url = "https://files.pythonhosted.org/packages/80/e9/aff53abbdd41b0ecca94285f325aff42357c6b5abc482a3fcb4994290b18/numpy-2.3.5-cp313-cp313t-win32.whl", hash = "sha256:70b37199913c1bd300ff6e2693316c6f869c7ee16378faf10e4f5e3275b299c3", size = 6405940, upload-time = "2025-11-16T22:51:11.541Z" },
- { url = "https://files.pythonhosted.org/packages/d5/81/50613fec9d4de5480de18d4f8ef59ad7e344d497edbef3cfd80f24f98461/numpy-2.3.5-cp313-cp313t-win_amd64.whl", hash = "sha256:b501b5fa195cc9e24fe102f21ec0a44dffc231d2af79950b451e0d99cea02234", size = 12920341, upload-time = "2025-11-16T22:51:14.312Z" },
- { url = "https://files.pythonhosted.org/packages/bb/ab/08fd63b9a74303947f34f0bd7c5903b9c5532c2d287bead5bdf4c556c486/numpy-2.3.5-cp313-cp313t-win_arm64.whl", hash = "sha256:a80afd79f45f3c4a7d341f13acbe058d1ca8ac017c165d3fa0d3de6bc1a079d7", size = 10262507, upload-time = "2025-11-16T22:51:16.846Z" },
- { url = "https://files.pythonhosted.org/packages/ba/97/1a914559c19e32d6b2e233cf9a6a114e67c856d35b1d6babca571a3e880f/numpy-2.3.5-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:bf06bc2af43fa8d32d30fae16ad965663e966b1a3202ed407b84c989c3221e82", size = 16735706, upload-time = "2025-11-16T22:51:19.558Z" },
- { url = "https://files.pythonhosted.org/packages/57/d4/51233b1c1b13ecd796311216ae417796b88b0616cfd8a33ae4536330748a/numpy-2.3.5-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:052e8c42e0c49d2575621c158934920524f6c5da05a1d3b9bab5d8e259e045f0", size = 12264507, upload-time = "2025-11-16T22:51:22.492Z" },
- { url = "https://files.pythonhosted.org/packages/45/98/2fe46c5c2675b8306d0b4a3ec3494273e93e1226a490f766e84298576956/numpy-2.3.5-cp314-cp314-macosx_14_0_arm64.whl", hash = "sha256:1ed1ec893cff7040a02c8aa1c8611b94d395590d553f6b53629a4461dc7f7b63", size = 5093049, upload-time = "2025-11-16T22:51:25.171Z" },
- { url = "https://files.pythonhosted.org/packages/ce/0e/0698378989bb0ac5f1660c81c78ab1fe5476c1a521ca9ee9d0710ce54099/numpy-2.3.5-cp314-cp314-macosx_14_0_x86_64.whl", hash = "sha256:2dcd0808a421a482a080f89859a18beb0b3d1e905b81e617a188bd80422d62e9", size = 6626603, upload-time = "2025-11-16T22:51:27Z" },
- { url = "https://files.pythonhosted.org/packages/5e/a6/9ca0eecc489640615642a6cbc0ca9e10df70df38c4d43f5a928ff18d8827/numpy-2.3.5-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:727fd05b57df37dc0bcf1a27767a3d9a78cbbc92822445f32cc3436ba797337b", size = 14262696, upload-time = "2025-11-16T22:51:29.402Z" },
- { url = "https://files.pythonhosted.org/packages/c8/f6/07ec185b90ec9d7217a00eeeed7383b73d7e709dae2a9a021b051542a708/numpy-2.3.5-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:fffe29a1ef00883599d1dc2c51aa2e5d80afe49523c261a74933df395c15c520", size = 16597350, upload-time = "2025-11-16T22:51:32.167Z" },
- { url = "https://files.pythonhosted.org/packages/75/37/164071d1dde6a1a84c9b8e5b414fa127981bad47adf3a6b7e23917e52190/numpy-2.3.5-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:8f7f0e05112916223d3f438f293abf0727e1181b5983f413dfa2fefc4098245c", size = 16040190, upload-time = "2025-11-16T22:51:35.403Z" },
- { url = "https://files.pythonhosted.org/packages/08/3c/f18b82a406b04859eb026d204e4e1773eb41c5be58410f41ffa511d114ae/numpy-2.3.5-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:2e2eb32ddb9ccb817d620ac1d8dae7c3f641c1e5f55f531a33e8ab97960a75b8", size = 18536749, upload-time = "2025-11-16T22:51:39.698Z" },
- { url = "https://files.pythonhosted.org/packages/40/79/f82f572bf44cf0023a2fe8588768e23e1592585020d638999f15158609e1/numpy-2.3.5-cp314-cp314-win32.whl", hash = "sha256:66f85ce62c70b843bab1fb14a05d5737741e74e28c7b8b5a064de10142fad248", size = 6335432, upload-time = "2025-11-16T22:51:42.476Z" },
- { url = "https://files.pythonhosted.org/packages/a3/2e/235b4d96619931192c91660805e5e49242389742a7a82c27665021db690c/numpy-2.3.5-cp314-cp314-win_amd64.whl", hash = "sha256:e6a0bc88393d65807d751a614207b7129a310ca4fe76a74e5c7da5fa5671417e", size = 12919388, upload-time = "2025-11-16T22:51:45.275Z" },
- { url = "https://files.pythonhosted.org/packages/07/2b/29fd75ce45d22a39c61aad74f3d718e7ab67ccf839ca8b60866054eb15f8/numpy-2.3.5-cp314-cp314-win_arm64.whl", hash = "sha256:aeffcab3d4b43712bb7a60b65f6044d444e75e563ff6180af8f98dd4b905dfd2", size = 10476651, upload-time = "2025-11-16T22:51:47.749Z" },
- { url = "https://files.pythonhosted.org/packages/17/e1/f6a721234ebd4d87084cfa68d081bcba2f5cfe1974f7de4e0e8b9b2a2ba1/numpy-2.3.5-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:17531366a2e3a9e30762c000f2c43a9aaa05728712e25c11ce1dbe700c53ad41", size = 16834503, upload-time = "2025-11-16T22:51:50.443Z" },
- { url = "https://files.pythonhosted.org/packages/5c/1c/baf7ffdc3af9c356e1c135e57ab7cf8d247931b9554f55c467efe2c69eff/numpy-2.3.5-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:d21644de1b609825ede2f48be98dfde4656aefc713654eeee280e37cadc4e0ad", size = 12381612, upload-time = "2025-11-16T22:51:53.609Z" },
- { url = "https://files.pythonhosted.org/packages/74/91/f7f0295151407ddc9ba34e699013c32c3c91944f9b35fcf9281163dc1468/numpy-2.3.5-cp314-cp314t-macosx_14_0_arm64.whl", hash = "sha256:c804e3a5aba5460c73955c955bdbd5c08c354954e9270a2c1565f62e866bdc39", size = 5210042, upload-time = "2025-11-16T22:51:56.213Z" },
- { url = "https://files.pythonhosted.org/packages/2e/3b/78aebf345104ec50dd50a4d06ddeb46a9ff5261c33bcc58b1c4f12f85ec2/numpy-2.3.5-cp314-cp314t-macosx_14_0_x86_64.whl", hash = "sha256:cc0a57f895b96ec78969c34f682c602bf8da1a0270b09bc65673df2e7638ec20", size = 6724502, upload-time = "2025-11-16T22:51:58.584Z" },
- { url = "https://files.pythonhosted.org/packages/02/c6/7c34b528740512e57ef1b7c8337ab0b4f0bddf34c723b8996c675bc2bc91/numpy-2.3.5-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:900218e456384ea676e24ea6a0417f030a3b07306d29d7ad843957b40a9d8d52", size = 14308962, upload-time = "2025-11-16T22:52:01.698Z" },
- { url = "https://files.pythonhosted.org/packages/80/35/09d433c5262bc32d725bafc619e095b6a6651caf94027a03da624146f655/numpy-2.3.5-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:09a1bea522b25109bf8e6f3027bd810f7c1085c64a0c7ce050c1676ad0ba010b", size = 16655054, upload-time = "2025-11-16T22:52:04.267Z" },
- { url = "https://files.pythonhosted.org/packages/7a/ab/6a7b259703c09a88804fa2430b43d6457b692378f6b74b356155283566ac/numpy-2.3.5-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:04822c00b5fd0323c8166d66c701dc31b7fbd252c100acd708c48f763968d6a3", size = 16091613, upload-time = "2025-11-16T22:52:08.651Z" },
- { url = "https://files.pythonhosted.org/packages/c2/88/330da2071e8771e60d1038166ff9d73f29da37b01ec3eb43cb1427464e10/numpy-2.3.5-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:d6889ec4ec662a1a37eb4b4fb26b6100841804dac55bd9df579e326cdc146227", size = 18591147, upload-time = "2025-11-16T22:52:11.453Z" },
- { url = "https://files.pythonhosted.org/packages/51/41/851c4b4082402d9ea860c3626db5d5df47164a712cb23b54be028b184c1c/numpy-2.3.5-cp314-cp314t-win32.whl", hash = "sha256:93eebbcf1aafdf7e2ddd44c2923e2672e1010bddc014138b229e49725b4d6be5", size = 6479806, upload-time = "2025-11-16T22:52:14.641Z" },
- { url = "https://files.pythonhosted.org/packages/90/30/d48bde1dfd93332fa557cff1972fbc039e055a52021fbef4c2c4b1eefd17/numpy-2.3.5-cp314-cp314t-win_amd64.whl", hash = "sha256:c8a9958e88b65c3b27e22ca2a076311636850b612d6bbfb76e8d156aacde2aaf", size = 13105760, upload-time = "2025-11-16T22:52:17.975Z" },
- { url = "https://files.pythonhosted.org/packages/2d/fd/4b5eb0b3e888d86aee4d198c23acec7d214baaf17ea93c1adec94c9518b9/numpy-2.3.5-cp314-cp314t-win_arm64.whl", hash = "sha256:6203fdf9f3dc5bdaed7319ad8698e685c7a3be10819f41d32a0723e611733b42", size = 10545459, upload-time = "2025-11-16T22:52:20.55Z" },
- { url = "https://files.pythonhosted.org/packages/c6/65/f9dea8e109371ade9c782b4e4756a82edf9d3366bca495d84d79859a0b79/numpy-2.3.5-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:f0963b55cdd70fad460fa4c1341f12f976bb26cb66021a5580329bd498988310", size = 16910689, upload-time = "2025-11-16T22:52:23.247Z" },
- { url = "https://files.pythonhosted.org/packages/00/4f/edb00032a8fb92ec0a679d3830368355da91a69cab6f3e9c21b64d0bb986/numpy-2.3.5-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:f4255143f5160d0de972d28c8f9665d882b5f61309d8362fdd3e103cf7bf010c", size = 12457053, upload-time = "2025-11-16T22:52:26.367Z" },
- { url = "https://files.pythonhosted.org/packages/16/a4/e8a53b5abd500a63836a29ebe145fc1ab1f2eefe1cfe59276020373ae0aa/numpy-2.3.5-pp311-pypy311_pp73-macosx_14_0_arm64.whl", hash = "sha256:a4b9159734b326535f4dd01d947f919c6eefd2d9827466a696c44ced82dfbc18", size = 5285635, upload-time = "2025-11-16T22:52:29.266Z" },
- { url = "https://files.pythonhosted.org/packages/a3/2f/37eeb9014d9c8b3e9c55bc599c68263ca44fdbc12a93e45a21d1d56df737/numpy-2.3.5-pp311-pypy311_pp73-macosx_14_0_x86_64.whl", hash = "sha256:2feae0d2c91d46e59fcd62784a3a83b3fb677fead592ce51b5a6fbb4f95965ff", size = 6801770, upload-time = "2025-11-16T22:52:31.421Z" },
- { url = "https://files.pythonhosted.org/packages/7d/e4/68d2f474df2cb671b2b6c2986a02e520671295647dad82484cde80ca427b/numpy-2.3.5-pp311-pypy311_pp73-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ffac52f28a7849ad7576293c0cb7b9f08304e8f7d738a8cb8a90ec4c55a998eb", size = 14391768, upload-time = "2025-11-16T22:52:33.593Z" },
- { url = "https://files.pythonhosted.org/packages/b8/50/94ccd8a2b141cb50651fddd4f6a48874acb3c91c8f0842b08a6afc4b0b21/numpy-2.3.5-pp311-pypy311_pp73-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:63c0e9e7eea69588479ebf4a8a270d5ac22763cc5854e9a7eae952a3908103f7", size = 16729263, upload-time = "2025-11-16T22:52:36.369Z" },
- { url = "https://files.pythonhosted.org/packages/2d/ee/346fa473e666fe14c52fcdd19ec2424157290a032d4c41f98127bfb31ac7/numpy-2.3.5-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:f16417ec91f12f814b10bafe79ef77e70113a2f5f7018640e7425ff979253425", size = 12967213, upload-time = "2025-11-16T22:52:39.38Z" },
+sdist = { url = "https://files.pythonhosted.org/packages/a4/7a/6a3d14e205d292b738db449d0de649b373a59edb0d0b4493821d0a3e8718/numpy-2.4.0.tar.gz", hash = "sha256:6e504f7b16118198f138ef31ba24d985b124c2c469fe8467007cf30fd992f934", size = 20685720, upload-time = "2025-12-20T16:18:19.023Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/26/7e/7bae7cbcc2f8132271967aa03e03954fc1e48aa1f3bf32b29ca95fbef352/numpy-2.4.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:316b2f2584682318539f0bcaca5a496ce9ca78c88066579ebd11fd06f8e4741e", size = 16940166, upload-time = "2025-12-20T16:15:43.434Z" },
+ { url = "https://files.pythonhosted.org/packages/0f/27/6c13f5b46776d6246ec884ac5817452672156a506d08a1f2abb39961930a/numpy-2.4.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:a2718c1de8504121714234b6f8241d0019450353276c88b9453c9c3d92e101db", size = 12641781, upload-time = "2025-12-20T16:15:45.701Z" },
+ { url = "https://files.pythonhosted.org/packages/14/1c/83b4998d4860d15283241d9e5215f28b40ac31f497c04b12fa7f428ff370/numpy-2.4.0-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:21555da4ec4a0c942520ead42c3b0dc9477441e085c42b0fbdd6a084869a6f6b", size = 5470247, upload-time = "2025-12-20T16:15:47.943Z" },
+ { url = "https://files.pythonhosted.org/packages/54/08/cbce72c835d937795571b0464b52069f869c9e78b0c076d416c5269d2718/numpy-2.4.0-cp311-cp311-macosx_14_0_x86_64.whl", hash = "sha256:413aa561266a4be2d06cd2b9665e89d9f54c543f418773076a76adcf2af08bc7", size = 6799807, upload-time = "2025-12-20T16:15:49.795Z" },
+ { url = "https://files.pythonhosted.org/packages/ff/be/2e647961cd8c980591d75cdcd9e8f647d69fbe05e2a25613dc0a2ea5fb1a/numpy-2.4.0-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0feafc9e03128074689183031181fac0897ff169692d8492066e949041096548", size = 14701992, upload-time = "2025-12-20T16:15:51.615Z" },
+ { url = "https://files.pythonhosted.org/packages/a2/fb/e1652fb8b6fd91ce6ed429143fe2e01ce714711e03e5b762615e7b36172c/numpy-2.4.0-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a8fdfed3deaf1928fb7667d96e0567cdf58c2b370ea2ee7e586aa383ec2cb346", size = 16646871, upload-time = "2025-12-20T16:15:54.129Z" },
+ { url = "https://files.pythonhosted.org/packages/62/23/d841207e63c4322842f7cd042ae981cffe715c73376dcad8235fb31debf1/numpy-2.4.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:e06a922a469cae9a57100864caf4f8a97a1026513793969f8ba5b63137a35d25", size = 16487190, upload-time = "2025-12-20T16:15:56.147Z" },
+ { url = "https://files.pythonhosted.org/packages/bc/a0/6a842c8421ebfdec0a230e65f61e0dabda6edbef443d999d79b87c273965/numpy-2.4.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:927ccf5cd17c48f801f4ed43a7e5673a2724bd2171460be3e3894e6e332ef83a", size = 18580762, upload-time = "2025-12-20T16:15:58.524Z" },
+ { url = "https://files.pythonhosted.org/packages/0a/d1/c79e0046641186f2134dde05e6181825b911f8bdcef31b19ddd16e232847/numpy-2.4.0-cp311-cp311-win32.whl", hash = "sha256:882567b7ae57c1b1a0250208cc21a7976d8cbcc49d5a322e607e6f09c9e0bd53", size = 6233359, upload-time = "2025-12-20T16:16:00.938Z" },
+ { url = "https://files.pythonhosted.org/packages/fc/f0/74965001d231f28184d6305b8cdc1b6fcd4bf23033f6cb039cfe76c9fca7/numpy-2.4.0-cp311-cp311-win_amd64.whl", hash = "sha256:8b986403023c8f3bf8f487c2e6186afda156174d31c175f747d8934dfddf3479", size = 12601132, upload-time = "2025-12-20T16:16:02.484Z" },
+ { url = "https://files.pythonhosted.org/packages/65/32/55408d0f46dfebce38017f5bd931affa7256ad6beac1a92a012e1fbc67a7/numpy-2.4.0-cp311-cp311-win_arm64.whl", hash = "sha256:3f3096405acc48887458bbf9f6814d43785ac7ba2a57ea6442b581dedbc60ce6", size = 10573977, upload-time = "2025-12-20T16:16:04.77Z" },
+ { url = "https://files.pythonhosted.org/packages/8b/ff/f6400ffec95de41c74b8e73df32e3fff1830633193a7b1e409be7fb1bb8c/numpy-2.4.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:2a8b6bb8369abefb8bd1801b054ad50e02b3275c8614dc6e5b0373c305291037", size = 16653117, upload-time = "2025-12-20T16:16:06.709Z" },
+ { url = "https://files.pythonhosted.org/packages/fd/28/6c23e97450035072e8d830a3c411bf1abd1f42c611ff9d29e3d8f55c6252/numpy-2.4.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:2e284ca13d5a8367e43734148622caf0b261b275673823593e3e3634a6490f83", size = 12369711, upload-time = "2025-12-20T16:16:08.758Z" },
+ { url = "https://files.pythonhosted.org/packages/bc/af/acbef97b630ab1bb45e6a7d01d1452e4251aa88ce680ac36e56c272120ec/numpy-2.4.0-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:49ff32b09f5aa0cd30a20c2b39db3e669c845589f2b7fc910365210887e39344", size = 5198355, upload-time = "2025-12-20T16:16:10.902Z" },
+ { url = "https://files.pythonhosted.org/packages/c1/c8/4e0d436b66b826f2e53330adaa6311f5cac9871a5b5c31ad773b27f25a74/numpy-2.4.0-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:36cbfb13c152b1c7c184ddac43765db8ad672567e7bafff2cc755a09917ed2e6", size = 6545298, upload-time = "2025-12-20T16:16:12.607Z" },
+ { url = "https://files.pythonhosted.org/packages/ef/27/e1f5d144ab54eac34875e79037011d511ac57b21b220063310cb96c80fbc/numpy-2.4.0-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:35ddc8f4914466e6fc954c76527aa91aa763682a4f6d73249ef20b418fe6effb", size = 14398387, upload-time = "2025-12-20T16:16:14.257Z" },
+ { url = "https://files.pythonhosted.org/packages/67/64/4cb909dd5ab09a9a5d086eff9586e69e827b88a5585517386879474f4cf7/numpy-2.4.0-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:dc578891de1db95b2a35001b695451767b580bb45753717498213c5ff3c41d63", size = 16363091, upload-time = "2025-12-20T16:16:17.32Z" },
+ { url = "https://files.pythonhosted.org/packages/9d/9c/8efe24577523ec6809261859737cf117b0eb6fdb655abdfdc81b2e468ce4/numpy-2.4.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:98e81648e0b36e325ab67e46b5400a7a6d4a22b8a7c8e8bbfe20e7db7906bf95", size = 16176394, upload-time = "2025-12-20T16:16:19.524Z" },
+ { url = "https://files.pythonhosted.org/packages/61/f0/1687441ece7b47a62e45a1f82015352c240765c707928edd8aef875d5951/numpy-2.4.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:d57b5046c120561ba8fa8e4030fbb8b822f3063910fa901ffadf16e2b7128ad6", size = 18287378, upload-time = "2025-12-20T16:16:22.866Z" },
+ { url = "https://files.pythonhosted.org/packages/d3/6f/f868765d44e6fc466467ed810ba9d8d6db1add7d4a748abfa2a4c99a3194/numpy-2.4.0-cp312-cp312-win32.whl", hash = "sha256:92190db305a6f48734d3982f2c60fa30d6b5ee9bff10f2887b930d7b40119f4c", size = 5955432, upload-time = "2025-12-20T16:16:25.06Z" },
+ { url = "https://files.pythonhosted.org/packages/d4/b5/94c1e79fcbab38d1ca15e13777477b2914dd2d559b410f96949d6637b085/numpy-2.4.0-cp312-cp312-win_amd64.whl", hash = "sha256:680060061adb2d74ce352628cb798cfdec399068aa7f07ba9fb818b2b3305f98", size = 12306201, upload-time = "2025-12-20T16:16:26.979Z" },
+ { url = "https://files.pythonhosted.org/packages/70/09/c39dadf0b13bb0768cd29d6a3aaff1fb7c6905ac40e9aaeca26b1c086e06/numpy-2.4.0-cp312-cp312-win_arm64.whl", hash = "sha256:39699233bc72dd482da1415dcb06076e32f60eddc796a796c5fb6c5efce94667", size = 10308234, upload-time = "2025-12-20T16:16:29.417Z" },
+ { url = "https://files.pythonhosted.org/packages/a7/0d/853fd96372eda07c824d24adf02e8bc92bb3731b43a9b2a39161c3667cc4/numpy-2.4.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:a152d86a3ae00ba5f47b3acf3b827509fd0b6cb7d3259665e63dafbad22a75ea", size = 16649088, upload-time = "2025-12-20T16:16:31.421Z" },
+ { url = "https://files.pythonhosted.org/packages/e3/37/cc636f1f2a9f585434e20a3e6e63422f70bfe4f7f6698e941db52ea1ac9a/numpy-2.4.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:39b19251dec4de8ff8496cd0806cbe27bf0684f765abb1f4809554de93785f2d", size = 12364065, upload-time = "2025-12-20T16:16:33.491Z" },
+ { url = "https://files.pythonhosted.org/packages/ed/69/0b78f37ca3690969beee54103ce5f6021709134e8020767e93ba691a72f1/numpy-2.4.0-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:009bd0ea12d3c784b6639a8457537016ce5172109e585338e11334f6a7bb88ee", size = 5192640, upload-time = "2025-12-20T16:16:35.636Z" },
+ { url = "https://files.pythonhosted.org/packages/1d/2a/08569f8252abf590294dbb09a430543ec8f8cc710383abfb3e75cc73aeda/numpy-2.4.0-cp313-cp313-macosx_14_0_x86_64.whl", hash = "sha256:5fe44e277225fd3dff6882d86d3d447205d43532c3627313d17e754fb3905a0e", size = 6541556, upload-time = "2025-12-20T16:16:37.276Z" },
+ { url = "https://files.pythonhosted.org/packages/93/e9/a949885a4e177493d61519377952186b6cbfdf1d6002764c664ba28349b5/numpy-2.4.0-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f935c4493eda9069851058fa0d9e39dbf6286be690066509305e52912714dbb2", size = 14396562, upload-time = "2025-12-20T16:16:38.953Z" },
+ { url = "https://files.pythonhosted.org/packages/99/98/9d4ad53b0e9ef901c2ef1d550d2136f5ac42d3fd2988390a6def32e23e48/numpy-2.4.0-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8cfa5f29a695cb7438965e6c3e8d06e0416060cf0d709c1b1c1653a939bf5c2a", size = 16351719, upload-time = "2025-12-20T16:16:41.503Z" },
+ { url = "https://files.pythonhosted.org/packages/28/de/5f3711a38341d6e8dd619f6353251a0cdd07f3d6d101a8fd46f4ef87f895/numpy-2.4.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:ba0cb30acd3ef11c94dc27fbfba68940652492bc107075e7ffe23057f9425681", size = 16176053, upload-time = "2025-12-20T16:16:44.552Z" },
+ { url = "https://files.pythonhosted.org/packages/2a/5b/2a3753dc43916501b4183532e7ace862e13211042bceafa253afb5c71272/numpy-2.4.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:60e8c196cd82cbbd4f130b5290007e13e6de3eca79f0d4d38014769d96a7c475", size = 18277859, upload-time = "2025-12-20T16:16:47.174Z" },
+ { url = "https://files.pythonhosted.org/packages/2c/c5/a18bcdd07a941db3076ef489d036ab16d2bfc2eae0cf27e5a26e29189434/numpy-2.4.0-cp313-cp313-win32.whl", hash = "sha256:5f48cb3e88fbc294dc90e215d86fbaf1c852c63dbdb6c3a3e63f45c4b57f7344", size = 5953849, upload-time = "2025-12-20T16:16:49.554Z" },
+ { url = "https://files.pythonhosted.org/packages/4f/f1/719010ff8061da6e8a26e1980cf090412d4f5f8060b31f0c45d77dd67a01/numpy-2.4.0-cp313-cp313-win_amd64.whl", hash = "sha256:a899699294f28f7be8992853c0c60741f16ff199205e2e6cdca155762cbaa59d", size = 12302840, upload-time = "2025-12-20T16:16:51.227Z" },
+ { url = "https://files.pythonhosted.org/packages/f5/5a/b3d259083ed8b4d335270c76966cb6cf14a5d1b69e1a608994ac57a659e6/numpy-2.4.0-cp313-cp313-win_arm64.whl", hash = "sha256:9198f447e1dc5647d07c9a6bbe2063cc0132728cc7175b39dbc796da5b54920d", size = 10308509, upload-time = "2025-12-20T16:16:53.313Z" },
+ { url = "https://files.pythonhosted.org/packages/31/01/95edcffd1bb6c0633df4e808130545c4f07383ab629ac7e316fb44fff677/numpy-2.4.0-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:74623f2ab5cc3f7c886add4f735d1031a1d2be4a4ae63c0546cfd74e7a31ddf6", size = 12491815, upload-time = "2025-12-20T16:16:55.496Z" },
+ { url = "https://files.pythonhosted.org/packages/59/ea/5644b8baa92cc1c7163b4b4458c8679852733fa74ca49c942cfa82ded4e0/numpy-2.4.0-cp313-cp313t-macosx_14_0_arm64.whl", hash = "sha256:0804a8e4ab070d1d35496e65ffd3cf8114c136a2b81f61dfab0de4b218aacfd5", size = 5320321, upload-time = "2025-12-20T16:16:57.468Z" },
+ { url = "https://files.pythonhosted.org/packages/26/4e/e10938106d70bc21319bd6a86ae726da37edc802ce35a3a71ecdf1fdfe7f/numpy-2.4.0-cp313-cp313t-macosx_14_0_x86_64.whl", hash = "sha256:02a2038eb27f9443a8b266a66911e926566b5a6ffd1a689b588f7f35b81e7dc3", size = 6641635, upload-time = "2025-12-20T16:16:59.379Z" },
+ { url = "https://files.pythonhosted.org/packages/b3/8d/a8828e3eaf5c0b4ab116924df82f24ce3416fa38d0674d8f708ddc6c8aac/numpy-2.4.0-cp313-cp313t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1889b3a3f47a7b5bee16bc25a2145bd7cb91897f815ce3499db64c7458b6d91d", size = 14456053, upload-time = "2025-12-20T16:17:01.768Z" },
+ { url = "https://files.pythonhosted.org/packages/68/a1/17d97609d87d4520aa5ae2dcfb32305654550ac6a35effb946d303e594ce/numpy-2.4.0-cp313-cp313t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:85eef4cb5625c47ee6425c58a3502555e10f45ee973da878ac8248ad58c136f3", size = 16401702, upload-time = "2025-12-20T16:17:04.235Z" },
+ { url = "https://files.pythonhosted.org/packages/18/32/0f13c1b2d22bea1118356b8b963195446f3af124ed7a5adfa8fdecb1b6ca/numpy-2.4.0-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:6dc8b7e2f4eb184b37655195f421836cfae6f58197b67e3ffc501f1333d993fa", size = 16242493, upload-time = "2025-12-20T16:17:06.856Z" },
+ { url = "https://files.pythonhosted.org/packages/ae/23/48f21e3d309fbc137c068a1475358cbd3a901b3987dcfc97a029ab3068e2/numpy-2.4.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:44aba2f0cafd287871a495fb3163408b0bd25bbce135c6f621534a07f4f7875c", size = 18324222, upload-time = "2025-12-20T16:17:09.392Z" },
+ { url = "https://files.pythonhosted.org/packages/ac/52/41f3d71296a3dcaa4f456aaa3c6fc8e745b43d0552b6bde56571bb4b4a0f/numpy-2.4.0-cp313-cp313t-win32.whl", hash = "sha256:20c115517513831860c573996e395707aa9fb691eb179200125c250e895fcd93", size = 6076216, upload-time = "2025-12-20T16:17:11.437Z" },
+ { url = "https://files.pythonhosted.org/packages/35/ff/46fbfe60ab0710d2a2b16995f708750307d30eccbb4c38371ea9e986866e/numpy-2.4.0-cp313-cp313t-win_amd64.whl", hash = "sha256:b48e35f4ab6f6a7597c46e301126ceba4c44cd3280e3750f85db48b082624fa4", size = 12444263, upload-time = "2025-12-20T16:17:13.182Z" },
+ { url = "https://files.pythonhosted.org/packages/a3/e3/9189ab319c01d2ed556c932ccf55064c5d75bb5850d1df7a482ce0badead/numpy-2.4.0-cp313-cp313t-win_arm64.whl", hash = "sha256:4d1cfce39e511069b11e67cd0bd78ceff31443b7c9e5c04db73c7a19f572967c", size = 10378265, upload-time = "2025-12-20T16:17:15.211Z" },
+ { url = "https://files.pythonhosted.org/packages/ab/ed/52eac27de39d5e5a6c9aadabe672bc06f55e24a3d9010cd1183948055d76/numpy-2.4.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:c95eb6db2884917d86cde0b4d4cf31adf485c8ec36bf8696dd66fa70de96f36b", size = 16647476, upload-time = "2025-12-20T16:17:17.671Z" },
+ { url = "https://files.pythonhosted.org/packages/77/c0/990ce1b7fcd4e09aeaa574e2a0a839589e4b08b2ca68070f1acb1fea6736/numpy-2.4.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:65167da969cd1ec3a1df31cb221ca3a19a8aaa25370ecb17d428415e93c1935e", size = 12374563, upload-time = "2025-12-20T16:17:20.216Z" },
+ { url = "https://files.pythonhosted.org/packages/37/7c/8c5e389c6ae8f5fd2277a988600d79e9625db3fff011a2d87ac80b881a4c/numpy-2.4.0-cp314-cp314-macosx_14_0_arm64.whl", hash = "sha256:3de19cfecd1465d0dcf8a5b5ea8b3155b42ed0b639dba4b71e323d74f2a3be5e", size = 5203107, upload-time = "2025-12-20T16:17:22.47Z" },
+ { url = "https://files.pythonhosted.org/packages/e6/94/ca5b3bd6a8a70a5eec9a0b8dd7f980c1eff4b8a54970a9a7fef248ef564f/numpy-2.4.0-cp314-cp314-macosx_14_0_x86_64.whl", hash = "sha256:6c05483c3136ac4c91b4e81903cb53a8707d316f488124d0398499a4f8e8ef51", size = 6538067, upload-time = "2025-12-20T16:17:24.001Z" },
+ { url = "https://files.pythonhosted.org/packages/79/43/993eb7bb5be6761dde2b3a3a594d689cec83398e3f58f4758010f3b85727/numpy-2.4.0-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:36667db4d6c1cea79c8930ab72fadfb4060feb4bfe724141cd4bd064d2e5f8ce", size = 14411926, upload-time = "2025-12-20T16:17:25.822Z" },
+ { url = "https://files.pythonhosted.org/packages/03/75/d4c43b61de473912496317a854dac54f1efec3eeb158438da6884b70bb90/numpy-2.4.0-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9a818668b674047fd88c4cddada7ab8f1c298812783e8328e956b78dc4807f9f", size = 16354295, upload-time = "2025-12-20T16:17:28.308Z" },
+ { url = "https://files.pythonhosted.org/packages/b8/0a/b54615b47ee8736a6461a4bb6749128dd3435c5a759d5663f11f0e9af4ac/numpy-2.4.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:1ee32359fb7543b7b7bd0b2f46294db27e29e7bbdf70541e81b190836cd83ded", size = 16190242, upload-time = "2025-12-20T16:17:30.993Z" },
+ { url = "https://files.pythonhosted.org/packages/98/ce/ea207769aacad6246525ec6c6bbd66a2bf56c72443dc10e2f90feed29290/numpy-2.4.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:e493962256a38f58283de033d8af176c5c91c084ea30f15834f7545451c42059", size = 18280875, upload-time = "2025-12-20T16:17:33.327Z" },
+ { url = "https://files.pythonhosted.org/packages/17/ef/ec409437aa962ea372ed601c519a2b141701683ff028f894b7466f0ab42b/numpy-2.4.0-cp314-cp314-win32.whl", hash = "sha256:6bbaebf0d11567fa8926215ae731e1d58e6ec28a8a25235b8a47405d301332db", size = 6002530, upload-time = "2025-12-20T16:17:35.729Z" },
+ { url = "https://files.pythonhosted.org/packages/5f/4a/5cb94c787a3ed1ac65e1271b968686521169a7b3ec0b6544bb3ca32960b0/numpy-2.4.0-cp314-cp314-win_amd64.whl", hash = "sha256:3d857f55e7fdf7c38ab96c4558c95b97d1c685be6b05c249f5fdafcbd6f9899e", size = 12435890, upload-time = "2025-12-20T16:17:37.599Z" },
+ { url = "https://files.pythonhosted.org/packages/48/a0/04b89db963af9de1104975e2544f30de89adbf75b9e75f7dd2599be12c79/numpy-2.4.0-cp314-cp314-win_arm64.whl", hash = "sha256:bb50ce5fb202a26fd5404620e7ef820ad1ab3558b444cb0b55beb7ef66cd2d63", size = 10591892, upload-time = "2025-12-20T16:17:39.649Z" },
+ { url = "https://files.pythonhosted.org/packages/53/e5/d74b5ccf6712c06c7a545025a6a71bfa03bdc7e0568b405b0d655232fd92/numpy-2.4.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:355354388cba60f2132df297e2d53053d4063f79077b67b481d21276d61fc4df", size = 12494312, upload-time = "2025-12-20T16:17:41.714Z" },
+ { url = "https://files.pythonhosted.org/packages/c2/08/3ca9cc2ddf54dfee7ae9a6479c071092a228c68aef08252aa08dac2af002/numpy-2.4.0-cp314-cp314t-macosx_14_0_arm64.whl", hash = "sha256:1d8f9fde5f6dc1b6fc34df8162f3b3079365468703fee7f31d4e0cc8c63baed9", size = 5322862, upload-time = "2025-12-20T16:17:44.145Z" },
+ { url = "https://files.pythonhosted.org/packages/87/74/0bb63a68394c0c1e52670cfff2e309afa41edbe11b3327d9af29e4383f34/numpy-2.4.0-cp314-cp314t-macosx_14_0_x86_64.whl", hash = "sha256:e0434aa22c821f44eeb4c650b81c7fbdd8c0122c6c4b5a576a76d5a35625ecd9", size = 6644986, upload-time = "2025-12-20T16:17:46.203Z" },
+ { url = "https://files.pythonhosted.org/packages/06/8f/9264d9bdbcf8236af2823623fe2f3981d740fc3461e2787e231d97c38c28/numpy-2.4.0-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:40483b2f2d3ba7aad426443767ff5632ec3156ef09742b96913787d13c336471", size = 14457958, upload-time = "2025-12-20T16:17:48.017Z" },
+ { url = "https://files.pythonhosted.org/packages/8c/d9/f9a69ae564bbc7236a35aa883319364ef5fd41f72aa320cc1cbe66148fe2/numpy-2.4.0-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d9e6a7664ddd9746e20b7325351fe1a8408d0a2bf9c63b5e898290ddc8f09544", size = 16398394, upload-time = "2025-12-20T16:17:50.409Z" },
+ { url = "https://files.pythonhosted.org/packages/34/c7/39241501408dde7f885d241a98caba5421061a2c6d2b2197ac5e3aa842d8/numpy-2.4.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:ecb0019d44f4cdb50b676c5d0cb4b1eae8e15d1ed3d3e6639f986fc92b2ec52c", size = 16241044, upload-time = "2025-12-20T16:17:52.661Z" },
+ { url = "https://files.pythonhosted.org/packages/7c/95/cae7effd90e065a95e59fe710eeee05d7328ed169776dfdd9f789e032125/numpy-2.4.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:d0ffd9e2e4441c96a9c91ec1783285d80bf835b677853fc2770a89d50c1e48ac", size = 18321772, upload-time = "2025-12-20T16:17:54.947Z" },
+ { url = "https://files.pythonhosted.org/packages/96/df/3c6c279accd2bfb968a76298e5b276310bd55d243df4fa8ac5816d79347d/numpy-2.4.0-cp314-cp314t-win32.whl", hash = "sha256:77f0d13fa87036d7553bf81f0e1fe3ce68d14c9976c9851744e4d3e91127e95f", size = 6148320, upload-time = "2025-12-20T16:17:57.249Z" },
+ { url = "https://files.pythonhosted.org/packages/92/8d/f23033cce252e7a75cae853d17f582e86534c46404dea1c8ee094a9d6d84/numpy-2.4.0-cp314-cp314t-win_amd64.whl", hash = "sha256:b1f5b45829ac1848893f0ddf5cb326110604d6df96cdc255b0bf9edd154104d4", size = 12623460, upload-time = "2025-12-20T16:17:58.963Z" },
+ { url = "https://files.pythonhosted.org/packages/a4/4f/1f8475907d1a7c4ef9020edf7f39ea2422ec896849245f00688e4b268a71/numpy-2.4.0-cp314-cp314t-win_arm64.whl", hash = "sha256:23a3e9d1a6f360267e8fbb38ba5db355a6a7e9be71d7fce7ab3125e88bb646c8", size = 10661799, upload-time = "2025-12-20T16:18:01.078Z" },
+ { url = "https://files.pythonhosted.org/packages/4b/ef/088e7c7342f300aaf3ee5f2c821c4b9996a1bef2aaf6a49cc8ab4883758e/numpy-2.4.0-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:b54c83f1c0c0f1d748dca0af516062b8829d53d1f0c402be24b4257a9c48ada6", size = 16819003, upload-time = "2025-12-20T16:18:03.41Z" },
+ { url = "https://files.pythonhosted.org/packages/ff/ce/a53017b5443b4b84517182d463fc7bcc2adb4faa8b20813f8e5f5aeb5faa/numpy-2.4.0-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:aabb081ca0ec5d39591fc33018cd4b3f96e1a2dd6756282029986d00a785fba4", size = 12567105, upload-time = "2025-12-20T16:18:05.594Z" },
+ { url = "https://files.pythonhosted.org/packages/77/58/5ff91b161f2ec650c88a626c3905d938c89aaadabd0431e6d9c1330c83e2/numpy-2.4.0-pp311-pypy311_pp73-macosx_14_0_arm64.whl", hash = "sha256:8eafe7c36c8430b7794edeab3087dec7bf31d634d92f2af9949434b9d1964cba", size = 5395590, upload-time = "2025-12-20T16:18:08.031Z" },
+ { url = "https://files.pythonhosted.org/packages/1d/4e/f1a084106df8c2df8132fc437e56987308e0524836aa7733721c8429d4fe/numpy-2.4.0-pp311-pypy311_pp73-macosx_14_0_x86_64.whl", hash = "sha256:2f585f52b2baf07ff3356158d9268ea095e221371f1074fadea2f42544d58b4d", size = 6709947, upload-time = "2025-12-20T16:18:09.836Z" },
+ { url = "https://files.pythonhosted.org/packages/63/09/3d8aeb809c0332c3f642da812ac2e3d74fc9252b3021f8c30c82e99e3f3d/numpy-2.4.0-pp311-pypy311_pp73-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:32ed06d0fe9cae27d8fb5f400c63ccee72370599c75e683a6358dd3a4fb50aaf", size = 14535119, upload-time = "2025-12-20T16:18:12.105Z" },
+ { url = "https://files.pythonhosted.org/packages/fd/7f/68f0fc43a2cbdc6bb239160c754d87c922f60fbaa0fa3cd3d312b8a7f5ee/numpy-2.4.0-pp311-pypy311_pp73-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:57c540ed8fb1f05cb997c6761cd56db72395b0d6985e90571ff660452ade4f98", size = 16475815, upload-time = "2025-12-20T16:18:14.433Z" },
+ { url = "https://files.pythonhosted.org/packages/11/73/edeacba3167b1ca66d51b1a5a14697c2c40098b5ffa01811c67b1785a5ab/numpy-2.4.0-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:a39fb973a726e63223287adc6dafe444ce75af952d711e400f3bf2b36ef55a7b", size = 12489376, upload-time = "2025-12-20T16:18:16.524Z" },
]
[[package]]
name = "nv-grouped-gemm"
-version = "1.1.4.post6"
+version = "1.1.4.post8"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "absl-py" },
{ name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version < '3.11' and extra == 'extra-13-megatron-core-dev') or (python_full_version < '3.11' and extra == 'extra-13-megatron-core-lts') or (extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" },
- { name = "numpy", version = "2.3.5", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version >= '3.11' and extra == 'extra-13-megatron-core-dev') or (python_full_version >= '3.11' and extra == 'extra-13-megatron-core-lts') or (extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" },
+ { name = "numpy", version = "2.4.0", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version >= '3.11' and extra == 'extra-13-megatron-core-dev') or (python_full_version >= '3.11' and extra == 'extra-13-megatron-core-lts') or (extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" },
{ name = "torch", marker = "sys_platform == 'never'" },
]
-sdist = { url = "https://files.pythonhosted.org/packages/05/79/87c45f32e661b25e0aaa1e325ba166511f57be5dff8f0fcabc12d3e73b64/nv_grouped_gemm-1.1.4.post6.tar.gz", hash = "sha256:dad6115f4b4ff7ceb0bc40ad44e923c13a24fc88cfe1e20b1a6b4c9cf24c445c", size = 26508, upload-time = "2025-10-10T18:52:29.508Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/02/ad/046a097b63a96c1ba1d85f0031dbe7fcbdb33e6c445dfbaba2ffaefdd497/nv_grouped_gemm-1.1.4.post8.tar.gz", hash = "sha256:ab321693f0292cfd8a26dc7b6f14decd9eb00e209494de7218e4fad36191275d", size = 20821209, upload-time = "2025-12-17T02:22:38.432Z" }
[[package]]
name = "nv-one-logger-core"
@@ -3123,21 +3149,21 @@ wheels = [
[[package]]
name = "nvidia-cudnn-frontend"
-version = "1.16.0"
+version = "1.17.0"
source = { registry = "https://pypi.org/simple" }
wheels = [
- { url = "https://files.pythonhosted.org/packages/fa/cf/3cd3cc682df5488288c6043fc0977090497ff015a082ab160076fecb080a/nvidia_cudnn_frontend-1.16.0-cp310-cp310-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:83ecbe6d1145dc208a9ae82aa0b45b2c8f74ed8a43d3a102a13eef2117e2fedd", size = 1835542, upload-time = "2025-11-07T01:28:20.133Z" },
- { url = "https://files.pythonhosted.org/packages/92/45/87f3f2d94a928be21459949b03b0b8bcea13531d30094ad84a8ae4fca761/nvidia_cudnn_frontend-1.16.0-cp310-cp310-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:77cb06b91877c8489363867434ba1d9936f3e10bf7ed98d82e98f5f578611920", size = 1950339, upload-time = "2025-11-07T01:31:41.69Z" },
- { url = "https://files.pythonhosted.org/packages/be/f5/1662f18084ef4441bfb3a01383cbf77194905b53474dcb51c0d0f373c74b/nvidia_cudnn_frontend-1.16.0-cp310-cp310-win_amd64.whl", hash = "sha256:ee3f3886f107919dad48cbc905fa6ae9207c8d7d5a24165e55625ea96f0fe40f", size = 1367883, upload-time = "2025-11-07T01:25:17.791Z" },
- { url = "https://files.pythonhosted.org/packages/10/b7/d0a3a337f5e83f26ff79a7fd63a859181ff2911f1d905d6fbab5fc80170d/nvidia_cudnn_frontend-1.16.0-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c360d5840d6eb597aade9e9c8780e24aec283b8e6bc97d52881c821a35c92aa9", size = 1837573, upload-time = "2025-11-07T01:29:05.507Z" },
- { url = "https://files.pythonhosted.org/packages/95/dc/465a14f2d235778405f2e84fce336d07ab045bf1c7df6404bdf8033e06a8/nvidia_cudnn_frontend-1.16.0-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5c4a8fc573d85a86e08b15d9bf37f729e2487298781867a492a59cde6ac295e2", size = 1952630, upload-time = "2025-11-07T01:32:00.242Z" },
- { url = "https://files.pythonhosted.org/packages/3b/89/f14435f616603a999975930c4456d6140127f6acb19a877c752beccad837/nvidia_cudnn_frontend-1.16.0-cp311-cp311-win_amd64.whl", hash = "sha256:a257f10a932ffde9741f644efd3611acf77e2fd89d493d81bc6a8353c48f1ec2", size = 1368775, upload-time = "2025-11-07T01:25:42.252Z" },
- { url = "https://files.pythonhosted.org/packages/00/39/79b606e805abd67ab4fa72f752a5413a496159f10d94fbdb1d67bb5ae86c/nvidia_cudnn_frontend-1.16.0-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:dd6fdd71c0896ff2ca1809d914cbd17f2904d55863f8881f47946e1d634c7a88", size = 1839271, upload-time = "2025-11-07T01:29:53.06Z" },
- { url = "https://files.pythonhosted.org/packages/09/21/a0e0d50ba8d7b639fe635500fee0d9c0319561b1ae72176d7024ec04b439/nvidia_cudnn_frontend-1.16.0-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:16efb069d4bda4d3b99134f59f376cfd4d09558298bd96af778fdc7f2851e696", size = 1954062, upload-time = "2025-11-07T01:32:18.556Z" },
- { url = "https://files.pythonhosted.org/packages/ce/d6/30ae67bb9c010e9459d1211c56d73373eb4e3dd9f57f4c3c1fe0966efcb1/nvidia_cudnn_frontend-1.16.0-cp312-cp312-win_amd64.whl", hash = "sha256:7b7860db03767c158accbe0b4e9c9553506513cc970ff08ed28c7761681ac466", size = 1368435, upload-time = "2025-11-07T01:26:28.022Z" },
- { url = "https://files.pythonhosted.org/packages/32/2c/b4376afef0a6342c56e82e3465c1f8f5c719f588293a50dd04019a22ae6e/nvidia_cudnn_frontend-1.16.0-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b6bcb3a2fbff80538958e21e2227520f082a961164865aaeedaac527f61084f9", size = 1839805, upload-time = "2025-11-07T01:30:31.056Z" },
- { url = "https://files.pythonhosted.org/packages/71/13/836b90354036154ab82db3861210e5736983fe1fc44bb39c146ad93b333b/nvidia_cudnn_frontend-1.16.0-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:cbdad88b2bec5dde837f8fa7632022334cddb4756f923b5421c06a712cb59d31", size = 1953953, upload-time = "2025-11-07T01:33:03.781Z" },
- { url = "https://files.pythonhosted.org/packages/e5/30/3025f34f2c86ceef85134dc1f323f8cf2a26d3ffddc5ada48528c80bfae1/nvidia_cudnn_frontend-1.16.0-cp313-cp313-win_amd64.whl", hash = "sha256:138de2bc4697fabb2eb2f0f601a7e31f8fe97874908e26e33d737276f335473c", size = 1368359, upload-time = "2025-11-07T01:26:51.561Z" },
+ { url = "https://files.pythonhosted.org/packages/14/94/b224e65becfb5ab02c5b331aeb73c98f6d95cde5326d7698a2fc0d20e84a/nvidia_cudnn_frontend-1.17.0-cp310-cp310-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4835ee3fc350782c89cdd290088ade69464faaa5dd66ccb0b215ad481ab3b41b", size = 1911670, upload-time = "2025-12-20T00:26:36.302Z" },
+ { url = "https://files.pythonhosted.org/packages/d5/05/54afda6fc47838bd68a029067d8019e6b495dca0570d7e970cbb2c3e0b32/nvidia_cudnn_frontend-1.17.0-cp310-cp310-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1da7e972dbba939ad21111f1208815b8c8024cbf72aa6c1eb223b14b2049d4b6", size = 2033618, upload-time = "2025-12-20T00:24:42.991Z" },
+ { url = "https://files.pythonhosted.org/packages/83/97/77ad90fac9372b0420885f16a2afaca95f78b082fa9d6a082d51a7c96bd3/nvidia_cudnn_frontend-1.17.0-cp310-cp310-win_amd64.whl", hash = "sha256:21c5b2ce097f72c6510cbf974ce8ea9a31b34989dd9209d7187584a6100e57e5", size = 1440589, upload-time = "2025-12-20T00:29:17.641Z" },
+ { url = "https://files.pythonhosted.org/packages/4e/4a/a903c57ef5aaa32aa074007ba4d50ed7cbc80a8092ddb84fe9d879a69bbb/nvidia_cudnn_frontend-1.17.0-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:961004000a2c21dd4a03f816534629105cf49125a643dbb49abbc97021e66d20", size = 1911775, upload-time = "2025-12-20T00:27:11.297Z" },
+ { url = "https://files.pythonhosted.org/packages/15/20/80c4f5d62ebc58b8db8d25a2ee11f3246bb8947addea37c229540bcc05ac/nvidia_cudnn_frontend-1.17.0-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6ea44a8f2c0cfd20868b239ea13a2e0f32895dab868f6ff2bee01caf3778d273", size = 2035158, upload-time = "2025-12-20T00:25:00.9Z" },
+ { url = "https://files.pythonhosted.org/packages/5f/18/c24375c8d579c53a99a2d7428397288a94c7ea411d1823e3b8dc3cef50dc/nvidia_cudnn_frontend-1.17.0-cp311-cp311-win_amd64.whl", hash = "sha256:8dd6cc197a58d63da4d146a1febc1f99d425374d159f9b00628b140c65acb486", size = 1441316, upload-time = "2025-12-20T00:29:34.951Z" },
+ { url = "https://files.pythonhosted.org/packages/42/d9/f58ed6292c9396f7422812a0a2d9f80cc5a623ea6c758bcb3d34d4795bb8/nvidia_cudnn_frontend-1.17.0-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:de0c473f32d705abcf14f351615f7ffbeed7320e3499cf2195ae5689652a2592", size = 1917620, upload-time = "2025-12-20T00:27:46.179Z" },
+ { url = "https://files.pythonhosted.org/packages/db/eb/c641135632bd2afc21339aadee96af4c5db1460dfa07ca74836de75a590f/nvidia_cudnn_frontend-1.17.0-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c913c87fca691a91385287f2587575531933acfebc85c33dbcecb191886c7a53", size = 2038994, upload-time = "2025-12-20T00:25:18.9Z" },
+ { url = "https://files.pythonhosted.org/packages/82/49/a92da03eb43bde90be770a43666c5ab26b4f8b15f6e46c4b0b0e84f37994/nvidia_cudnn_frontend-1.17.0-cp312-cp312-win_amd64.whl", hash = "sha256:a0d4cfd03961592108abd1ba246e43c8bb7540aed984df860256d0bff181de98", size = 1441271, upload-time = "2025-12-20T00:29:52.056Z" },
+ { url = "https://files.pythonhosted.org/packages/99/96/4d55a559dff3175599fe15d83c853f051526b91994b083ec36b12caae776/nvidia_cudnn_frontend-1.17.0-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3800a1fe3d41a9206281475b1c8c438b02cb7e3c7e262d13f0a101edec223cb6", size = 1917065, upload-time = "2025-12-20T00:28:21.402Z" },
+ { url = "https://files.pythonhosted.org/packages/20/f6/5af63c254d7260dd1e974b2300eae9b157998b9d958f79c98ddaada0a0bf/nvidia_cudnn_frontend-1.17.0-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5adaf4a930b3be5ed019e1a25cfec7cc2bf444592a54a7639c28149b9227c2a4", size = 2039180, upload-time = "2025-12-20T00:25:36.695Z" },
+ { url = "https://files.pythonhosted.org/packages/64/ee/6de6aec1e42c859134312e6d5348d6f036b2f1b825e6eae92f9a429eccc4/nvidia_cudnn_frontend-1.17.0-cp313-cp313-win_amd64.whl", hash = "sha256:5c6a120fb54b157585ce6587153fc7086081af961f284f2553e01ba7c7a80c1a", size = 1441177, upload-time = "2025-12-20T00:30:09.927Z" },
]
[[package]]
@@ -3212,23 +3238,23 @@ wheels = [
[[package]]
name = "nvidia-cutlass-dsl"
-version = "4.3.2"
+version = "4.3.4"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "cuda-python" },
{ name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version < '3.11' and extra == 'extra-13-megatron-core-dev') or (python_full_version < '3.11' and extra == 'extra-13-megatron-core-lts') or (extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" },
- { name = "numpy", version = "2.3.5", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version >= '3.11' and extra == 'extra-13-megatron-core-dev') or (python_full_version >= '3.11' and extra == 'extra-13-megatron-core-lts') or (extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" },
+ { name = "numpy", version = "2.4.0", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version >= '3.11' and extra == 'extra-13-megatron-core-dev') or (python_full_version >= '3.11' and extra == 'extra-13-megatron-core-lts') or (extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" },
{ name = "typing-extensions" },
]
wheels = [
- { url = "https://files.pythonhosted.org/packages/b5/ac/81de35349a9c427bfb0c1c9e652c80d7b48009f55241af7200184ffffe7f/nvidia_cutlass_dsl-4.3.2-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:ea7bccdb2bf45da0a7ecd731fb8e076fcd4e83c8a3789cd71f0cee6583c3d37c", size = 58726575, upload-time = "2025-12-04T07:39:54.233Z" },
- { url = "https://files.pythonhosted.org/packages/35/9a/8f5d5c534407fa730623e41e379d8814e6afd3061e882f9c9fb1c857e3a7/nvidia_cutlass_dsl-4.3.2-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:f138a08efc2d22e2bff93e044e2981e64362dc4129548dd53023e9444dd3828c", size = 58594145, upload-time = "2025-12-04T07:42:46.57Z" },
- { url = "https://files.pythonhosted.org/packages/4f/55/4a0a07ed6a0e55998c1b7998b829dc4a6e77353a04aaa360ef14f1178843/nvidia_cutlass_dsl-4.3.2-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:120fa6779cf9f01dc5f0cf972c833203a604bbd2cdc91c84955bde59d9a5f4f1", size = 58722268, upload-time = "2025-12-04T07:38:26.712Z" },
- { url = "https://files.pythonhosted.org/packages/ed/f8/35b831eae98e15bda7ca7ef041dc73a5f8bd29c84ef8b944aa60aa119b5e/nvidia_cutlass_dsl-4.3.2-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:cc5b53796aff9a3a428292d837f5f28298a2d171cb9508573d8ce409acabd194", size = 58594891, upload-time = "2025-12-04T07:41:59.75Z" },
- { url = "https://files.pythonhosted.org/packages/cf/e2/429f98e8385af6f0df7d3fcc15fbc8e2677c9257a486f9e6cb3b21444290/nvidia_cutlass_dsl-4.3.2-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:8c2d998cdfadb65b1175def1e6f40618a6cbd4181b59e145c3a06469a95997ef", size = 58725911, upload-time = "2025-12-04T07:41:07.992Z" },
- { url = "https://files.pythonhosted.org/packages/84/68/6b6ca8e89abf0e707fd5e58222c136f5014f84157034a56d01a22c903bad/nvidia_cutlass_dsl-4.3.2-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:0bdbd5b161ec08635a4b50f32d1afaddc6a2deca7f7ca9f528d5a35dd8b62939", size = 58595663, upload-time = "2025-12-04T07:39:09.141Z" },
- { url = "https://files.pythonhosted.org/packages/f5/ab/b59fdb89391b2981a431f870e02f878030262977e2b18fcb84d9c2cdbeab/nvidia_cutlass_dsl-4.3.2-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:97e9620f3454e11a6617861e386b7c27ac27d1b32095797ce740a92a880ad4e5", size = 58722087, upload-time = "2025-12-04T07:37:33.38Z" },
- { url = "https://files.pythonhosted.org/packages/8b/c0/3aa1ae5a81efc74bab7617f7f614b425bdb8754b946b6c6548af7f125c96/nvidia_cutlass_dsl-4.3.2-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:93125f6bd2df43b437b239f0c423fd64287a890dac087f5d8f0aa922b64ffe1d", size = 58594796, upload-time = "2025-12-04T07:43:25.042Z" },
+ { url = "https://files.pythonhosted.org/packages/ba/1f/83e48a71e0b7bed6b33b01732ae53e9f2e61dc518ab273e56ec859bb05f1/nvidia_cutlass_dsl-4.3.4-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:118508bc84f2a55ec7af3affd379bb713edf837d593218329909db67b518e700", size = 58736512, upload-time = "2025-12-21T07:40:34.715Z" },
+ { url = "https://files.pythonhosted.org/packages/27/f1/21166ae0b6da766e11448d32c1e69fc60ba4023de9040f6ef9c333e7b0b5/nvidia_cutlass_dsl-4.3.4-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:3fdf0603ab7ec1bf6a499fbf72cff65e73b597d6e1359286808317c69aeb7c3d", size = 58598504, upload-time = "2025-12-21T07:39:43.124Z" },
+ { url = "https://files.pythonhosted.org/packages/43/01/3067eaad7454a3e36523b6814f09344afa0d36f71719072a6eecd6c87a40/nvidia_cutlass_dsl-4.3.4-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:c5bd21ed877da171f115123a12aae4a920035fc47eb57c807f9fba9f3df97cf4", size = 58733573, upload-time = "2025-12-21T07:41:51.364Z" },
+ { url = "https://files.pythonhosted.org/packages/86/3b/f8255a1fe6841955eea7a211bc9f30fd46bd8424ea15f361d5c09b29520a/nvidia_cutlass_dsl-4.3.4-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:671936f1df909e7de377d0cc00cb4287a3458c013d34947600423e9deb827e41", size = 58598831, upload-time = "2025-12-21T07:39:17.853Z" },
+ { url = "https://files.pythonhosted.org/packages/86/ee/53d22e2e14cb763927d85f7ec9748f6af6d27a2b7f43d52de014728da10e/nvidia_cutlass_dsl-4.3.4-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:57693d87677919572ab9eefa386b3f39e8e888bc4a9db7ab8730a97e8dbe06b4", size = 58736300, upload-time = "2025-12-21T07:41:25.723Z" },
+ { url = "https://files.pythonhosted.org/packages/66/f6/47489e07081cd4060f08bfa4166f8ff32beaecf71c06060d03bde88f3b6c/nvidia_cutlass_dsl-4.3.4-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:a48fbff859e44dd548f8f26819d97d0595acea70e3b057c91dfdb47929015c72", size = 58599014, upload-time = "2025-12-21T07:38:51.632Z" },
+ { url = "https://files.pythonhosted.org/packages/c7/2e/3aaf6121842351ec0231d5ab9d9ebe9a6e2269e9a8f7345e02f096db1ba8/nvidia_cutlass_dsl-4.3.4-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:36bde25160f461f393beba81868ef9e54d5ba2e0e7666ed3e44b6dbf788af493", size = 58735620, upload-time = "2025-12-21T07:40:59.729Z" },
+ { url = "https://files.pythonhosted.org/packages/62/90/1da2583bda001bf678066bc970963aad3986036ac15e95eb38447fa1b51e/nvidia_cutlass_dsl-4.3.4-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:be127f0f087028fa498f50a994c49f95b2c6a518e11e2567bc3d71528bf0a504", size = 58600158, upload-time = "2025-12-21T07:40:09.36Z" },
]
[[package]]
@@ -3241,21 +3267,21 @@ wheels = [
[[package]]
name = "nvidia-ml-py"
-version = "13.580.82"
+version = "13.590.44"
source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/dd/6c/4a533f2c0185027c465adb6063086bc3728301e95f483665bfa9ebafb2d3/nvidia_ml_py-13.580.82.tar.gz", hash = "sha256:0c028805dc53a0e2a6985ea801888197765ac2ef8f1c9e29a7bf0d3616a5efc7", size = 47999, upload-time = "2025-09-11T16:44:56.267Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/1b/23/3871537f204aee823c574ba25cbeb08cae779979d4d43c01adddda00bab9/nvidia_ml_py-13.590.44.tar.gz", hash = "sha256:b358c7614b0fdeea4b95f046f1c90123bfe25d148ab93bb1c00248b834703373", size = 49737, upload-time = "2025-12-08T14:41:10.872Z" }
wheels = [
- { url = "https://files.pythonhosted.org/packages/7f/96/d6d25a4c307d6645f4a9b91d620c0151c544ad38b5e371313a87d2761004/nvidia_ml_py-13.580.82-py3-none-any.whl", hash = "sha256:4361db337b0c551e2d101936dae2e9a60f957af26818e8c0c3a1f32b8db8d0a7", size = 49008, upload-time = "2025-09-11T16:44:54.915Z" },
+ { url = "https://files.pythonhosted.org/packages/e4/47/4c822bd37a008e72fd5a0eae33524ae3ac97b13f7030f63bae1728b8957e/nvidia_ml_py-13.590.44-py3-none-any.whl", hash = "sha256:18feb54eca7d0e3cdc8d1a040a771eda72d9ec3148e5443087970dbfd7377ecc", size = 50683, upload-time = "2025-12-08T14:41:09.597Z" },
]
[[package]]
name = "nvidia-modelopt"
-version = "0.39.0"
+version = "0.40.0"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "ninja" },
{ name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version < '3.11' and extra == 'extra-13-megatron-core-dev') or (extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" },
- { name = "numpy", version = "2.3.5", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version >= '3.11' and extra == 'extra-13-megatron-core-dev') or (extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" },
+ { name = "numpy", version = "2.4.0", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version >= '3.11' and extra == 'extra-13-megatron-core-dev') or (extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" },
{ name = "nvidia-ml-py" },
{ name = "packaging" },
{ name = "pulp" },
@@ -3270,7 +3296,7 @@ dependencies = [
{ name = "tqdm" },
]
wheels = [
- { url = "https://files.pythonhosted.org/packages/b0/d5/b03ad3ffa28984b629a72da678fa98f912fc45bac3b514c4a70cf2a82fe3/nvidia_modelopt-0.39.0-py3-none-any.whl", hash = "sha256:32f05317c81be1ff2ffeab749e5258b7bea8e4c6e60a09c760584f25ad03f648", size = 864981, upload-time = "2025-11-13T07:35:42.761Z" },
+ { url = "https://files.pythonhosted.org/packages/7f/4a/4b4c339637fdbd54bc98b92c87c8b22f5efee05ca9e31e40a8d49ee66187/nvidia_modelopt-0.40.0-py3-none-any.whl", hash = "sha256:0315f53aef014b902866e427038db5803e3c6787a8e1f09c3650031550885051", size = 901421, upload-time = "2025-12-12T10:35:28.506Z" },
]
[[package]]
@@ -3396,7 +3422,7 @@ source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "ml-dtypes" },
{ name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version < '3.11' and extra == 'extra-13-megatron-core-dev') or (python_full_version < '3.11' and extra == 'extra-13-megatron-core-lts') or (extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" },
- { name = "numpy", version = "2.3.5", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version >= '3.11' and extra == 'extra-13-megatron-core-dev') or (python_full_version >= '3.11' and extra == 'extra-13-megatron-core-lts') or (extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" },
+ { name = "numpy", version = "2.4.0", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version >= '3.11' and extra == 'extra-13-megatron-core-dev') or (python_full_version >= '3.11' and extra == 'extra-13-megatron-core-lts') or (extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" },
{ name = "protobuf" },
{ name = "typing-extensions" },
]
@@ -3428,36 +3454,36 @@ wheels = [
[[package]]
name = "onnx-ir"
-version = "0.1.12"
+version = "0.1.13"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "ml-dtypes" },
{ name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version < '3.11' and extra == 'extra-13-megatron-core-dev') or (python_full_version < '3.11' and extra == 'extra-13-megatron-core-lts') or (extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" },
- { name = "numpy", version = "2.3.5", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version >= '3.11' and extra == 'extra-13-megatron-core-dev') or (python_full_version >= '3.11' and extra == 'extra-13-megatron-core-lts') or (extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" },
+ { name = "numpy", version = "2.4.0", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version >= '3.11' and extra == 'extra-13-megatron-core-dev') or (python_full_version >= '3.11' and extra == 'extra-13-megatron-core-lts') or (extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" },
{ name = "onnx" },
{ name = "typing-extensions" },
]
-sdist = { url = "https://files.pythonhosted.org/packages/6c/1a/2a94112a39d01a9d1490f5ef3c205d8a17fe1ca27f307b026c40d62d8e9f/onnx_ir-0.1.12.tar.gz", hash = "sha256:742e0bff875d0547724187560b3f441833191c8aa939c05f14176f4892784deb", size = 112699, upload-time = "2025-10-28T23:43:54.129Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/a8/c2/6db31dc3132e540076f15ed0cdf4a8db7ab75557f4d6c19eda655cac666e/onnx_ir-0.1.13.tar.gz", hash = "sha256:e08f00d30579bdbff2152692a6f1bc1f0523d3321ac6348aadcd40595e56231e", size = 115872, upload-time = "2025-12-17T18:03:13.86Z" }
wheels = [
- { url = "https://files.pythonhosted.org/packages/c8/36/c4df116f5dcaa82ec7944e5d25624a3811f6603fd190660b0b079ea759fb/onnx_ir-0.1.12-py3-none-any.whl", hash = "sha256:17f86faf8a53b979430bde1bc6022c7a162b0d1534550ddb17a1d37eb993e765", size = 129277, upload-time = "2025-10-28T23:43:52.493Z" },
+ { url = "https://files.pythonhosted.org/packages/37/b6/f60fd79ff5bc617d49db1378eb7c4c315b21b786502674e4a2d48e64491a/onnx_ir-0.1.13-py3-none-any.whl", hash = "sha256:2791493d1529fdbea60c257dc7bc0933dc812e6d68f4976d8b59aa7b4c2de8cf", size = 133063, upload-time = "2025-12-17T18:03:12.268Z" },
]
[[package]]
name = "onnxscript"
-version = "0.5.6"
+version = "0.5.7"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "ml-dtypes" },
{ name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version < '3.11' and extra == 'extra-13-megatron-core-dev') or (python_full_version < '3.11' and extra == 'extra-13-megatron-core-lts') or (extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" },
- { name = "numpy", version = "2.3.5", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version >= '3.11' and extra == 'extra-13-megatron-core-dev') or (python_full_version >= '3.11' and extra == 'extra-13-megatron-core-lts') or (extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" },
+ { name = "numpy", version = "2.4.0", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version >= '3.11' and extra == 'extra-13-megatron-core-dev') or (python_full_version >= '3.11' and extra == 'extra-13-megatron-core-lts') or (extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" },
{ name = "onnx" },
{ name = "onnx-ir" },
{ name = "packaging" },
{ name = "typing-extensions" },
]
-sdist = { url = "https://files.pythonhosted.org/packages/fb/4b/eed2199327bbf12c3443d7835893e3c4c23b1c1a4aa13efe0f7fbe0a6bf9/onnxscript-0.5.6.tar.gz", hash = "sha256:cc3338b2976daffd2af0bb6ac4866a4dca76aefface1666a0d7bc65ad9850822", size = 587017, upload-time = "2025-10-31T03:50:38.656Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/2c/f8/358a7d982ea51bc1b0c264f29c08adf096c62ba9f258ba13c954b41c46f5/onnxscript-0.5.7.tar.gz", hash = "sha256:480d572451bc233ed7f742b5005cb0c899594b2fdc28e15167dab26f7fd777ad", size = 596306, upload-time = "2025-12-16T20:47:15.762Z" }
wheels = [
- { url = "https://files.pythonhosted.org/packages/fd/1e/a5462bfe28a2add00dc0abec7dd9b742ac3207b73e5c97bde9747b503971/onnxscript-0.5.6-py3-none-any.whl", hash = "sha256:b0c3355fea3eecab8ca291da8b77afddcaacd3ada5ee59294390a049ea123938", size = 683045, upload-time = "2025-10-31T03:50:41.15Z" },
+ { url = "https://files.pythonhosted.org/packages/f6/ec/1656ea93be1e50baf429c20603dce249fa3571f3a180407cee79b1afa013/onnxscript-0.5.7-py3-none-any.whl", hash = "sha256:f94a66059c56d13b44908e9b7fd9dae4b4faa6681c784f3fd4c29cfa863e454e", size = 693353, upload-time = "2025-12-16T20:47:17.897Z" },
]
[[package]]
@@ -3497,7 +3523,7 @@ version = "2.3.3"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11' or (extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" },
- { name = "numpy", version = "2.3.5", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11' or (extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" },
+ { name = "numpy", version = "2.4.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11' or (extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" },
{ name = "python-dateutil" },
{ name = "pytz" },
{ name = "tzdata" },
@@ -3570,11 +3596,11 @@ wheels = [
[[package]]
name = "pathspec"
-version = "0.12.1"
+version = "1.0.0"
source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/ca/bc/f35b8446f4531a7cb215605d100cd88b7ac6f44ab3fc94870c120ab3adbf/pathspec-0.12.1.tar.gz", hash = "sha256:a482d51503a1ab33b1c67a6c3813a26953dbdc71c31dacaef9a838c4e29f5712", size = 51043, upload-time = "2023-12-10T22:30:45Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/c2/97/39352be14d20d377a387828daf9d3f765fad1ff29bd49913d5bbf4cefe61/pathspec-1.0.0.tar.gz", hash = "sha256:9ada63a23541746b0cf7d5672a39ea77eac31dd23a80470be90df83537512131", size = 129410, upload-time = "2026-01-06T03:21:22.892Z" }
wheels = [
- { url = "https://files.pythonhosted.org/packages/cc/20/ff623b09d963f88bfde16306a54e12ee5ea43e9b597108672ff3a408aad6/pathspec-0.12.1-py3-none-any.whl", hash = "sha256:a0d503e138a4c123b27490a4f7beda6a01c6f288df0e4a8b79c7eb0dc7b4cc08", size = 31191, upload-time = "2023-12-10T22:30:43.14Z" },
+ { url = "https://files.pythonhosted.org/packages/05/bb/39e6768529454cc2b57e1e2fa0a0a18ff64397a16303270e215a3e03285f/pathspec-1.0.0-py3-none-any.whl", hash = "sha256:1373719036e64a2b9de3b8ddd9e30afb082a915619f07265ed76d9ae507800ae", size = 54316, upload-time = "2026-01-06T03:21:21.74Z" },
]
[[package]]
@@ -3588,109 +3614,109 @@ wheels = [
[[package]]
name = "pillow"
-version = "12.0.0"
-source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/5a/b0/cace85a1b0c9775a9f8f5d5423c8261c858760e2466c79b2dd184638b056/pillow-12.0.0.tar.gz", hash = "sha256:87d4f8125c9988bfbed67af47dd7a953e2fc7b0cc1e7800ec6d2080d490bb353", size = 47008828, upload-time = "2025-10-15T18:24:14.008Z" }
-wheels = [
- { url = "https://files.pythonhosted.org/packages/5d/08/26e68b6b5da219c2a2cb7b563af008b53bb8e6b6fcb3fa40715fcdb2523a/pillow-12.0.0-cp310-cp310-macosx_10_10_x86_64.whl", hash = "sha256:3adfb466bbc544b926d50fe8f4a4e6abd8c6bffd28a26177594e6e9b2b76572b", size = 5289809, upload-time = "2025-10-15T18:21:27.791Z" },
- { url = "https://files.pythonhosted.org/packages/cb/e9/4e58fb097fb74c7b4758a680aacd558810a417d1edaa7000142976ef9d2f/pillow-12.0.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:1ac11e8ea4f611c3c0147424eae514028b5e9077dd99ab91e1bd7bc33ff145e1", size = 4650606, upload-time = "2025-10-15T18:21:29.823Z" },
- { url = "https://files.pythonhosted.org/packages/4b/e0/1fa492aa9f77b3bc6d471c468e62bfea1823056bf7e5e4f1914d7ab2565e/pillow-12.0.0-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:d49e2314c373f4c2b39446fb1a45ed333c850e09d0c59ac79b72eb3b95397363", size = 6221023, upload-time = "2025-10-15T18:21:31.415Z" },
- { url = "https://files.pythonhosted.org/packages/c1/09/4de7cd03e33734ccd0c876f0251401f1314e819cbfd89a0fcb6e77927cc6/pillow-12.0.0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:c7b2a63fd6d5246349f3d3f37b14430d73ee7e8173154461785e43036ffa96ca", size = 8024937, upload-time = "2025-10-15T18:21:33.453Z" },
- { url = "https://files.pythonhosted.org/packages/2e/69/0688e7c1390666592876d9d474f5e135abb4acb39dcb583c4dc5490f1aff/pillow-12.0.0-cp310-cp310-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d64317d2587c70324b79861babb9c09f71fbb780bad212018874b2c013d8600e", size = 6334139, upload-time = "2025-10-15T18:21:35.395Z" },
- { url = "https://files.pythonhosted.org/packages/ed/1c/880921e98f525b9b44ce747ad1ea8f73fd7e992bafe3ca5e5644bf433dea/pillow-12.0.0-cp310-cp310-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d77153e14b709fd8b8af6f66a3afbb9ed6e9fc5ccf0b6b7e1ced7b036a228782", size = 7026074, upload-time = "2025-10-15T18:21:37.219Z" },
- { url = "https://files.pythonhosted.org/packages/28/03/96f718331b19b355610ef4ebdbbde3557c726513030665071fd025745671/pillow-12.0.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:32ed80ea8a90ee3e6fa08c21e2e091bba6eda8eccc83dbc34c95169507a91f10", size = 6448852, upload-time = "2025-10-15T18:21:39.168Z" },
- { url = "https://files.pythonhosted.org/packages/3a/a0/6a193b3f0cc9437b122978d2c5cbce59510ccf9a5b48825096ed7472da2f/pillow-12.0.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:c828a1ae702fc712978bda0320ba1b9893d99be0badf2647f693cc01cf0f04fa", size = 7117058, upload-time = "2025-10-15T18:21:40.997Z" },
- { url = "https://files.pythonhosted.org/packages/a7/c4/043192375eaa4463254e8e61f0e2ec9a846b983929a8d0a7122e0a6d6fff/pillow-12.0.0-cp310-cp310-win32.whl", hash = "sha256:bd87e140e45399c818fac4247880b9ce719e4783d767e030a883a970be632275", size = 6295431, upload-time = "2025-10-15T18:21:42.518Z" },
- { url = "https://files.pythonhosted.org/packages/92/c6/c2f2fc7e56301c21827e689bb8b0b465f1b52878b57471a070678c0c33cd/pillow-12.0.0-cp310-cp310-win_amd64.whl", hash = "sha256:455247ac8a4cfb7b9bc45b7e432d10421aea9fc2e74d285ba4072688a74c2e9d", size = 7000412, upload-time = "2025-10-15T18:21:44.404Z" },
- { url = "https://files.pythonhosted.org/packages/b2/d2/5f675067ba82da7a1c238a73b32e3fd78d67f9d9f80fbadd33a40b9c0481/pillow-12.0.0-cp310-cp310-win_arm64.whl", hash = "sha256:6ace95230bfb7cd79ef66caa064bbe2f2a1e63d93471c3a2e1f1348d9f22d6b7", size = 2435903, upload-time = "2025-10-15T18:21:46.29Z" },
- { url = "https://files.pythonhosted.org/packages/0e/5a/a2f6773b64edb921a756eb0729068acad9fc5208a53f4a349396e9436721/pillow-12.0.0-cp311-cp311-macosx_10_10_x86_64.whl", hash = "sha256:0fd00cac9c03256c8b2ff58f162ebcd2587ad3e1f2e397eab718c47e24d231cc", size = 5289798, upload-time = "2025-10-15T18:21:47.763Z" },
- { url = "https://files.pythonhosted.org/packages/2e/05/069b1f8a2e4b5a37493da6c5868531c3f77b85e716ad7a590ef87d58730d/pillow-12.0.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:a3475b96f5908b3b16c47533daaa87380c491357d197564e0ba34ae75c0f3257", size = 4650589, upload-time = "2025-10-15T18:21:49.515Z" },
- { url = "https://files.pythonhosted.org/packages/61/e3/2c820d6e9a36432503ead175ae294f96861b07600a7156154a086ba7111a/pillow-12.0.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:110486b79f2d112cf6add83b28b627e369219388f64ef2f960fef9ebaf54c642", size = 6230472, upload-time = "2025-10-15T18:21:51.052Z" },
- { url = "https://files.pythonhosted.org/packages/4f/89/63427f51c64209c5e23d4d52071c8d0f21024d3a8a487737caaf614a5795/pillow-12.0.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:5269cc1caeedb67e6f7269a42014f381f45e2e7cd42d834ede3c703a1d915fe3", size = 8033887, upload-time = "2025-10-15T18:21:52.604Z" },
- { url = "https://files.pythonhosted.org/packages/f6/1b/c9711318d4901093c15840f268ad649459cd81984c9ec9887756cca049a5/pillow-12.0.0-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:aa5129de4e174daccbc59d0a3b6d20eaf24417d59851c07ebb37aeb02947987c", size = 6343964, upload-time = "2025-10-15T18:21:54.619Z" },
- { url = "https://files.pythonhosted.org/packages/41/1e/db9470f2d030b4995083044cd8738cdd1bf773106819f6d8ba12597d5352/pillow-12.0.0-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bee2a6db3a7242ea309aa7ee8e2780726fed67ff4e5b40169f2c940e7eb09227", size = 7034756, upload-time = "2025-10-15T18:21:56.151Z" },
- { url = "https://files.pythonhosted.org/packages/cc/b0/6177a8bdd5ee4ed87cba2de5a3cc1db55ffbbec6176784ce5bb75aa96798/pillow-12.0.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:90387104ee8400a7b4598253b4c406f8958f59fcf983a6cea2b50d59f7d63d0b", size = 6458075, upload-time = "2025-10-15T18:21:57.759Z" },
- { url = "https://files.pythonhosted.org/packages/bc/5e/61537aa6fa977922c6a03253a0e727e6e4a72381a80d63ad8eec350684f2/pillow-12.0.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:bc91a56697869546d1b8f0a3ff35224557ae7f881050e99f615e0119bf934b4e", size = 7125955, upload-time = "2025-10-15T18:21:59.372Z" },
- { url = "https://files.pythonhosted.org/packages/1f/3d/d5033539344ee3cbd9a4d69e12e63ca3a44a739eb2d4c8da350a3d38edd7/pillow-12.0.0-cp311-cp311-win32.whl", hash = "sha256:27f95b12453d165099c84f8a8bfdfd46b9e4bda9e0e4b65f0635430027f55739", size = 6298440, upload-time = "2025-10-15T18:22:00.982Z" },
- { url = "https://files.pythonhosted.org/packages/4d/42/aaca386de5cc8bd8a0254516957c1f265e3521c91515b16e286c662854c4/pillow-12.0.0-cp311-cp311-win_amd64.whl", hash = "sha256:b583dc9070312190192631373c6c8ed277254aa6e6084b74bdd0a6d3b221608e", size = 6999256, upload-time = "2025-10-15T18:22:02.617Z" },
- { url = "https://files.pythonhosted.org/packages/ba/f1/9197c9c2d5708b785f631a6dfbfa8eb3fb9672837cb92ae9af812c13b4ed/pillow-12.0.0-cp311-cp311-win_arm64.whl", hash = "sha256:759de84a33be3b178a64c8ba28ad5c135900359e85fb662bc6e403ad4407791d", size = 2436025, upload-time = "2025-10-15T18:22:04.598Z" },
- { url = "https://files.pythonhosted.org/packages/2c/90/4fcce2c22caf044e660a198d740e7fbc14395619e3cb1abad12192c0826c/pillow-12.0.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:53561a4ddc36facb432fae7a9d8afbfaf94795414f5cdc5fc52f28c1dca90371", size = 5249377, upload-time = "2025-10-15T18:22:05.993Z" },
- { url = "https://files.pythonhosted.org/packages/fd/e0/ed960067543d080691d47d6938ebccbf3976a931c9567ab2fbfab983a5dd/pillow-12.0.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:71db6b4c1653045dacc1585c1b0d184004f0d7e694c7b34ac165ca70c0838082", size = 4650343, upload-time = "2025-10-15T18:22:07.718Z" },
- { url = "https://files.pythonhosted.org/packages/e7/a1/f81fdeddcb99c044bf7d6faa47e12850f13cee0849537a7d27eeab5534d4/pillow-12.0.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:2fa5f0b6716fc88f11380b88b31fe591a06c6315e955c096c35715788b339e3f", size = 6232981, upload-time = "2025-10-15T18:22:09.287Z" },
- { url = "https://files.pythonhosted.org/packages/88/e1/9098d3ce341a8750b55b0e00c03f1630d6178f38ac191c81c97a3b047b44/pillow-12.0.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:82240051c6ca513c616f7f9da06e871f61bfd7805f566275841af15015b8f98d", size = 8041399, upload-time = "2025-10-15T18:22:10.872Z" },
- { url = "https://files.pythonhosted.org/packages/a7/62/a22e8d3b602ae8cc01446d0c57a54e982737f44b6f2e1e019a925143771d/pillow-12.0.0-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:55f818bd74fe2f11d4d7cbc65880a843c4075e0ac7226bc1a23261dbea531953", size = 6347740, upload-time = "2025-10-15T18:22:12.769Z" },
- { url = "https://files.pythonhosted.org/packages/4f/87/424511bdcd02c8d7acf9f65caa09f291a519b16bd83c3fb3374b3d4ae951/pillow-12.0.0-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b87843e225e74576437fd5b6a4c2205d422754f84a06942cfaf1dc32243e45a8", size = 7040201, upload-time = "2025-10-15T18:22:14.813Z" },
- { url = "https://files.pythonhosted.org/packages/dc/4d/435c8ac688c54d11755aedfdd9f29c9eeddf68d150fe42d1d3dbd2365149/pillow-12.0.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:c607c90ba67533e1b2355b821fef6764d1dd2cbe26b8c1005ae84f7aea25ff79", size = 6462334, upload-time = "2025-10-15T18:22:16.375Z" },
- { url = "https://files.pythonhosted.org/packages/2b/f2/ad34167a8059a59b8ad10bc5c72d4d9b35acc6b7c0877af8ac885b5f2044/pillow-12.0.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:21f241bdd5080a15bc86d3466a9f6074a9c2c2b314100dd896ac81ee6db2f1ba", size = 7134162, upload-time = "2025-10-15T18:22:17.996Z" },
- { url = "https://files.pythonhosted.org/packages/0c/b1/a7391df6adacf0a5c2cf6ac1cf1fcc1369e7d439d28f637a847f8803beb3/pillow-12.0.0-cp312-cp312-win32.whl", hash = "sha256:dd333073e0cacdc3089525c7df7d39b211bcdf31fc2824e49d01c6b6187b07d0", size = 6298769, upload-time = "2025-10-15T18:22:19.923Z" },
- { url = "https://files.pythonhosted.org/packages/a2/0b/d87733741526541c909bbf159e338dcace4f982daac6e5a8d6be225ca32d/pillow-12.0.0-cp312-cp312-win_amd64.whl", hash = "sha256:9fe611163f6303d1619bbcb653540a4d60f9e55e622d60a3108be0d5b441017a", size = 7001107, upload-time = "2025-10-15T18:22:21.644Z" },
- { url = "https://files.pythonhosted.org/packages/bc/96/aaa61ce33cc98421fb6088af2a03be4157b1e7e0e87087c888e2370a7f45/pillow-12.0.0-cp312-cp312-win_arm64.whl", hash = "sha256:7dfb439562f234f7d57b1ac6bc8fe7f838a4bd49c79230e0f6a1da93e82f1fad", size = 2436012, upload-time = "2025-10-15T18:22:23.621Z" },
- { url = "https://files.pythonhosted.org/packages/62/f2/de993bb2d21b33a98d031ecf6a978e4b61da207bef02f7b43093774c480d/pillow-12.0.0-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:0869154a2d0546545cde61d1789a6524319fc1897d9ee31218eae7a60ccc5643", size = 4045493, upload-time = "2025-10-15T18:22:25.758Z" },
- { url = "https://files.pythonhosted.org/packages/0e/b6/bc8d0c4c9f6f111a783d045310945deb769b806d7574764234ffd50bc5ea/pillow-12.0.0-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:a7921c5a6d31b3d756ec980f2f47c0cfdbce0fc48c22a39347a895f41f4a6ea4", size = 4120461, upload-time = "2025-10-15T18:22:27.286Z" },
- { url = "https://files.pythonhosted.org/packages/5d/57/d60d343709366a353dc56adb4ee1e7d8a2cc34e3fbc22905f4167cfec119/pillow-12.0.0-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:1ee80a59f6ce048ae13cda1abf7fbd2a34ab9ee7d401c46be3ca685d1999a399", size = 3576912, upload-time = "2025-10-15T18:22:28.751Z" },
- { url = "https://files.pythonhosted.org/packages/a4/a4/a0a31467e3f83b94d37568294b01d22b43ae3c5d85f2811769b9c66389dd/pillow-12.0.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:c50f36a62a22d350c96e49ad02d0da41dbd17ddc2e29750dbdba4323f85eb4a5", size = 5249132, upload-time = "2025-10-15T18:22:30.641Z" },
- { url = "https://files.pythonhosted.org/packages/83/06/48eab21dd561de2914242711434c0c0eb992ed08ff3f6107a5f44527f5e9/pillow-12.0.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:5193fde9a5f23c331ea26d0cf171fbf67e3f247585f50c08b3e205c7aeb4589b", size = 4650099, upload-time = "2025-10-15T18:22:32.73Z" },
- { url = "https://files.pythonhosted.org/packages/fc/bd/69ed99fd46a8dba7c1887156d3572fe4484e3f031405fcc5a92e31c04035/pillow-12.0.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:bde737cff1a975b70652b62d626f7785e0480918dece11e8fef3c0cf057351c3", size = 6230808, upload-time = "2025-10-15T18:22:34.337Z" },
- { url = "https://files.pythonhosted.org/packages/ea/94/8fad659bcdbf86ed70099cb60ae40be6acca434bbc8c4c0d4ef356d7e0de/pillow-12.0.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:a6597ff2b61d121172f5844b53f21467f7082f5fb385a9a29c01414463f93b07", size = 8037804, upload-time = "2025-10-15T18:22:36.402Z" },
- { url = "https://files.pythonhosted.org/packages/20/39/c685d05c06deecfd4e2d1950e9a908aa2ca8bc4e6c3b12d93b9cafbd7837/pillow-12.0.0-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0b817e7035ea7f6b942c13aa03bb554fc44fea70838ea21f8eb31c638326584e", size = 6345553, upload-time = "2025-10-15T18:22:38.066Z" },
- { url = "https://files.pythonhosted.org/packages/38/57/755dbd06530a27a5ed74f8cb0a7a44a21722ebf318edbe67ddbd7fb28f88/pillow-12.0.0-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f4f1231b7dec408e8670264ce63e9c71409d9583dd21d32c163e25213ee2a344", size = 7037729, upload-time = "2025-10-15T18:22:39.769Z" },
- { url = "https://files.pythonhosted.org/packages/ca/b6/7e94f4c41d238615674d06ed677c14883103dce1c52e4af16f000338cfd7/pillow-12.0.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:6e51b71417049ad6ab14c49608b4a24d8fb3fe605e5dfabfe523b58064dc3d27", size = 6459789, upload-time = "2025-10-15T18:22:41.437Z" },
- { url = "https://files.pythonhosted.org/packages/9c/14/4448bb0b5e0f22dd865290536d20ec8a23b64e2d04280b89139f09a36bb6/pillow-12.0.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:d120c38a42c234dc9a8c5de7ceaaf899cf33561956acb4941653f8bdc657aa79", size = 7130917, upload-time = "2025-10-15T18:22:43.152Z" },
- { url = "https://files.pythonhosted.org/packages/dd/ca/16c6926cc1c015845745d5c16c9358e24282f1e588237a4c36d2b30f182f/pillow-12.0.0-cp313-cp313-win32.whl", hash = "sha256:4cc6b3b2efff105c6a1656cfe59da4fdde2cda9af1c5e0b58529b24525d0a098", size = 6302391, upload-time = "2025-10-15T18:22:44.753Z" },
- { url = "https://files.pythonhosted.org/packages/6d/2a/dd43dcfd6dae9b6a49ee28a8eedb98c7d5ff2de94a5d834565164667b97b/pillow-12.0.0-cp313-cp313-win_amd64.whl", hash = "sha256:4cf7fed4b4580601c4345ceb5d4cbf5a980d030fd5ad07c4d2ec589f95f09905", size = 7007477, upload-time = "2025-10-15T18:22:46.838Z" },
- { url = "https://files.pythonhosted.org/packages/77/f0/72ea067f4b5ae5ead653053212af05ce3705807906ba3f3e8f58ddf617e6/pillow-12.0.0-cp313-cp313-win_arm64.whl", hash = "sha256:9f0b04c6b8584c2c193babcccc908b38ed29524b29dd464bc8801bf10d746a3a", size = 2435918, upload-time = "2025-10-15T18:22:48.399Z" },
- { url = "https://files.pythonhosted.org/packages/f5/5e/9046b423735c21f0487ea6cb5b10f89ea8f8dfbe32576fe052b5ba9d4e5b/pillow-12.0.0-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:7fa22993bac7b77b78cae22bad1e2a987ddf0d9015c63358032f84a53f23cdc3", size = 5251406, upload-time = "2025-10-15T18:22:49.905Z" },
- { url = "https://files.pythonhosted.org/packages/12/66/982ceebcdb13c97270ef7a56c3969635b4ee7cd45227fa707c94719229c5/pillow-12.0.0-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:f135c702ac42262573fe9714dfe99c944b4ba307af5eb507abef1667e2cbbced", size = 4653218, upload-time = "2025-10-15T18:22:51.587Z" },
- { url = "https://files.pythonhosted.org/packages/16/b3/81e625524688c31859450119bf12674619429cab3119eec0e30a7a1029cb/pillow-12.0.0-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:c85de1136429c524e55cfa4e033b4a7940ac5c8ee4d9401cc2d1bf48154bbc7b", size = 6266564, upload-time = "2025-10-15T18:22:53.215Z" },
- { url = "https://files.pythonhosted.org/packages/98/59/dfb38f2a41240d2408096e1a76c671d0a105a4a8471b1871c6902719450c/pillow-12.0.0-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:38df9b4bfd3db902c9c2bd369bcacaf9d935b2fff73709429d95cc41554f7b3d", size = 8069260, upload-time = "2025-10-15T18:22:54.933Z" },
- { url = "https://files.pythonhosted.org/packages/dc/3d/378dbea5cd1874b94c312425ca77b0f47776c78e0df2df751b820c8c1d6c/pillow-12.0.0-cp313-cp313t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:7d87ef5795da03d742bf49439f9ca4d027cde49c82c5371ba52464aee266699a", size = 6379248, upload-time = "2025-10-15T18:22:56.605Z" },
- { url = "https://files.pythonhosted.org/packages/84/b0/d525ef47d71590f1621510327acec75ae58c721dc071b17d8d652ca494d8/pillow-12.0.0-cp313-cp313t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:aff9e4d82d082ff9513bdd6acd4f5bd359f5b2c870907d2b0a9c5e10d40c88fe", size = 7066043, upload-time = "2025-10-15T18:22:58.53Z" },
- { url = "https://files.pythonhosted.org/packages/61/2c/aced60e9cf9d0cde341d54bf7932c9ffc33ddb4a1595798b3a5150c7ec4e/pillow-12.0.0-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:8d8ca2b210ada074d57fcee40c30446c9562e542fc46aedc19baf758a93532ee", size = 6490915, upload-time = "2025-10-15T18:23:00.582Z" },
- { url = "https://files.pythonhosted.org/packages/ef/26/69dcb9b91f4e59f8f34b2332a4a0a951b44f547c4ed39d3e4dcfcff48f89/pillow-12.0.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:99a7f72fb6249302aa62245680754862a44179b545ded638cf1fef59befb57ef", size = 7157998, upload-time = "2025-10-15T18:23:02.627Z" },
- { url = "https://files.pythonhosted.org/packages/61/2b/726235842220ca95fa441ddf55dd2382b52ab5b8d9c0596fe6b3f23dafe8/pillow-12.0.0-cp313-cp313t-win32.whl", hash = "sha256:4078242472387600b2ce8d93ade8899c12bf33fa89e55ec89fe126e9d6d5d9e9", size = 6306201, upload-time = "2025-10-15T18:23:04.709Z" },
- { url = "https://files.pythonhosted.org/packages/c0/3d/2afaf4e840b2df71344ababf2f8edd75a705ce500e5dc1e7227808312ae1/pillow-12.0.0-cp313-cp313t-win_amd64.whl", hash = "sha256:2c54c1a783d6d60595d3514f0efe9b37c8808746a66920315bfd34a938d7994b", size = 7013165, upload-time = "2025-10-15T18:23:06.46Z" },
- { url = "https://files.pythonhosted.org/packages/6f/75/3fa09aa5cf6ed04bee3fa575798ddf1ce0bace8edb47249c798077a81f7f/pillow-12.0.0-cp313-cp313t-win_arm64.whl", hash = "sha256:26d9f7d2b604cd23aba3e9faf795787456ac25634d82cd060556998e39c6fa47", size = 2437834, upload-time = "2025-10-15T18:23:08.194Z" },
- { url = "https://files.pythonhosted.org/packages/54/2a/9a8c6ba2c2c07b71bec92cf63e03370ca5e5f5c5b119b742bcc0cde3f9c5/pillow-12.0.0-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:beeae3f27f62308f1ddbcfb0690bf44b10732f2ef43758f169d5e9303165d3f9", size = 4045531, upload-time = "2025-10-15T18:23:10.121Z" },
- { url = "https://files.pythonhosted.org/packages/84/54/836fdbf1bfb3d66a59f0189ff0b9f5f666cee09c6188309300df04ad71fa/pillow-12.0.0-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:d4827615da15cd59784ce39d3388275ec093ae3ee8d7f0c089b76fa87af756c2", size = 4120554, upload-time = "2025-10-15T18:23:12.14Z" },
- { url = "https://files.pythonhosted.org/packages/0d/cd/16aec9f0da4793e98e6b54778a5fbce4f375c6646fe662e80600b8797379/pillow-12.0.0-cp314-cp314-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:3e42edad50b6909089750e65c91aa09aaf1e0a71310d383f11321b27c224ed8a", size = 3576812, upload-time = "2025-10-15T18:23:13.962Z" },
- { url = "https://files.pythonhosted.org/packages/f6/b7/13957fda356dc46339298b351cae0d327704986337c3c69bb54628c88155/pillow-12.0.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:e5d8efac84c9afcb40914ab49ba063d94f5dbdf5066db4482c66a992f47a3a3b", size = 5252689, upload-time = "2025-10-15T18:23:15.562Z" },
- { url = "https://files.pythonhosted.org/packages/fc/f5/eae31a306341d8f331f43edb2e9122c7661b975433de5e447939ae61c5da/pillow-12.0.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:266cd5f2b63ff316d5a1bba46268e603c9caf5606d44f38c2873c380950576ad", size = 4650186, upload-time = "2025-10-15T18:23:17.379Z" },
- { url = "https://files.pythonhosted.org/packages/86/62/2a88339aa40c4c77e79108facbd307d6091e2c0eb5b8d3cf4977cfca2fe6/pillow-12.0.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:58eea5ebe51504057dd95c5b77d21700b77615ab0243d8152793dc00eb4faf01", size = 6230308, upload-time = "2025-10-15T18:23:18.971Z" },
- { url = "https://files.pythonhosted.org/packages/c7/33/5425a8992bcb32d1cb9fa3dd39a89e613d09a22f2c8083b7bf43c455f760/pillow-12.0.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:f13711b1a5ba512d647a0e4ba79280d3a9a045aaf7e0cc6fbe96b91d4cdf6b0c", size = 8039222, upload-time = "2025-10-15T18:23:20.909Z" },
- { url = "https://files.pythonhosted.org/packages/d8/61/3f5d3b35c5728f37953d3eec5b5f3e77111949523bd2dd7f31a851e50690/pillow-12.0.0-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6846bd2d116ff42cba6b646edf5bf61d37e5cbd256425fa089fee4ff5c07a99e", size = 6346657, upload-time = "2025-10-15T18:23:23.077Z" },
- { url = "https://files.pythonhosted.org/packages/3a/be/ee90a3d79271227e0f0a33c453531efd6ed14b2e708596ba5dd9be948da3/pillow-12.0.0-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c98fa880d695de164b4135a52fd2e9cd7b7c90a9d8ac5e9e443a24a95ef9248e", size = 7038482, upload-time = "2025-10-15T18:23:25.005Z" },
- { url = "https://files.pythonhosted.org/packages/44/34/a16b6a4d1ad727de390e9bd9f19f5f669e079e5826ec0f329010ddea492f/pillow-12.0.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:fa3ed2a29a9e9d2d488b4da81dcb54720ac3104a20bf0bd273f1e4648aff5af9", size = 6461416, upload-time = "2025-10-15T18:23:27.009Z" },
- { url = "https://files.pythonhosted.org/packages/b6/39/1aa5850d2ade7d7ba9f54e4e4c17077244ff7a2d9e25998c38a29749eb3f/pillow-12.0.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:d034140032870024e6b9892c692fe2968493790dd57208b2c37e3fb35f6df3ab", size = 7131584, upload-time = "2025-10-15T18:23:29.752Z" },
- { url = "https://files.pythonhosted.org/packages/bf/db/4fae862f8fad0167073a7733973bfa955f47e2cac3dc3e3e6257d10fab4a/pillow-12.0.0-cp314-cp314-win32.whl", hash = "sha256:1b1b133e6e16105f524a8dec491e0586d072948ce15c9b914e41cdadd209052b", size = 6400621, upload-time = "2025-10-15T18:23:32.06Z" },
- { url = "https://files.pythonhosted.org/packages/2b/24/b350c31543fb0107ab2599464d7e28e6f856027aadda995022e695313d94/pillow-12.0.0-cp314-cp314-win_amd64.whl", hash = "sha256:8dc232e39d409036af549c86f24aed8273a40ffa459981146829a324e0848b4b", size = 7142916, upload-time = "2025-10-15T18:23:34.71Z" },
- { url = "https://files.pythonhosted.org/packages/0f/9b/0ba5a6fd9351793996ef7487c4fdbde8d3f5f75dbedc093bb598648fddf0/pillow-12.0.0-cp314-cp314-win_arm64.whl", hash = "sha256:d52610d51e265a51518692045e372a4c363056130d922a7351429ac9f27e70b0", size = 2523836, upload-time = "2025-10-15T18:23:36.967Z" },
- { url = "https://files.pythonhosted.org/packages/f5/7a/ceee0840aebc579af529b523d530840338ecf63992395842e54edc805987/pillow-12.0.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:1979f4566bb96c1e50a62d9831e2ea2d1211761e5662afc545fa766f996632f6", size = 5255092, upload-time = "2025-10-15T18:23:38.573Z" },
- { url = "https://files.pythonhosted.org/packages/44/76/20776057b4bfd1aef4eeca992ebde0f53a4dce874f3ae693d0ec90a4f79b/pillow-12.0.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:b2e4b27a6e15b04832fe9bf292b94b5ca156016bbc1ea9c2c20098a0320d6cf6", size = 4653158, upload-time = "2025-10-15T18:23:40.238Z" },
- { url = "https://files.pythonhosted.org/packages/82/3f/d9ff92ace07be8836b4e7e87e6a4c7a8318d47c2f1463ffcf121fc57d9cb/pillow-12.0.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:fb3096c30df99fd01c7bf8e544f392103d0795b9f98ba71a8054bcbf56b255f1", size = 6267882, upload-time = "2025-10-15T18:23:42.434Z" },
- { url = "https://files.pythonhosted.org/packages/9f/7a/4f7ff87f00d3ad33ba21af78bfcd2f032107710baf8280e3722ceec28cda/pillow-12.0.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:7438839e9e053ef79f7112c881cef684013855016f928b168b81ed5835f3e75e", size = 8071001, upload-time = "2025-10-15T18:23:44.29Z" },
- { url = "https://files.pythonhosted.org/packages/75/87/fcea108944a52dad8cca0715ae6247e271eb80459364a98518f1e4f480c1/pillow-12.0.0-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5d5c411a8eaa2299322b647cd932586b1427367fd3184ffbb8f7a219ea2041ca", size = 6380146, upload-time = "2025-10-15T18:23:46.065Z" },
- { url = "https://files.pythonhosted.org/packages/91/52/0d31b5e571ef5fd111d2978b84603fce26aba1b6092f28e941cb46570745/pillow-12.0.0-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d7e091d464ac59d2c7ad8e7e08105eaf9dafbc3883fd7265ffccc2baad6ac925", size = 7067344, upload-time = "2025-10-15T18:23:47.898Z" },
- { url = "https://files.pythonhosted.org/packages/7b/f4/2dd3d721f875f928d48e83bb30a434dee75a2531bca839bb996bb0aa5a91/pillow-12.0.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:792a2c0be4dcc18af9d4a2dfd8a11a17d5e25274a1062b0ec1c2d79c76f3e7f8", size = 6491864, upload-time = "2025-10-15T18:23:49.607Z" },
- { url = "https://files.pythonhosted.org/packages/30/4b/667dfcf3d61fc309ba5a15b141845cece5915e39b99c1ceab0f34bf1d124/pillow-12.0.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:afbefa430092f71a9593a99ab6a4e7538bc9eabbf7bf94f91510d3503943edc4", size = 7158911, upload-time = "2025-10-15T18:23:51.351Z" },
- { url = "https://files.pythonhosted.org/packages/a2/2f/16cabcc6426c32218ace36bf0d55955e813f2958afddbf1d391849fee9d1/pillow-12.0.0-cp314-cp314t-win32.whl", hash = "sha256:3830c769decf88f1289680a59d4f4c46c72573446352e2befec9a8512104fa52", size = 6408045, upload-time = "2025-10-15T18:23:53.177Z" },
- { url = "https://files.pythonhosted.org/packages/35/73/e29aa0c9c666cf787628d3f0dcf379f4791fba79f4936d02f8b37165bdf8/pillow-12.0.0-cp314-cp314t-win_amd64.whl", hash = "sha256:905b0365b210c73afb0ebe9101a32572152dfd1c144c7e28968a331b9217b94a", size = 7148282, upload-time = "2025-10-15T18:23:55.316Z" },
- { url = "https://files.pythonhosted.org/packages/c1/70/6b41bdcddf541b437bbb9f47f94d2db5d9ddef6c37ccab8c9107743748a4/pillow-12.0.0-cp314-cp314t-win_arm64.whl", hash = "sha256:99353a06902c2e43b43e8ff74ee65a7d90307d82370604746738a1e0661ccca7", size = 2525630, upload-time = "2025-10-15T18:23:57.149Z" },
- { url = "https://files.pythonhosted.org/packages/1d/b3/582327e6c9f86d037b63beebe981425d6811104cb443e8193824ef1a2f27/pillow-12.0.0-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:b22bd8c974942477156be55a768f7aa37c46904c175be4e158b6a86e3a6b7ca8", size = 5215068, upload-time = "2025-10-15T18:23:59.594Z" },
- { url = "https://files.pythonhosted.org/packages/fd/d6/67748211d119f3b6540baf90f92fae73ae51d5217b171b0e8b5f7e5d558f/pillow-12.0.0-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:805ebf596939e48dbb2e4922a1d3852cfc25c38160751ce02da93058b48d252a", size = 4614994, upload-time = "2025-10-15T18:24:01.669Z" },
- { url = "https://files.pythonhosted.org/packages/2d/e1/f8281e5d844c41872b273b9f2c34a4bf64ca08905668c8ae730eedc7c9fa/pillow-12.0.0-pp311-pypy311_pp73-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:cae81479f77420d217def5f54b5b9d279804d17e982e0f2fa19b1d1e14ab5197", size = 5246639, upload-time = "2025-10-15T18:24:03.403Z" },
- { url = "https://files.pythonhosted.org/packages/94/5a/0d8ab8ffe8a102ff5df60d0de5af309015163bf710c7bb3e8311dd3b3ad0/pillow-12.0.0-pp311-pypy311_pp73-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:aeaefa96c768fc66818730b952a862235d68825c178f1b3ffd4efd7ad2edcb7c", size = 6986839, upload-time = "2025-10-15T18:24:05.344Z" },
- { url = "https://files.pythonhosted.org/packages/20/2e/3434380e8110b76cd9eb00a363c484b050f949b4bbe84ba770bb8508a02c/pillow-12.0.0-pp311-pypy311_pp73-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:09f2d0abef9e4e2f349305a4f8cc784a8a6c2f58a8c4892eea13b10a943bd26e", size = 5313505, upload-time = "2025-10-15T18:24:07.137Z" },
- { url = "https://files.pythonhosted.org/packages/57/ca/5a9d38900d9d74785141d6580950fe705de68af735ff6e727cb911b64740/pillow-12.0.0-pp311-pypy311_pp73-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bdee52571a343d721fb2eb3b090a82d959ff37fc631e3f70422e0c2e029f3e76", size = 5963654, upload-time = "2025-10-15T18:24:09.579Z" },
- { url = "https://files.pythonhosted.org/packages/95/7e/f896623c3c635a90537ac093c6a618ebe1a90d87206e42309cb5d98a1b9e/pillow-12.0.0-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:b290fd8aa38422444d4b50d579de197557f182ef1068b75f5aa8558638b8d0a5", size = 6997850, upload-time = "2025-10-15T18:24:11.495Z" },
+version = "12.1.0"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/d0/02/d52c733a2452ef1ffcc123b68e6606d07276b0e358db70eabad7e40042b7/pillow-12.1.0.tar.gz", hash = "sha256:5c5ae0a06e9ea030ab786b0251b32c7e4ce10e58d983c0d5c56029455180b5b9", size = 46977283, upload-time = "2026-01-02T09:13:29.892Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/fe/41/f73d92b6b883a579e79600d391f2e21cb0df767b2714ecbd2952315dfeef/pillow-12.1.0-cp310-cp310-macosx_10_10_x86_64.whl", hash = "sha256:fb125d860738a09d363a88daa0f59c4533529a90e564785e20fe875b200b6dbd", size = 5304089, upload-time = "2026-01-02T09:10:24.953Z" },
+ { url = "https://files.pythonhosted.org/packages/94/55/7aca2891560188656e4a91ed9adba305e914a4496800da6b5c0a15f09edf/pillow-12.1.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:cad302dc10fac357d3467a74a9561c90609768a6f73a1923b0fd851b6486f8b0", size = 4657815, upload-time = "2026-01-02T09:10:27.063Z" },
+ { url = "https://files.pythonhosted.org/packages/e9/d2/b28221abaa7b4c40b7dba948f0f6a708bd7342c4d47ce342f0ea39643974/pillow-12.1.0-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:a40905599d8079e09f25027423aed94f2823adaf2868940de991e53a449e14a8", size = 6222593, upload-time = "2026-01-02T09:10:29.115Z" },
+ { url = "https://files.pythonhosted.org/packages/71/b8/7a61fb234df6a9b0b479f69e66901209d89ff72a435b49933f9122f94cac/pillow-12.1.0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:92a7fe4225365c5e3a8e598982269c6d6698d3e783b3b1ae979e7819f9cd55c1", size = 8027579, upload-time = "2026-01-02T09:10:31.182Z" },
+ { url = "https://files.pythonhosted.org/packages/ea/51/55c751a57cc524a15a0e3db20e5cde517582359508d62305a627e77fd295/pillow-12.1.0-cp310-cp310-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f10c98f49227ed8383d28174ee95155a675c4ed7f85e2e573b04414f7e371bda", size = 6335760, upload-time = "2026-01-02T09:10:33.02Z" },
+ { url = "https://files.pythonhosted.org/packages/dc/7c/60e3e6f5e5891a1a06b4c910f742ac862377a6fe842f7184df4a274ce7bf/pillow-12.1.0-cp310-cp310-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8637e29d13f478bc4f153d8daa9ffb16455f0a6cb287da1b432fdad2bfbd66c7", size = 7027127, upload-time = "2026-01-02T09:10:35.009Z" },
+ { url = "https://files.pythonhosted.org/packages/06/37/49d47266ba50b00c27ba63a7c898f1bb41a29627ced8c09e25f19ebec0ff/pillow-12.1.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:21e686a21078b0f9cb8c8a961d99e6a4ddb88e0fc5ea6e130172ddddc2e5221a", size = 6449896, upload-time = "2026-01-02T09:10:36.793Z" },
+ { url = "https://files.pythonhosted.org/packages/f9/e5/67fd87d2913902462cd9b79c6211c25bfe95fcf5783d06e1367d6d9a741f/pillow-12.1.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:2415373395a831f53933c23ce051021e79c8cd7979822d8cc478547a3f4da8ef", size = 7151345, upload-time = "2026-01-02T09:10:39.064Z" },
+ { url = "https://files.pythonhosted.org/packages/bd/15/f8c7abf82af68b29f50d77c227e7a1f87ce02fdc66ded9bf603bc3b41180/pillow-12.1.0-cp310-cp310-win32.whl", hash = "sha256:e75d3dba8fc1ddfec0cd752108f93b83b4f8d6ab40e524a95d35f016b9683b09", size = 6325568, upload-time = "2026-01-02T09:10:41.035Z" },
+ { url = "https://files.pythonhosted.org/packages/d4/24/7d1c0e160b6b5ac2605ef7d8be537e28753c0db5363d035948073f5513d7/pillow-12.1.0-cp310-cp310-win_amd64.whl", hash = "sha256:64efdf00c09e31efd754448a383ea241f55a994fd079866b92d2bbff598aad91", size = 7032367, upload-time = "2026-01-02T09:10:43.09Z" },
+ { url = "https://files.pythonhosted.org/packages/f4/03/41c038f0d7a06099254c60f618d0ec7be11e79620fc23b8e85e5b31d9a44/pillow-12.1.0-cp310-cp310-win_arm64.whl", hash = "sha256:f188028b5af6b8fb2e9a76ac0f841a575bd1bd396e46ef0840d9b88a48fdbcea", size = 2452345, upload-time = "2026-01-02T09:10:44.795Z" },
+ { url = "https://files.pythonhosted.org/packages/43/c4/bf8328039de6cc22182c3ef007a2abfbbdab153661c0a9aa78af8d706391/pillow-12.1.0-cp311-cp311-macosx_10_10_x86_64.whl", hash = "sha256:a83e0850cb8f5ac975291ebfc4170ba481f41a28065277f7f735c202cd8e0af3", size = 5304057, upload-time = "2026-01-02T09:10:46.627Z" },
+ { url = "https://files.pythonhosted.org/packages/43/06/7264c0597e676104cc22ca73ee48f752767cd4b1fe084662620b17e10120/pillow-12.1.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:b6e53e82ec2db0717eabb276aa56cf4e500c9a7cec2c2e189b55c24f65a3e8c0", size = 4657811, upload-time = "2026-01-02T09:10:49.548Z" },
+ { url = "https://files.pythonhosted.org/packages/72/64/f9189e44474610daf83da31145fa56710b627b5c4c0b9c235e34058f6b31/pillow-12.1.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:40a8e3b9e8773876d6e30daed22f016509e3987bab61b3b7fe309d7019a87451", size = 6232243, upload-time = "2026-01-02T09:10:51.62Z" },
+ { url = "https://files.pythonhosted.org/packages/ef/30/0df458009be6a4caca4ca2c52975e6275c387d4e5c95544e34138b41dc86/pillow-12.1.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:800429ac32c9b72909c671aaf17ecd13110f823ddb7db4dfef412a5587c2c24e", size = 8037872, upload-time = "2026-01-02T09:10:53.446Z" },
+ { url = "https://files.pythonhosted.org/packages/e4/86/95845d4eda4f4f9557e25381d70876aa213560243ac1a6d619c46caaedd9/pillow-12.1.0-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0b022eaaf709541b391ee069f0022ee5b36c709df71986e3f7be312e46f42c84", size = 6345398, upload-time = "2026-01-02T09:10:55.426Z" },
+ { url = "https://files.pythonhosted.org/packages/5c/1f/8e66ab9be3aaf1435bc03edd1ebdf58ffcd17f7349c1d970cafe87af27d9/pillow-12.1.0-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1f345e7bc9d7f368887c712aa5054558bad44d2a301ddf9248599f4161abc7c0", size = 7034667, upload-time = "2026-01-02T09:10:57.11Z" },
+ { url = "https://files.pythonhosted.org/packages/f9/f6/683b83cb9b1db1fb52b87951b1c0b99bdcfceaa75febf11406c19f82cb5e/pillow-12.1.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:d70347c8a5b7ccd803ec0c85c8709f036e6348f1e6a5bf048ecd9c64d3550b8b", size = 6458743, upload-time = "2026-01-02T09:10:59.331Z" },
+ { url = "https://files.pythonhosted.org/packages/9a/7d/de833d63622538c1d58ce5395e7c6cb7e7dce80decdd8bde4a484e095d9f/pillow-12.1.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:1fcc52d86ce7a34fd17cb04e87cfdb164648a3662a6f20565910a99653d66c18", size = 7159342, upload-time = "2026-01-02T09:11:01.82Z" },
+ { url = "https://files.pythonhosted.org/packages/8c/40/50d86571c9e5868c42b81fe7da0c76ca26373f3b95a8dd675425f4a92ec1/pillow-12.1.0-cp311-cp311-win32.whl", hash = "sha256:3ffaa2f0659e2f740473bcf03c702c39a8d4b2b7ffc629052028764324842c64", size = 6328655, upload-time = "2026-01-02T09:11:04.556Z" },
+ { url = "https://files.pythonhosted.org/packages/6c/af/b1d7e301c4cd26cd45d4af884d9ee9b6fab893b0ad2450d4746d74a6968c/pillow-12.1.0-cp311-cp311-win_amd64.whl", hash = "sha256:806f3987ffe10e867bab0ddad45df1148a2b98221798457fa097ad85d6e8bc75", size = 7031469, upload-time = "2026-01-02T09:11:06.538Z" },
+ { url = "https://files.pythonhosted.org/packages/48/36/d5716586d887fb2a810a4a61518a327a1e21c8b7134c89283af272efe84b/pillow-12.1.0-cp311-cp311-win_arm64.whl", hash = "sha256:9f5fefaca968e700ad1a4a9de98bf0869a94e397fe3524c4c9450c1445252304", size = 2452515, upload-time = "2026-01-02T09:11:08.226Z" },
+ { url = "https://files.pythonhosted.org/packages/20/31/dc53fe21a2f2996e1b7d92bf671cdb157079385183ef7c1ae08b485db510/pillow-12.1.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:a332ac4ccb84b6dde65dbace8431f3af08874bf9770719d32a635c4ef411b18b", size = 5262642, upload-time = "2026-01-02T09:11:10.138Z" },
+ { url = "https://files.pythonhosted.org/packages/ab/c1/10e45ac9cc79419cedf5121b42dcca5a50ad2b601fa080f58c22fb27626e/pillow-12.1.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:907bfa8a9cb790748a9aa4513e37c88c59660da3bcfffbd24a7d9e6abf224551", size = 4657464, upload-time = "2026-01-02T09:11:12.319Z" },
+ { url = "https://files.pythonhosted.org/packages/ad/26/7b82c0ab7ef40ebede7a97c72d473bda5950f609f8e0c77b04af574a0ddb/pillow-12.1.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:efdc140e7b63b8f739d09a99033aa430accce485ff78e6d311973a67b6bf3208", size = 6234878, upload-time = "2026-01-02T09:11:14.096Z" },
+ { url = "https://files.pythonhosted.org/packages/76/25/27abc9792615b5e886ca9411ba6637b675f1b77af3104710ac7353fe5605/pillow-12.1.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:bef9768cab184e7ae6e559c032e95ba8d07b3023c289f79a2bd36e8bf85605a5", size = 8044868, upload-time = "2026-01-02T09:11:15.903Z" },
+ { url = "https://files.pythonhosted.org/packages/0a/ea/f200a4c36d836100e7bc738fc48cd963d3ba6372ebc8298a889e0cfc3359/pillow-12.1.0-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:742aea052cf5ab5034a53c3846165bc3ce88d7c38e954120db0ab867ca242661", size = 6349468, upload-time = "2026-01-02T09:11:17.631Z" },
+ { url = "https://files.pythonhosted.org/packages/11/8f/48d0b77ab2200374c66d344459b8958c86693be99526450e7aee714e03e4/pillow-12.1.0-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a6dfc2af5b082b635af6e08e0d1f9f1c4e04d17d4e2ca0ef96131e85eda6eb17", size = 7041518, upload-time = "2026-01-02T09:11:19.389Z" },
+ { url = "https://files.pythonhosted.org/packages/1d/23/c281182eb986b5d31f0a76d2a2c8cd41722d6fb8ed07521e802f9bba52de/pillow-12.1.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:609e89d9f90b581c8d16358c9087df76024cf058fa693dd3e1e1620823f39670", size = 6462829, upload-time = "2026-01-02T09:11:21.28Z" },
+ { url = "https://files.pythonhosted.org/packages/25/ef/7018273e0faac099d7b00982abdcc39142ae6f3bd9ceb06de09779c4a9d6/pillow-12.1.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:43b4899cfd091a9693a1278c4982f3e50f7fb7cff5153b05174b4afc9593b616", size = 7166756, upload-time = "2026-01-02T09:11:23.559Z" },
+ { url = "https://files.pythonhosted.org/packages/8f/c8/993d4b7ab2e341fe02ceef9576afcf5830cdec640be2ac5bee1820d693d4/pillow-12.1.0-cp312-cp312-win32.whl", hash = "sha256:aa0c9cc0b82b14766a99fbe6084409972266e82f459821cd26997a488a7261a7", size = 6328770, upload-time = "2026-01-02T09:11:25.661Z" },
+ { url = "https://files.pythonhosted.org/packages/a7/87/90b358775a3f02765d87655237229ba64a997b87efa8ccaca7dd3e36e7a7/pillow-12.1.0-cp312-cp312-win_amd64.whl", hash = "sha256:d70534cea9e7966169ad29a903b99fc507e932069a881d0965a1a84bb57f6c6d", size = 7033406, upload-time = "2026-01-02T09:11:27.474Z" },
+ { url = "https://files.pythonhosted.org/packages/5d/cf/881b457eccacac9e5b2ddd97d5071fb6d668307c57cbf4e3b5278e06e536/pillow-12.1.0-cp312-cp312-win_arm64.whl", hash = "sha256:65b80c1ee7e14a87d6a068dd3b0aea268ffcabfe0498d38661b00c5b4b22e74c", size = 2452612, upload-time = "2026-01-02T09:11:29.309Z" },
+ { url = "https://files.pythonhosted.org/packages/dd/c7/2530a4aa28248623e9d7f27316b42e27c32ec410f695929696f2e0e4a778/pillow-12.1.0-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:7b5dd7cbae20285cdb597b10eb5a2c13aa9de6cde9bb64a3c1317427b1db1ae1", size = 4062543, upload-time = "2026-01-02T09:11:31.566Z" },
+ { url = "https://files.pythonhosted.org/packages/8f/1f/40b8eae823dc1519b87d53c30ed9ef085506b05281d313031755c1705f73/pillow-12.1.0-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:29a4cef9cb672363926f0470afc516dbf7305a14d8c54f7abbb5c199cd8f8179", size = 4138373, upload-time = "2026-01-02T09:11:33.367Z" },
+ { url = "https://files.pythonhosted.org/packages/d4/77/6fa60634cf06e52139fd0e89e5bbf055e8166c691c42fb162818b7fda31d/pillow-12.1.0-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:681088909d7e8fa9e31b9799aaa59ba5234c58e5e4f1951b4c4d1082a2e980e0", size = 3601241, upload-time = "2026-01-02T09:11:35.011Z" },
+ { url = "https://files.pythonhosted.org/packages/4f/bf/28ab865de622e14b747f0cd7877510848252d950e43002e224fb1c9ababf/pillow-12.1.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:983976c2ab753166dc66d36af6e8ec15bb511e4a25856e2227e5f7e00a160587", size = 5262410, upload-time = "2026-01-02T09:11:36.682Z" },
+ { url = "https://files.pythonhosted.org/packages/1c/34/583420a1b55e715937a85bd48c5c0991598247a1fd2eb5423188e765ea02/pillow-12.1.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:db44d5c160a90df2d24a24760bbd37607d53da0b34fb546c4c232af7192298ac", size = 4657312, upload-time = "2026-01-02T09:11:38.535Z" },
+ { url = "https://files.pythonhosted.org/packages/1d/fd/f5a0896839762885b3376ff04878f86ab2b097c2f9a9cdccf4eda8ba8dc0/pillow-12.1.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:6b7a9d1db5dad90e2991645874f708e87d9a3c370c243c2d7684d28f7e133e6b", size = 6232605, upload-time = "2026-01-02T09:11:40.602Z" },
+ { url = "https://files.pythonhosted.org/packages/98/aa/938a09d127ac1e70e6ed467bd03834350b33ef646b31edb7452d5de43792/pillow-12.1.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:6258f3260986990ba2fa8a874f8b6e808cf5abb51a94015ca3dc3c68aa4f30ea", size = 8041617, upload-time = "2026-01-02T09:11:42.721Z" },
+ { url = "https://files.pythonhosted.org/packages/17/e8/538b24cb426ac0186e03f80f78bc8dc7246c667f58b540bdd57c71c9f79d/pillow-12.1.0-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e115c15e3bc727b1ca3e641a909f77f8ca72a64fff150f666fcc85e57701c26c", size = 6346509, upload-time = "2026-01-02T09:11:44.955Z" },
+ { url = "https://files.pythonhosted.org/packages/01/9a/632e58ec89a32738cabfd9ec418f0e9898a2b4719afc581f07c04a05e3c9/pillow-12.1.0-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6741e6f3074a35e47c77b23a4e4f2d90db3ed905cb1c5e6e0d49bff2045632bc", size = 7038117, upload-time = "2026-01-02T09:11:46.736Z" },
+ { url = "https://files.pythonhosted.org/packages/c7/a2/d40308cf86eada842ca1f3ffa45d0ca0df7e4ab33c83f81e73f5eaed136d/pillow-12.1.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:935b9d1aed48fcfb3f838caac506f38e29621b44ccc4f8a64d575cb1b2a88644", size = 6460151, upload-time = "2026-01-02T09:11:48.625Z" },
+ { url = "https://files.pythonhosted.org/packages/f1/88/f5b058ad6453a085c5266660a1417bdad590199da1b32fb4efcff9d33b05/pillow-12.1.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:5fee4c04aad8932da9f8f710af2c1a15a83582cfb884152a9caa79d4efcdbf9c", size = 7164534, upload-time = "2026-01-02T09:11:50.445Z" },
+ { url = "https://files.pythonhosted.org/packages/19/ce/c17334caea1db789163b5d855a5735e47995b0b5dc8745e9a3605d5f24c0/pillow-12.1.0-cp313-cp313-win32.whl", hash = "sha256:a786bf667724d84aa29b5db1c61b7bfdde380202aaca12c3461afd6b71743171", size = 6332551, upload-time = "2026-01-02T09:11:52.234Z" },
+ { url = "https://files.pythonhosted.org/packages/e5/07/74a9d941fa45c90a0d9465098fe1ec85de3e2afbdc15cc4766622d516056/pillow-12.1.0-cp313-cp313-win_amd64.whl", hash = "sha256:461f9dfdafa394c59cd6d818bdfdbab4028b83b02caadaff0ffd433faf4c9a7a", size = 7040087, upload-time = "2026-01-02T09:11:54.822Z" },
+ { url = "https://files.pythonhosted.org/packages/88/09/c99950c075a0e9053d8e880595926302575bc742b1b47fe1bbcc8d388d50/pillow-12.1.0-cp313-cp313-win_arm64.whl", hash = "sha256:9212d6b86917a2300669511ed094a9406888362e085f2431a7da985a6b124f45", size = 2452470, upload-time = "2026-01-02T09:11:56.522Z" },
+ { url = "https://files.pythonhosted.org/packages/b5/ba/970b7d85ba01f348dee4d65412476321d40ee04dcb51cd3735b9dc94eb58/pillow-12.1.0-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:00162e9ca6d22b7c3ee8e61faa3c3253cd19b6a37f126cad04f2f88b306f557d", size = 5264816, upload-time = "2026-01-02T09:11:58.227Z" },
+ { url = "https://files.pythonhosted.org/packages/10/60/650f2fb55fdba7a510d836202aa52f0baac633e50ab1cf18415d332188fb/pillow-12.1.0-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:7d6daa89a00b58c37cb1747ec9fb7ac3bc5ffd5949f5888657dfddde6d1312e0", size = 4660472, upload-time = "2026-01-02T09:12:00.798Z" },
+ { url = "https://files.pythonhosted.org/packages/2b/c0/5273a99478956a099d533c4f46cbaa19fd69d606624f4334b85e50987a08/pillow-12.1.0-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:e2479c7f02f9d505682dc47df8c0ea1fc5e264c4d1629a5d63fe3e2334b89554", size = 6268974, upload-time = "2026-01-02T09:12:02.572Z" },
+ { url = "https://files.pythonhosted.org/packages/b4/26/0bf714bc2e73d5267887d47931d53c4ceeceea6978148ed2ab2a4e6463c4/pillow-12.1.0-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:f188d580bd870cda1e15183790d1cc2fa78f666e76077d103edf048eed9c356e", size = 8073070, upload-time = "2026-01-02T09:12:04.75Z" },
+ { url = "https://files.pythonhosted.org/packages/43/cf/1ea826200de111a9d65724c54f927f3111dc5ae297f294b370a670c17786/pillow-12.1.0-cp313-cp313t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0fde7ec5538ab5095cc02df38ee99b0443ff0e1c847a045554cf5f9af1f4aa82", size = 6380176, upload-time = "2026-01-02T09:12:06.626Z" },
+ { url = "https://files.pythonhosted.org/packages/03/e0/7938dd2b2013373fd85d96e0f38d62b7a5a262af21ac274250c7ca7847c9/pillow-12.1.0-cp313-cp313t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0ed07dca4a8464bada6139ab38f5382f83e5f111698caf3191cb8dbf27d908b4", size = 7067061, upload-time = "2026-01-02T09:12:08.624Z" },
+ { url = "https://files.pythonhosted.org/packages/86/ad/a2aa97d37272a929a98437a8c0ac37b3cf012f4f8721e1bd5154699b2518/pillow-12.1.0-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:f45bd71d1fa5e5749587613037b172e0b3b23159d1c00ef2fc920da6f470e6f0", size = 6491824, upload-time = "2026-01-02T09:12:10.488Z" },
+ { url = "https://files.pythonhosted.org/packages/a4/44/80e46611b288d51b115826f136fb3465653c28f491068a72d3da49b54cd4/pillow-12.1.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:277518bf4fe74aa91489e1b20577473b19ee70fb97c374aa50830b279f25841b", size = 7190911, upload-time = "2026-01-02T09:12:12.772Z" },
+ { url = "https://files.pythonhosted.org/packages/86/77/eacc62356b4cf81abe99ff9dbc7402750044aed02cfd6a503f7c6fc11f3e/pillow-12.1.0-cp313-cp313t-win32.whl", hash = "sha256:7315f9137087c4e0ee73a761b163fc9aa3b19f5f606a7fc08d83fd3e4379af65", size = 6336445, upload-time = "2026-01-02T09:12:14.775Z" },
+ { url = "https://files.pythonhosted.org/packages/e7/3c/57d81d0b74d218706dafccb87a87ea44262c43eef98eb3b164fd000e0491/pillow-12.1.0-cp313-cp313t-win_amd64.whl", hash = "sha256:0ddedfaa8b5f0b4ffbc2fa87b556dc59f6bb4ecb14a53b33f9189713ae8053c0", size = 7045354, upload-time = "2026-01-02T09:12:16.599Z" },
+ { url = "https://files.pythonhosted.org/packages/ac/82/8b9b97bba2e3576a340f93b044a3a3a09841170ab4c1eb0d5c93469fd32f/pillow-12.1.0-cp313-cp313t-win_arm64.whl", hash = "sha256:80941e6d573197a0c28f394753de529bb436b1ca990ed6e765cf42426abc39f8", size = 2454547, upload-time = "2026-01-02T09:12:18.704Z" },
+ { url = "https://files.pythonhosted.org/packages/8c/87/bdf971d8bbcf80a348cc3bacfcb239f5882100fe80534b0ce67a784181d8/pillow-12.1.0-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:5cb7bc1966d031aec37ddb9dcf15c2da5b2e9f7cc3ca7c54473a20a927e1eb91", size = 4062533, upload-time = "2026-01-02T09:12:20.791Z" },
+ { url = "https://files.pythonhosted.org/packages/ff/4f/5eb37a681c68d605eb7034c004875c81f86ec9ef51f5be4a63eadd58859a/pillow-12.1.0-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:97e9993d5ed946aba26baf9c1e8cf18adbab584b99f452ee72f7ee8acb882796", size = 4138546, upload-time = "2026-01-02T09:12:23.664Z" },
+ { url = "https://files.pythonhosted.org/packages/11/6d/19a95acb2edbace40dcd582d077b991646b7083c41b98da4ed7555b59733/pillow-12.1.0-cp314-cp314-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:414b9a78e14ffeb98128863314e62c3f24b8a86081066625700b7985b3f529bd", size = 3601163, upload-time = "2026-01-02T09:12:26.338Z" },
+ { url = "https://files.pythonhosted.org/packages/fc/36/2b8138e51cb42e4cc39c3297713455548be855a50558c3ac2beebdc251dd/pillow-12.1.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:e6bdb408f7c9dd2a5ff2b14a3b0bb6d4deb29fb9961e6eb3ae2031ae9a5cec13", size = 5266086, upload-time = "2026-01-02T09:12:28.782Z" },
+ { url = "https://files.pythonhosted.org/packages/53/4b/649056e4d22e1caa90816bf99cef0884aed607ed38075bd75f091a607a38/pillow-12.1.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:3413c2ae377550f5487991d444428f1a8ae92784aac79caa8b1e3b89b175f77e", size = 4657344, upload-time = "2026-01-02T09:12:31.117Z" },
+ { url = "https://files.pythonhosted.org/packages/6c/6b/c5742cea0f1ade0cd61485dc3d81f05261fc2276f537fbdc00802de56779/pillow-12.1.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:e5dcbe95016e88437ecf33544ba5db21ef1b8dd6e1b434a2cb2a3d605299e643", size = 6232114, upload-time = "2026-01-02T09:12:32.936Z" },
+ { url = "https://files.pythonhosted.org/packages/bf/8f/9f521268ce22d63991601aafd3d48d5ff7280a246a1ef62d626d67b44064/pillow-12.1.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:d0a7735df32ccbcc98b98a1ac785cc4b19b580be1bdf0aeb5c03223220ea09d5", size = 8042708, upload-time = "2026-01-02T09:12:34.78Z" },
+ { url = "https://files.pythonhosted.org/packages/1a/eb/257f38542893f021502a1bbe0c2e883c90b5cff26cc33b1584a841a06d30/pillow-12.1.0-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0c27407a2d1b96774cbc4a7594129cc027339fd800cd081e44497722ea1179de", size = 6347762, upload-time = "2026-01-02T09:12:36.748Z" },
+ { url = "https://files.pythonhosted.org/packages/c4/5a/8ba375025701c09b309e8d5163c5a4ce0102fa86bbf8800eb0d7ac87bc51/pillow-12.1.0-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:15c794d74303828eaa957ff8070846d0efe8c630901a1c753fdc63850e19ecd9", size = 7039265, upload-time = "2026-01-02T09:12:39.082Z" },
+ { url = "https://files.pythonhosted.org/packages/cf/dc/cf5e4cdb3db533f539e88a7bbf9f190c64ab8a08a9bc7a4ccf55067872e4/pillow-12.1.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:c990547452ee2800d8506c4150280757f88532f3de2a58e3022e9b179107862a", size = 6462341, upload-time = "2026-01-02T09:12:40.946Z" },
+ { url = "https://files.pythonhosted.org/packages/d0/47/0291a25ac9550677e22eda48510cfc4fa4b2ef0396448b7fbdc0a6946309/pillow-12.1.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:b63e13dd27da389ed9475b3d28510f0f954bca0041e8e551b2a4eb1eab56a39a", size = 7165395, upload-time = "2026-01-02T09:12:42.706Z" },
+ { url = "https://files.pythonhosted.org/packages/4f/4c/e005a59393ec4d9416be06e6b45820403bb946a778e39ecec62f5b2b991e/pillow-12.1.0-cp314-cp314-win32.whl", hash = "sha256:1a949604f73eb07a8adab38c4fe50791f9919344398bdc8ac6b307f755fc7030", size = 6431413, upload-time = "2026-01-02T09:12:44.944Z" },
+ { url = "https://files.pythonhosted.org/packages/1c/af/f23697f587ac5f9095d67e31b81c95c0249cd461a9798a061ed6709b09b5/pillow-12.1.0-cp314-cp314-win_amd64.whl", hash = "sha256:4f9f6a650743f0ddee5593ac9e954ba1bdbc5e150bc066586d4f26127853ab94", size = 7176779, upload-time = "2026-01-02T09:12:46.727Z" },
+ { url = "https://files.pythonhosted.org/packages/b3/36/6a51abf8599232f3e9afbd16d52829376a68909fe14efe29084445db4b73/pillow-12.1.0-cp314-cp314-win_arm64.whl", hash = "sha256:808b99604f7873c800c4840f55ff389936ef1948e4e87645eaf3fccbc8477ac4", size = 2543105, upload-time = "2026-01-02T09:12:49.243Z" },
+ { url = "https://files.pythonhosted.org/packages/82/54/2e1dd20c8749ff225080d6ba465a0cab4387f5db0d1c5fb1439e2d99923f/pillow-12.1.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:bc11908616c8a283cf7d664f77411a5ed2a02009b0097ff8abbba5e79128ccf2", size = 5268571, upload-time = "2026-01-02T09:12:51.11Z" },
+ { url = "https://files.pythonhosted.org/packages/57/61/571163a5ef86ec0cf30d265ac2a70ae6fc9e28413d1dc94fa37fae6bda89/pillow-12.1.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:896866d2d436563fa2a43a9d72f417874f16b5545955c54a64941e87c1376c61", size = 4660426, upload-time = "2026-01-02T09:12:52.865Z" },
+ { url = "https://files.pythonhosted.org/packages/5e/e1/53ee5163f794aef1bf84243f755ee6897a92c708505350dd1923f4afec48/pillow-12.1.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:8e178e3e99d3c0ea8fc64b88447f7cac8ccf058af422a6cedc690d0eadd98c51", size = 6269908, upload-time = "2026-01-02T09:12:54.884Z" },
+ { url = "https://files.pythonhosted.org/packages/bc/0b/b4b4106ff0ee1afa1dc599fde6ab230417f800279745124f6c50bcffed8e/pillow-12.1.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:079af2fb0c599c2ec144ba2c02766d1b55498e373b3ac64687e43849fbbef5bc", size = 8074733, upload-time = "2026-01-02T09:12:56.802Z" },
+ { url = "https://files.pythonhosted.org/packages/19/9f/80b411cbac4a732439e629a26ad3ef11907a8c7fc5377b7602f04f6fe4e7/pillow-12.1.0-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:bdec5e43377761c5dbca620efb69a77f6855c5a379e32ac5b158f54c84212b14", size = 6381431, upload-time = "2026-01-02T09:12:58.823Z" },
+ { url = "https://files.pythonhosted.org/packages/8f/b7/d65c45db463b66ecb6abc17c6ba6917a911202a07662247e1355ce1789e7/pillow-12.1.0-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:565c986f4b45c020f5421a4cea13ef294dde9509a8577f29b2fc5edc7587fff8", size = 7068529, upload-time = "2026-01-02T09:13:00.885Z" },
+ { url = "https://files.pythonhosted.org/packages/50/96/dfd4cd726b4a45ae6e3c669fc9e49deb2241312605d33aba50499e9d9bd1/pillow-12.1.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:43aca0a55ce1eefc0aefa6253661cb54571857b1a7b2964bd8a1e3ef4b729924", size = 6492981, upload-time = "2026-01-02T09:13:03.314Z" },
+ { url = "https://files.pythonhosted.org/packages/4d/1c/b5dc52cf713ae46033359c5ca920444f18a6359ce1020dd3e9c553ea5bc6/pillow-12.1.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:0deedf2ea233722476b3a81e8cdfbad786f7adbed5d848469fa59fe52396e4ef", size = 7191878, upload-time = "2026-01-02T09:13:05.276Z" },
+ { url = "https://files.pythonhosted.org/packages/53/26/c4188248bd5edaf543864fe4834aebe9c9cb4968b6f573ce014cc42d0720/pillow-12.1.0-cp314-cp314t-win32.whl", hash = "sha256:b17fbdbe01c196e7e159aacb889e091f28e61020a8abeac07b68079b6e626988", size = 6438703, upload-time = "2026-01-02T09:13:07.491Z" },
+ { url = "https://files.pythonhosted.org/packages/b8/0e/69ed296de8ea05cb03ee139cee600f424ca166e632567b2d66727f08c7ed/pillow-12.1.0-cp314-cp314t-win_amd64.whl", hash = "sha256:27b9baecb428899db6c0de572d6d305cfaf38ca1596b5c0542a5182e3e74e8c6", size = 7182927, upload-time = "2026-01-02T09:13:09.841Z" },
+ { url = "https://files.pythonhosted.org/packages/fc/f5/68334c015eed9b5cff77814258717dec591ded209ab5b6fb70e2ae873d1d/pillow-12.1.0-cp314-cp314t-win_arm64.whl", hash = "sha256:f61333d817698bdcdd0f9d7793e365ac3d2a21c1f1eb02b32ad6aefb8d8ea831", size = 2545104, upload-time = "2026-01-02T09:13:12.068Z" },
+ { url = "https://files.pythonhosted.org/packages/8b/bc/224b1d98cffd7164b14707c91aac83c07b047fbd8f58eba4066a3e53746a/pillow-12.1.0-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:ca94b6aac0d7af2a10ba08c0f888b3d5114439b6b3ef39968378723622fed377", size = 5228605, upload-time = "2026-01-02T09:13:14.084Z" },
+ { url = "https://files.pythonhosted.org/packages/0c/ca/49ca7769c4550107de049ed85208240ba0f330b3f2e316f24534795702ce/pillow-12.1.0-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:351889afef0f485b84078ea40fe33727a0492b9af3904661b0abbafee0355b72", size = 4622245, upload-time = "2026-01-02T09:13:15.964Z" },
+ { url = "https://files.pythonhosted.org/packages/73/48/fac807ce82e5955bcc2718642b94b1bd22a82a6d452aea31cbb678cddf12/pillow-12.1.0-pp311-pypy311_pp73-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:bb0984b30e973f7e2884362b7d23d0a348c7143ee559f38ef3eaab640144204c", size = 5247593, upload-time = "2026-01-02T09:13:17.913Z" },
+ { url = "https://files.pythonhosted.org/packages/d2/95/3e0742fe358c4664aed4fd05d5f5373dcdad0b27af52aa0972568541e3f4/pillow-12.1.0-pp311-pypy311_pp73-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:84cabc7095dd535ca934d57e9ce2a72ffd216e435a84acb06b2277b1de2689bd", size = 6989008, upload-time = "2026-01-02T09:13:20.083Z" },
+ { url = "https://files.pythonhosted.org/packages/5a/74/fe2ac378e4e202e56d50540d92e1ef4ff34ed687f3c60f6a121bcf99437e/pillow-12.1.0-pp311-pypy311_pp73-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:53d8b764726d3af1a138dd353116f774e3862ec7e3794e0c8781e30db0f35dfc", size = 5313824, upload-time = "2026-01-02T09:13:22.405Z" },
+ { url = "https://files.pythonhosted.org/packages/f3/77/2a60dee1adee4e2655ac328dd05c02a955c1cd683b9f1b82ec3feb44727c/pillow-12.1.0-pp311-pypy311_pp73-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5da841d81b1a05ef940a8567da92decaa15bc4d7dedb540a8c219ad83d91808a", size = 5963278, upload-time = "2026-01-02T09:13:24.706Z" },
+ { url = "https://files.pythonhosted.org/packages/2d/71/64e9b1c7f04ae0027f788a248e6297d7fcc29571371fe7d45495a78172c0/pillow-12.1.0-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:75af0b4c229ac519b155028fa1be632d812a519abba9b46b20e50c6caa184f19", size = 7029809, upload-time = "2026-01-02T09:13:26.541Z" },
]
[[package]]
name = "platformdirs"
-version = "4.5.0"
+version = "4.5.1"
source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/61/33/9611380c2bdb1225fdef633e2a9610622310fed35ab11dac9620972ee088/platformdirs-4.5.0.tar.gz", hash = "sha256:70ddccdd7c99fc5942e9fc25636a8b34d04c24b335100223152c2803e4063312", size = 21632, upload-time = "2025-10-08T17:44:48.791Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/cf/86/0248f086a84f01b37aaec0fa567b397df1a119f73c16f6c7a9aac73ea309/platformdirs-4.5.1.tar.gz", hash = "sha256:61d5cdcc6065745cdd94f0f878977f8de9437be93de97c1c12f853c9c0cdcbda", size = 21715, upload-time = "2025-12-05T13:52:58.638Z" }
wheels = [
- { url = "https://files.pythonhosted.org/packages/73/cb/ac7874b3e5d58441674fb70742e6c374b28b0c7cb988d37d991cde47166c/platformdirs-4.5.0-py3-none-any.whl", hash = "sha256:e578a81bb873cbb89a41fcc904c7ef523cc18284b7e3b3ccf06aca1403b7ebd3", size = 18651, upload-time = "2025-10-08T17:44:47.223Z" },
+ { url = "https://files.pythonhosted.org/packages/cb/28/3bfe2fa5a7b9c46fe7e13c97bda14c895fb10fa2ebf1d0abb90e0cea7ee1/platformdirs-4.5.1-py3-none-any.whl", hash = "sha256:d03afa3963c806a9bed9d5125c8f4cb2fdaf74a55ab60e5d59b3fde758104d31", size = 18731, upload-time = "2025-12-05T13:52:56.823Z" },
]
[[package]]
@@ -3864,43 +3890,45 @@ wheels = [
[[package]]
name = "protobuf"
-version = "6.33.1"
+version = "6.33.2"
source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/0a/03/a1440979a3f74f16cab3b75b0da1a1a7f922d56a8ddea96092391998edc0/protobuf-6.33.1.tar.gz", hash = "sha256:97f65757e8d09870de6fd973aeddb92f85435607235d20b2dfed93405d00c85b", size = 443432, upload-time = "2025-11-13T16:44:18.895Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/34/44/e49ecff446afeec9d1a66d6bbf9adc21e3c7cea7803a920ca3773379d4f6/protobuf-6.33.2.tar.gz", hash = "sha256:56dc370c91fbb8ac85bc13582c9e373569668a290aa2e66a590c2a0d35ddb9e4", size = 444296, upload-time = "2025-12-06T00:17:53.311Z" }
wheels = [
- { url = "https://files.pythonhosted.org/packages/06/f1/446a9bbd2c60772ca36556bac8bfde40eceb28d9cc7838755bc41e001d8f/protobuf-6.33.1-cp310-abi3-win32.whl", hash = "sha256:f8d3fdbc966aaab1d05046d0240dd94d40f2a8c62856d41eaa141ff64a79de6b", size = 425593, upload-time = "2025-11-13T16:44:06.275Z" },
- { url = "https://files.pythonhosted.org/packages/a6/79/8780a378c650e3df849b73de8b13cf5412f521ca2ff9b78a45c247029440/protobuf-6.33.1-cp310-abi3-win_amd64.whl", hash = "sha256:923aa6d27a92bf44394f6abf7ea0500f38769d4b07f4be41cb52bd8b1123b9ed", size = 436883, upload-time = "2025-11-13T16:44:09.222Z" },
- { url = "https://files.pythonhosted.org/packages/cd/93/26213ff72b103ae55bb0d73e7fb91ea570ef407c3ab4fd2f1f27cac16044/protobuf-6.33.1-cp39-abi3-macosx_10_9_universal2.whl", hash = "sha256:fe34575f2bdde76ac429ec7b570235bf0c788883e70aee90068e9981806f2490", size = 427522, upload-time = "2025-11-13T16:44:10.475Z" },
- { url = "https://files.pythonhosted.org/packages/c2/32/df4a35247923393aa6b887c3b3244a8c941c32a25681775f96e2b418f90e/protobuf-6.33.1-cp39-abi3-manylinux2014_aarch64.whl", hash = "sha256:f8adba2e44cde2d7618996b3fc02341f03f5bc3f2748be72dc7b063319276178", size = 324445, upload-time = "2025-11-13T16:44:11.869Z" },
- { url = "https://files.pythonhosted.org/packages/8e/d0/d796e419e2ec93d2f3fa44888861c3f88f722cde02b7c3488fcc6a166820/protobuf-6.33.1-cp39-abi3-manylinux2014_s390x.whl", hash = "sha256:0f4cf01222c0d959c2b399142deb526de420be8236f22c71356e2a544e153c53", size = 339161, upload-time = "2025-11-13T16:44:12.778Z" },
- { url = "https://files.pythonhosted.org/packages/1d/2a/3c5f05a4af06649547027d288747f68525755de692a26a7720dced3652c0/protobuf-6.33.1-cp39-abi3-manylinux2014_x86_64.whl", hash = "sha256:8fd7d5e0eb08cd5b87fd3df49bc193f5cfd778701f47e11d127d0afc6c39f1d1", size = 323171, upload-time = "2025-11-13T16:44:14.035Z" },
- { url = "https://files.pythonhosted.org/packages/08/b4/46310463b4f6ceef310f8348786f3cff181cea671578e3d9743ba61a459e/protobuf-6.33.1-py3-none-any.whl", hash = "sha256:d595a9fd694fdeb061a62fbe10eb039cc1e444df81ec9bb70c7fc59ebcb1eafa", size = 170477, upload-time = "2025-11-13T16:44:17.633Z" },
+ { url = "https://files.pythonhosted.org/packages/bc/91/1e3a34881a88697a7354ffd177e8746e97a722e5e8db101544b47e84afb1/protobuf-6.33.2-cp310-abi3-win32.whl", hash = "sha256:87eb388bd2d0f78febd8f4c8779c79247b26a5befad525008e49a6955787ff3d", size = 425603, upload-time = "2025-12-06T00:17:41.114Z" },
+ { url = "https://files.pythonhosted.org/packages/64/20/4d50191997e917ae13ad0a235c8b42d8c1ab9c3e6fd455ca16d416944355/protobuf-6.33.2-cp310-abi3-win_amd64.whl", hash = "sha256:fc2a0e8b05b180e5fc0dd1559fe8ebdae21a27e81ac77728fb6c42b12c7419b4", size = 436930, upload-time = "2025-12-06T00:17:43.278Z" },
+ { url = "https://files.pythonhosted.org/packages/b2/ca/7e485da88ba45c920fb3f50ae78de29ab925d9e54ef0de678306abfbb497/protobuf-6.33.2-cp39-abi3-macosx_10_9_universal2.whl", hash = "sha256:d9b19771ca75935b3a4422957bc518b0cecb978b31d1dd12037b088f6bcc0e43", size = 427621, upload-time = "2025-12-06T00:17:44.445Z" },
+ { url = "https://files.pythonhosted.org/packages/7d/4f/f743761e41d3b2b2566748eb76bbff2b43e14d5fcab694f494a16458b05f/protobuf-6.33.2-cp39-abi3-manylinux2014_aarch64.whl", hash = "sha256:b5d3b5625192214066d99b2b605f5783483575656784de223f00a8d00754fc0e", size = 324460, upload-time = "2025-12-06T00:17:45.678Z" },
+ { url = "https://files.pythonhosted.org/packages/b1/fa/26468d00a92824020f6f2090d827078c09c9c587e34cbfd2d0c7911221f8/protobuf-6.33.2-cp39-abi3-manylinux2014_s390x.whl", hash = "sha256:8cd7640aee0b7828b6d03ae518b5b4806fdfc1afe8de82f79c3454f8aef29872", size = 339168, upload-time = "2025-12-06T00:17:46.813Z" },
+ { url = "https://files.pythonhosted.org/packages/56/13/333b8f421738f149d4fe5e49553bc2a2ab75235486259f689b4b91f96cec/protobuf-6.33.2-cp39-abi3-manylinux2014_x86_64.whl", hash = "sha256:1f8017c48c07ec5859106533b682260ba3d7c5567b1ca1f24297ce03384d1b4f", size = 323270, upload-time = "2025-12-06T00:17:48.253Z" },
+ { url = "https://files.pythonhosted.org/packages/0e/15/4f02896cc3df04fc465010a4c6a0cd89810f54617a32a70ef531ed75d61c/protobuf-6.33.2-py3-none-any.whl", hash = "sha256:7636aad9bb01768870266de5dc009de2d1b936771b38a793f73cbbf279c91c5c", size = 170501, upload-time = "2025-12-06T00:17:52.211Z" },
]
[[package]]
name = "psutil"
-version = "7.1.3"
-source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/e1/88/bdd0a41e5857d5d703287598cbf08dad90aed56774ea52ae071bae9071b6/psutil-7.1.3.tar.gz", hash = "sha256:6c86281738d77335af7aec228328e944b30930899ea760ecf33a4dba66be5e74", size = 489059, upload-time = "2025-11-02T12:25:54.619Z" }
-wheels = [
- { url = "https://files.pythonhosted.org/packages/bd/93/0c49e776b8734fef56ec9c5c57f923922f2cf0497d62e0f419465f28f3d0/psutil-7.1.3-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:0005da714eee687b4b8decd3d6cc7c6db36215c9e74e5ad2264b90c3df7d92dc", size = 239751, upload-time = "2025-11-02T12:25:58.161Z" },
- { url = "https://files.pythonhosted.org/packages/6f/8d/b31e39c769e70780f007969815195a55c81a63efebdd4dbe9e7a113adb2f/psutil-7.1.3-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:19644c85dcb987e35eeeaefdc3915d059dac7bd1167cdcdbf27e0ce2df0c08c0", size = 240368, upload-time = "2025-11-02T12:26:00.491Z" },
- { url = "https://files.pythonhosted.org/packages/62/61/23fd4acc3c9eebbf6b6c78bcd89e5d020cfde4acf0a9233e9d4e3fa698b4/psutil-7.1.3-cp313-cp313t-manylinux2010_x86_64.manylinux_2_12_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:95ef04cf2e5ba0ab9eaafc4a11eaae91b44f4ef5541acd2ee91d9108d00d59a7", size = 287134, upload-time = "2025-11-02T12:26:02.613Z" },
- { url = "https://files.pythonhosted.org/packages/30/1c/f921a009ea9ceb51aa355cb0cc118f68d354db36eae18174bab63affb3e6/psutil-7.1.3-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1068c303be3a72f8e18e412c5b2a8f6d31750fb152f9cb106b54090296c9d251", size = 289904, upload-time = "2025-11-02T12:26:05.207Z" },
- { url = "https://files.pythonhosted.org/packages/a6/82/62d68066e13e46a5116df187d319d1724b3f437ddd0f958756fc052677f4/psutil-7.1.3-cp313-cp313t-win_amd64.whl", hash = "sha256:18349c5c24b06ac5612c0428ec2a0331c26443d259e2a0144a9b24b4395b58fa", size = 249642, upload-time = "2025-11-02T12:26:07.447Z" },
- { url = "https://files.pythonhosted.org/packages/df/ad/c1cd5fe965c14a0392112f68362cfceb5230819dbb5b1888950d18a11d9f/psutil-7.1.3-cp313-cp313t-win_arm64.whl", hash = "sha256:c525ffa774fe4496282fb0b1187725793de3e7c6b29e41562733cae9ada151ee", size = 245518, upload-time = "2025-11-02T12:26:09.719Z" },
- { url = "https://files.pythonhosted.org/packages/2e/bb/6670bded3e3236eb4287c7bcdc167e9fae6e1e9286e437f7111caed2f909/psutil-7.1.3-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:b403da1df4d6d43973dc004d19cee3b848e998ae3154cc8097d139b77156c353", size = 239843, upload-time = "2025-11-02T12:26:11.968Z" },
- { url = "https://files.pythonhosted.org/packages/b8/66/853d50e75a38c9a7370ddbeefabdd3d3116b9c31ef94dc92c6729bc36bec/psutil-7.1.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:ad81425efc5e75da3f39b3e636293360ad8d0b49bed7df824c79764fb4ba9b8b", size = 240369, upload-time = "2025-11-02T12:26:14.358Z" },
- { url = "https://files.pythonhosted.org/packages/41/bd/313aba97cb5bfb26916dc29cf0646cbe4dd6a89ca69e8c6edce654876d39/psutil-7.1.3-cp314-cp314t-manylinux2010_x86_64.manylinux_2_12_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8f33a3702e167783a9213db10ad29650ebf383946e91bc77f28a5eb083496bc9", size = 288210, upload-time = "2025-11-02T12:26:16.699Z" },
- { url = "https://files.pythonhosted.org/packages/c2/fa/76e3c06e760927a0cfb5705eb38164254de34e9bd86db656d4dbaa228b04/psutil-7.1.3-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fac9cd332c67f4422504297889da5ab7e05fd11e3c4392140f7370f4208ded1f", size = 291182, upload-time = "2025-11-02T12:26:18.848Z" },
- { url = "https://files.pythonhosted.org/packages/0f/1d/5774a91607035ee5078b8fd747686ebec28a962f178712de100d00b78a32/psutil-7.1.3-cp314-cp314t-win_amd64.whl", hash = "sha256:3792983e23b69843aea49c8f5b8f115572c5ab64c153bada5270086a2123c7e7", size = 250466, upload-time = "2025-11-02T12:26:21.183Z" },
- { url = "https://files.pythonhosted.org/packages/00/ca/e426584bacb43a5cb1ac91fae1937f478cd8fbe5e4ff96574e698a2c77cd/psutil-7.1.3-cp314-cp314t-win_arm64.whl", hash = "sha256:31d77fcedb7529f27bb3a0472bea9334349f9a04160e8e6e5020f22c59893264", size = 245756, upload-time = "2025-11-02T12:26:23.148Z" },
- { url = "https://files.pythonhosted.org/packages/ef/94/46b9154a800253e7ecff5aaacdf8ebf43db99de4a2dfa18575b02548654e/psutil-7.1.3-cp36-abi3-macosx_10_9_x86_64.whl", hash = "sha256:2bdbcd0e58ca14996a42adf3621a6244f1bb2e2e528886959c72cf1e326677ab", size = 238359, upload-time = "2025-11-02T12:26:25.284Z" },
- { url = "https://files.pythonhosted.org/packages/68/3a/9f93cff5c025029a36d9a92fef47220ab4692ee7f2be0fba9f92813d0cb8/psutil-7.1.3-cp36-abi3-macosx_11_0_arm64.whl", hash = "sha256:bc31fa00f1fbc3c3802141eede66f3a2d51d89716a194bf2cd6fc68310a19880", size = 239171, upload-time = "2025-11-02T12:26:27.23Z" },
- { url = "https://files.pythonhosted.org/packages/ce/b1/5f49af514f76431ba4eea935b8ad3725cdeb397e9245ab919dbc1d1dc20f/psutil-7.1.3-cp36-abi3-manylinux2010_x86_64.manylinux_2_12_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3bb428f9f05c1225a558f53e30ccbad9930b11c3fc206836242de1091d3e7dd3", size = 263261, upload-time = "2025-11-02T12:26:29.48Z" },
- { url = "https://files.pythonhosted.org/packages/e0/95/992c8816a74016eb095e73585d747e0a8ea21a061ed3689474fabb29a395/psutil-7.1.3-cp36-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:56d974e02ca2c8eb4812c3f76c30e28836fffc311d55d979f1465c1feeb2b68b", size = 264635, upload-time = "2025-11-02T12:26:31.74Z" },
- { url = "https://files.pythonhosted.org/packages/55/4c/c3ed1a622b6ae2fd3c945a366e64eb35247a31e4db16cf5095e269e8eb3c/psutil-7.1.3-cp37-abi3-win_amd64.whl", hash = "sha256:f39c2c19fe824b47484b96f9692932248a54c43799a84282cfe58d05a6449efd", size = 247633, upload-time = "2025-11-02T12:26:33.887Z" },
- { url = "https://files.pythonhosted.org/packages/c9/ad/33b2ccec09bf96c2b2ef3f9a6f66baac8253d7565d8839e024a6b905d45d/psutil-7.1.3-cp37-abi3-win_arm64.whl", hash = "sha256:bd0d69cee829226a761e92f28140bec9a5ee9d5b4fb4b0cc589068dbfff559b1", size = 244608, upload-time = "2025-11-02T12:26:36.136Z" },
+version = "7.2.1"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/73/cb/09e5184fb5fc0358d110fc3ca7f6b1d033800734d34cac10f4136cfac10e/psutil-7.2.1.tar.gz", hash = "sha256:f7583aec590485b43ca601dd9cea0dcd65bd7bb21d30ef4ddbf4ea6b5ed1bdd3", size = 490253, upload-time = "2025-12-29T08:26:00.169Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/77/8e/f0c242053a368c2aa89584ecd1b054a18683f13d6e5a318fc9ec36582c94/psutil-7.2.1-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:ba9f33bb525b14c3ea563b2fd521a84d2fa214ec59e3e6a2858f78d0844dd60d", size = 129624, upload-time = "2025-12-29T08:26:04.255Z" },
+ { url = "https://files.pythonhosted.org/packages/26/97/a58a4968f8990617decee234258a2b4fc7cd9e35668387646c1963e69f26/psutil-7.2.1-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:81442dac7abfc2f4f4385ea9e12ddf5a796721c0f6133260687fec5c3780fa49", size = 130132, upload-time = "2025-12-29T08:26:06.228Z" },
+ { url = "https://files.pythonhosted.org/packages/db/6d/ed44901e830739af5f72a85fa7ec5ff1edea7f81bfbf4875e409007149bd/psutil-7.2.1-cp313-cp313t-manylinux2010_x86_64.manylinux_2_12_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ea46c0d060491051d39f0d2cff4f98d5c72b288289f57a21556cc7d504db37fc", size = 180612, upload-time = "2025-12-29T08:26:08.276Z" },
+ { url = "https://files.pythonhosted.org/packages/c7/65/b628f8459bca4efbfae50d4bf3feaab803de9a160b9d5f3bd9295a33f0c2/psutil-7.2.1-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:35630d5af80d5d0d49cfc4d64c1c13838baf6717a13effb35869a5919b854cdf", size = 183201, upload-time = "2025-12-29T08:26:10.622Z" },
+ { url = "https://files.pythonhosted.org/packages/fb/23/851cadc9764edcc18f0effe7d0bf69f727d4cf2442deb4a9f78d4e4f30f2/psutil-7.2.1-cp313-cp313t-win_amd64.whl", hash = "sha256:923f8653416604e356073e6e0bccbe7c09990acef442def2f5640dd0faa9689f", size = 139081, upload-time = "2025-12-29T08:26:12.483Z" },
+ { url = "https://files.pythonhosted.org/packages/59/82/d63e8494ec5758029f31c6cb06d7d161175d8281e91d011a4a441c8a43b5/psutil-7.2.1-cp313-cp313t-win_arm64.whl", hash = "sha256:cfbe6b40ca48019a51827f20d830887b3107a74a79b01ceb8cc8de4ccb17b672", size = 134767, upload-time = "2025-12-29T08:26:14.528Z" },
+ { url = "https://files.pythonhosted.org/packages/05/c2/5fb764bd61e40e1fe756a44bd4c21827228394c17414ade348e28f83cd79/psutil-7.2.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:494c513ccc53225ae23eec7fe6e1482f1b8a44674241b54561f755a898650679", size = 129716, upload-time = "2025-12-29T08:26:16.017Z" },
+ { url = "https://files.pythonhosted.org/packages/c9/d2/935039c20e06f615d9ca6ca0ab756cf8408a19d298ffaa08666bc18dc805/psutil-7.2.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:3fce5f92c22b00cdefd1645aa58ab4877a01679e901555067b1bd77039aa589f", size = 130133, upload-time = "2025-12-29T08:26:18.009Z" },
+ { url = "https://files.pythonhosted.org/packages/77/69/19f1eb0e01d24c2b3eacbc2f78d3b5add8a89bf0bb69465bc8d563cc33de/psutil-7.2.1-cp314-cp314t-manylinux2010_x86_64.manylinux_2_12_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:93f3f7b0bb07711b49626e7940d6fe52aa9940ad86e8f7e74842e73189712129", size = 181518, upload-time = "2025-12-29T08:26:20.241Z" },
+ { url = "https://files.pythonhosted.org/packages/e1/6d/7e18b1b4fa13ad370787626c95887b027656ad4829c156bb6569d02f3262/psutil-7.2.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d34d2ca888208eea2b5c68186841336a7f5e0b990edec929be909353a202768a", size = 184348, upload-time = "2025-12-29T08:26:22.215Z" },
+ { url = "https://files.pythonhosted.org/packages/98/60/1672114392dd879586d60dd97896325df47d9a130ac7401318005aab28ec/psutil-7.2.1-cp314-cp314t-win_amd64.whl", hash = "sha256:2ceae842a78d1603753561132d5ad1b2f8a7979cb0c283f5b52fb4e6e14b1a79", size = 140400, upload-time = "2025-12-29T08:26:23.993Z" },
+ { url = "https://files.pythonhosted.org/packages/fb/7b/d0e9d4513c46e46897b46bcfc410d51fc65735837ea57a25170f298326e6/psutil-7.2.1-cp314-cp314t-win_arm64.whl", hash = "sha256:08a2f175e48a898c8eb8eace45ce01777f4785bc744c90aa2cc7f2fa5462a266", size = 135430, upload-time = "2025-12-29T08:26:25.999Z" },
+ { url = "https://files.pythonhosted.org/packages/c5/cf/5180eb8c8bdf6a503c6919f1da28328bd1e6b3b1b5b9d5b01ae64f019616/psutil-7.2.1-cp36-abi3-macosx_10_9_x86_64.whl", hash = "sha256:b2e953fcfaedcfbc952b44744f22d16575d3aa78eb4f51ae74165b4e96e55f42", size = 128137, upload-time = "2025-12-29T08:26:27.759Z" },
+ { url = "https://files.pythonhosted.org/packages/c5/2c/78e4a789306a92ade5000da4f5de3255202c534acdadc3aac7b5458fadef/psutil-7.2.1-cp36-abi3-macosx_11_0_arm64.whl", hash = "sha256:05cc68dbb8c174828624062e73078e7e35406f4ca2d0866c272c2410d8ef06d1", size = 128947, upload-time = "2025-12-29T08:26:29.548Z" },
+ { url = "https://files.pythonhosted.org/packages/29/f8/40e01c350ad9a2b3cb4e6adbcc8a83b17ee50dd5792102b6142385937db5/psutil-7.2.1-cp36-abi3-manylinux2010_x86_64.manylinux_2_12_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5e38404ca2bb30ed7267a46c02f06ff842e92da3bb8c5bfdadbd35a5722314d8", size = 154694, upload-time = "2025-12-29T08:26:32.147Z" },
+ { url = "https://files.pythonhosted.org/packages/06/e4/b751cdf839c011a9714a783f120e6a86b7494eb70044d7d81a25a5cd295f/psutil-7.2.1-cp36-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ab2b98c9fc19f13f59628d94df5cc4cc4844bc572467d113a8b517d634e362c6", size = 156136, upload-time = "2025-12-29T08:26:34.079Z" },
+ { url = "https://files.pythonhosted.org/packages/44/ad/bbf6595a8134ee1e94a4487af3f132cef7fce43aef4a93b49912a48c3af7/psutil-7.2.1-cp36-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:f78baafb38436d5a128f837fab2d92c276dfb48af01a240b861ae02b2413ada8", size = 148108, upload-time = "2025-12-29T08:26:36.225Z" },
+ { url = "https://files.pythonhosted.org/packages/1c/15/dd6fd869753ce82ff64dcbc18356093471a5a5adf4f77ed1f805d473d859/psutil-7.2.1-cp36-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:99a4cd17a5fdd1f3d014396502daa70b5ec21bf4ffe38393e152f8e449757d67", size = 147402, upload-time = "2025-12-29T08:26:39.21Z" },
+ { url = "https://files.pythonhosted.org/packages/34/68/d9317542e3f2b180c4306e3f45d3c922d7e86d8ce39f941bb9e2e9d8599e/psutil-7.2.1-cp37-abi3-win_amd64.whl", hash = "sha256:b1b0671619343aa71c20ff9767eced0483e4fc9e1f489d50923738caf6a03c17", size = 136938, upload-time = "2025-12-29T08:26:41.036Z" },
+ { url = "https://files.pythonhosted.org/packages/3e/73/2ce007f4198c80fcf2cb24c169884f833fe93fbc03d55d302627b094ee91/psutil-7.2.1-cp37-abi3-win_arm64.whl", hash = "sha256:0d67c1822c355aa6f7314d92018fb4268a76668a536f133599b91edd48759442", size = 133836, upload-time = "2025-12-29T08:26:43.086Z" },
]
[[package]]
@@ -4217,39 +4245,37 @@ wheels = [
[[package]]
name = "pynacl"
-version = "1.6.1"
+version = "1.6.2"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "cffi", marker = "platform_python_implementation != 'PyPy' or (extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" },
]
-sdist = { url = "https://files.pythonhosted.org/packages/b2/46/aeca065d227e2265125aea590c9c47fbf5786128c9400ee0eb7c88931f06/pynacl-1.6.1.tar.gz", hash = "sha256:8d361dac0309f2b6ad33b349a56cd163c98430d409fa503b10b70b3ad66eaa1d", size = 3506616, upload-time = "2025-11-10T16:02:13.195Z" }
-wheels = [
- { url = "https://files.pythonhosted.org/packages/75/d6/4b2dca33ed512de8f54e5c6074aa06eaeb225bfbcd9b16f33a414389d6bd/pynacl-1.6.1-cp314-cp314t-macosx_10_10_universal2.whl", hash = "sha256:7d7c09749450c385301a3c20dca967a525152ae4608c0a096fe8464bfc3df93d", size = 389109, upload-time = "2025-11-10T16:01:28.79Z" },
- { url = "https://files.pythonhosted.org/packages/3c/30/e8dbb8ff4fa2559bbbb2187ba0d0d7faf728d17cb8396ecf4a898b22d3da/pynacl-1.6.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:fc734c1696ffd49b40f7c1779c89ba908157c57345cf626be2e0719488a076d3", size = 808254, upload-time = "2025-11-10T16:01:37.839Z" },
- { url = "https://files.pythonhosted.org/packages/44/f9/f5449c652f31da00249638dbab065ad4969c635119094b79b17c3a4da2ab/pynacl-1.6.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:3cd787ec1f5c155dc8ecf39b1333cfef41415dc96d392f1ce288b4fe970df489", size = 1407365, upload-time = "2025-11-10T16:01:40.454Z" },
- { url = "https://files.pythonhosted.org/packages/eb/2f/9aa5605f473b712065c0a193ebf4ad4725d7a245533f0cd7e5dcdbc78f35/pynacl-1.6.1-cp314-cp314t-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6b35d93ab2df03ecb3aa506be0d3c73609a51449ae0855c2e89c7ed44abde40b", size = 843842, upload-time = "2025-11-10T16:01:30.524Z" },
- { url = "https://files.pythonhosted.org/packages/32/8d/748f0f6956e207453da8f5f21a70885fbbb2e060d5c9d78e0a4a06781451/pynacl-1.6.1-cp314-cp314t-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:dece79aecbb8f4640a1adbb81e4aa3bfb0e98e99834884a80eb3f33c7c30e708", size = 1445559, upload-time = "2025-11-10T16:01:33.663Z" },
- { url = "https://files.pythonhosted.org/packages/78/d0/2387f0dcb0e9816f38373999e48db4728ed724d31accdd4e737473319d35/pynacl-1.6.1-cp314-cp314t-manylinux_2_34_aarch64.whl", hash = "sha256:c2228054f04bf32d558fb89bb99f163a8197d5a9bf4efa13069a7fa8d4b93fc3", size = 825791, upload-time = "2025-11-10T16:01:34.823Z" },
- { url = "https://files.pythonhosted.org/packages/18/3d/ef6fb7eb072aaf15f280bc66f26ab97e7fc9efa50fb1927683013ef47473/pynacl-1.6.1-cp314-cp314t-manylinux_2_34_x86_64.whl", hash = "sha256:2b12f1b97346f177affcdfdc78875ff42637cb40dcf79484a97dae3448083a78", size = 1410843, upload-time = "2025-11-10T16:01:36.401Z" },
- { url = "https://files.pythonhosted.org/packages/e3/fb/23824a017526850ee7d8a1cc4cd1e3e5082800522c10832edbbca8619537/pynacl-1.6.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:e735c3a1bdfde3834503baf1a6d74d4a143920281cb724ba29fb84c9f49b9c48", size = 801140, upload-time = "2025-11-10T16:01:42.013Z" },
- { url = "https://files.pythonhosted.org/packages/5d/d1/ebc6b182cb98603a35635b727d62f094bc201bf610f97a3bb6357fe688d2/pynacl-1.6.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:3384a454adf5d716a9fadcb5eb2e3e72cd49302d1374a60edc531c9957a9b014", size = 1371966, upload-time = "2025-11-10T16:01:43.297Z" },
- { url = "https://files.pythonhosted.org/packages/64/f4/c9d7b6f02924b1f31db546c7bd2a83a2421c6b4a8e6a2e53425c9f2802e0/pynacl-1.6.1-cp314-cp314t-win32.whl", hash = "sha256:d8615ee34d01c8e0ab3f302dcdd7b32e2bcf698ba5f4809e7cc407c8cdea7717", size = 230482, upload-time = "2025-11-10T16:01:47.688Z" },
- { url = "https://files.pythonhosted.org/packages/c4/2c/942477957fba22da7bf99131850e5ebdff66623418ab48964e78a7a8293e/pynacl-1.6.1-cp314-cp314t-win_amd64.whl", hash = "sha256:5f5b35c1a266f8a9ad22525049280a600b19edd1f785bccd01ae838437dcf935", size = 243232, upload-time = "2025-11-10T16:01:45.208Z" },
- { url = "https://files.pythonhosted.org/packages/7a/0c/bdbc0d04a53b96a765ab03aa2cf9a76ad8653d70bf1665459b9a0dedaa1c/pynacl-1.6.1-cp314-cp314t-win_arm64.whl", hash = "sha256:d984c91fe3494793b2a1fb1e91429539c6c28e9ec8209d26d25041ec599ccf63", size = 187907, upload-time = "2025-11-10T16:01:46.328Z" },
- { url = "https://files.pythonhosted.org/packages/49/41/3cfb3b4f3519f6ff62bf71bf1722547644bcfb1b05b8fdbdc300249ba113/pynacl-1.6.1-cp38-abi3-macosx_10_10_universal2.whl", hash = "sha256:a6f9fd6d6639b1e81115c7f8ff16b8dedba1e8098d2756275d63d208b0e32021", size = 387591, upload-time = "2025-11-10T16:01:49.1Z" },
- { url = "https://files.pythonhosted.org/packages/18/21/b8a6563637799f617a3960f659513eccb3fcc655d5fc2be6e9dc6416826f/pynacl-1.6.1-cp38-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:e49a3f3d0da9f79c1bec2aa013261ab9fa651c7da045d376bd306cf7c1792993", size = 798866, upload-time = "2025-11-10T16:01:55.688Z" },
- { url = "https://files.pythonhosted.org/packages/e8/6c/dc38033bc3ea461e05ae8f15a81e0e67ab9a01861d352ae971c99de23e7c/pynacl-1.6.1-cp38-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:7713f8977b5d25f54a811ec9efa2738ac592e846dd6e8a4d3f7578346a841078", size = 1398001, upload-time = "2025-11-10T16:01:57.101Z" },
- { url = "https://files.pythonhosted.org/packages/9f/05/3ec0796a9917100a62c5073b20c4bce7bf0fea49e99b7906d1699cc7b61b/pynacl-1.6.1-cp38-abi3-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5a3becafc1ee2e5ea7f9abc642f56b82dcf5be69b961e782a96ea52b55d8a9fc", size = 834024, upload-time = "2025-11-10T16:01:50.228Z" },
- { url = "https://files.pythonhosted.org/packages/f0/b7/ae9982be0f344f58d9c64a1c25d1f0125c79201634efe3c87305ac7cb3e3/pynacl-1.6.1-cp38-abi3-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4ce50d19f1566c391fedc8dc2f2f5be265ae214112ebe55315e41d1f36a7f0a9", size = 1436766, upload-time = "2025-11-10T16:01:51.886Z" },
- { url = "https://files.pythonhosted.org/packages/b4/51/b2ccbf89cf3025a02e044dd68a365cad593ebf70f532299f2c047d2b7714/pynacl-1.6.1-cp38-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:543f869140f67d42b9b8d47f922552d7a967e6c116aad028c9bfc5f3f3b3a7b7", size = 817275, upload-time = "2025-11-10T16:01:53.351Z" },
- { url = "https://files.pythonhosted.org/packages/a8/6c/dd9ee8214edf63ac563b08a9b30f98d116942b621d39a751ac3256694536/pynacl-1.6.1-cp38-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:a2bb472458c7ca959aeeff8401b8efef329b0fc44a89d3775cffe8fad3398ad8", size = 1401891, upload-time = "2025-11-10T16:01:54.587Z" },
- { url = "https://files.pythonhosted.org/packages/0f/c1/97d3e1c83772d78ee1db3053fd674bc6c524afbace2bfe8d419fd55d7ed1/pynacl-1.6.1-cp38-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:3206fa98737fdc66d59b8782cecc3d37d30aeec4593d1c8c145825a345bba0f0", size = 772291, upload-time = "2025-11-10T16:01:58.111Z" },
- { url = "https://files.pythonhosted.org/packages/4d/ca/691ff2fe12f3bb3e43e8e8df4b806f6384593d427f635104d337b8e00291/pynacl-1.6.1-cp38-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:53543b4f3d8acb344f75fd4d49f75e6572fce139f4bfb4815a9282296ff9f4c0", size = 1370839, upload-time = "2025-11-10T16:01:59.252Z" },
- { url = "https://files.pythonhosted.org/packages/30/27/06fe5389d30391fce006442246062cc35773c84fbcad0209fbbf5e173734/pynacl-1.6.1-cp38-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:319de653ef84c4f04e045eb250e6101d23132372b0a61a7acf91bac0fda8e58c", size = 791371, upload-time = "2025-11-10T16:02:01.075Z" },
- { url = "https://files.pythonhosted.org/packages/2c/7a/e2bde8c9d39074a5aa046c7d7953401608d1f16f71e237f4bef3fb9d7e49/pynacl-1.6.1-cp38-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:262a8de6bba4aee8a66f5edf62c214b06647461c9b6b641f8cd0cb1e3b3196fe", size = 1363031, upload-time = "2025-11-10T16:02:02.656Z" },
- { url = "https://files.pythonhosted.org/packages/dd/b6/63fd77264dae1087770a1bb414bc604470f58fbc21d83822fc9c76248076/pynacl-1.6.1-cp38-abi3-win32.whl", hash = "sha256:9fd1a4eb03caf8a2fe27b515a998d26923adb9ddb68db78e35ca2875a3830dde", size = 226585, upload-time = "2025-11-10T16:02:07.116Z" },
- { url = "https://files.pythonhosted.org/packages/12/c8/b419180f3fdb72ab4d45e1d88580761c267c7ca6eda9a20dcbcba254efe6/pynacl-1.6.1-cp38-abi3-win_amd64.whl", hash = "sha256:a569a4069a7855f963940040f35e87d8bc084cb2d6347428d5ad20550a0a1a21", size = 238923, upload-time = "2025-11-10T16:02:04.401Z" },
- { url = "https://files.pythonhosted.org/packages/35/76/c34426d532e4dce7ff36e4d92cb20f4cbbd94b619964b93d24e8f5b5510f/pynacl-1.6.1-cp38-abi3-win_arm64.whl", hash = "sha256:5953e8b8cfadb10889a6e7bd0f53041a745d1b3d30111386a1bb37af171e6daf", size = 183970, upload-time = "2025-11-10T16:02:05.786Z" },
+sdist = { url = "https://files.pythonhosted.org/packages/d9/9a/4019b524b03a13438637b11538c82781a5eda427394380381af8f04f467a/pynacl-1.6.2.tar.gz", hash = "sha256:018494d6d696ae03c7e656e5e74cdfd8ea1326962cc401bcf018f1ed8436811c", size = 3511692, upload-time = "2026-01-01T17:48:10.851Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/4b/79/0e3c34dc3c4671f67d251c07aa8eb100916f250ee470df230b0ab89551b4/pynacl-1.6.2-cp314-cp314t-macosx_10_10_universal2.whl", hash = "sha256:622d7b07cc5c02c666795792931b50c91f3ce3c2649762efb1ef0d5684c81594", size = 390064, upload-time = "2026-01-01T17:31:57.264Z" },
+ { url = "https://files.pythonhosted.org/packages/eb/1c/23a26e931736e13b16483795c8a6b2f641bf6a3d5238c22b070a5112722c/pynacl-1.6.2-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:d071c6a9a4c94d79eb665db4ce5cedc537faf74f2355e4d502591d850d3913c0", size = 809370, upload-time = "2026-01-01T17:31:59.198Z" },
+ { url = "https://files.pythonhosted.org/packages/87/74/8d4b718f8a22aea9e8dcc8b95deb76d4aae380e2f5b570cc70b5fd0a852d/pynacl-1.6.2-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:fe9847ca47d287af41e82be1dd5e23023d3c31a951da134121ab02e42ac218c9", size = 1408304, upload-time = "2026-01-01T17:32:01.162Z" },
+ { url = "https://files.pythonhosted.org/packages/fd/73/be4fdd3a6a87fe8a4553380c2b47fbd1f7f58292eb820902f5c8ac7de7b0/pynacl-1.6.2-cp314-cp314t-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:04316d1fc625d860b6c162fff704eb8426b1a8bcd3abacea11142cbd99a6b574", size = 844871, upload-time = "2026-01-01T17:32:02.824Z" },
+ { url = "https://files.pythonhosted.org/packages/55/ad/6efc57ab75ee4422e96b5f2697d51bbcf6cdcc091e66310df91fbdc144a8/pynacl-1.6.2-cp314-cp314t-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:44081faff368d6c5553ccf55322ef2819abb40e25afaec7e740f159f74813634", size = 1446356, upload-time = "2026-01-01T17:32:04.452Z" },
+ { url = "https://files.pythonhosted.org/packages/78/b7/928ee9c4779caa0a915844311ab9fb5f99585621c5d6e4574538a17dca07/pynacl-1.6.2-cp314-cp314t-manylinux_2_34_aarch64.whl", hash = "sha256:a9f9932d8d2811ce1a8ffa79dcbdf3970e7355b5c8eb0c1a881a57e7f7d96e88", size = 826814, upload-time = "2026-01-01T17:32:06.078Z" },
+ { url = "https://files.pythonhosted.org/packages/f7/a9/1bdba746a2be20f8809fee75c10e3159d75864ef69c6b0dd168fc60e485d/pynacl-1.6.2-cp314-cp314t-manylinux_2_34_x86_64.whl", hash = "sha256:bc4a36b28dd72fb4845e5d8f9760610588a96d5a51f01d84d8c6ff9849968c14", size = 1411742, upload-time = "2026-01-01T17:32:07.651Z" },
+ { url = "https://files.pythonhosted.org/packages/f3/2f/5e7ea8d85f9f3ea5b6b87db1d8388daa3587eed181bdeb0306816fdbbe79/pynacl-1.6.2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:3bffb6d0f6becacb6526f8f42adfb5efb26337056ee0831fb9a7044d1a964444", size = 801714, upload-time = "2026-01-01T17:32:09.558Z" },
+ { url = "https://files.pythonhosted.org/packages/06/ea/43fe2f7eab5f200e40fb10d305bf6f87ea31b3bbc83443eac37cd34a9e1e/pynacl-1.6.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:2fef529ef3ee487ad8113d287a593fa26f48ee3620d92ecc6f1d09ea38e0709b", size = 1372257, upload-time = "2026-01-01T17:32:11.026Z" },
+ { url = "https://files.pythonhosted.org/packages/4d/54/c9ea116412788629b1347e415f72195c25eb2f3809b2d3e7b25f5c79f13a/pynacl-1.6.2-cp314-cp314t-win32.whl", hash = "sha256:a84bf1c20339d06dc0c85d9aea9637a24f718f375d861b2668b2f9f96fa51145", size = 231319, upload-time = "2026-01-01T17:32:12.46Z" },
+ { url = "https://files.pythonhosted.org/packages/ce/04/64e9d76646abac2dccf904fccba352a86e7d172647557f35b9fe2a5ee4a1/pynacl-1.6.2-cp314-cp314t-win_amd64.whl", hash = "sha256:320ef68a41c87547c91a8b58903c9caa641ab01e8512ce291085b5fe2fcb7590", size = 244044, upload-time = "2026-01-01T17:32:13.781Z" },
+ { url = "https://files.pythonhosted.org/packages/33/33/7873dc161c6a06f43cda13dec67b6fe152cb2f982581151956fa5e5cdb47/pynacl-1.6.2-cp314-cp314t-win_arm64.whl", hash = "sha256:d29bfe37e20e015a7d8b23cfc8bd6aa7909c92a1b8f41ee416bbb3e79ef182b2", size = 188740, upload-time = "2026-01-01T17:32:15.083Z" },
+ { url = "https://files.pythonhosted.org/packages/be/7b/4845bbf88e94586ec47a432da4e9107e3fc3ce37eb412b1398630a37f7dd/pynacl-1.6.2-cp38-abi3-macosx_10_10_universal2.whl", hash = "sha256:c949ea47e4206af7c8f604b8278093b674f7c79ed0d4719cc836902bf4517465", size = 388458, upload-time = "2026-01-01T17:32:16.829Z" },
+ { url = "https://files.pythonhosted.org/packages/1e/b4/e927e0653ba63b02a4ca5b4d852a8d1d678afbf69b3dbf9c4d0785ac905c/pynacl-1.6.2-cp38-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:8845c0631c0be43abdd865511c41eab235e0be69c81dc66a50911594198679b0", size = 800020, upload-time = "2026-01-01T17:32:18.34Z" },
+ { url = "https://files.pythonhosted.org/packages/7f/81/d60984052df5c97b1d24365bc1e30024379b42c4edcd79d2436b1b9806f2/pynacl-1.6.2-cp38-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:22de65bb9010a725b0dac248f353bb072969c94fa8d6b1f34b87d7953cf7bbe4", size = 1399174, upload-time = "2026-01-01T17:32:20.239Z" },
+ { url = "https://files.pythonhosted.org/packages/68/f7/322f2f9915c4ef27d140101dd0ed26b479f7e6f5f183590fd32dfc48c4d3/pynacl-1.6.2-cp38-abi3-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:46065496ab748469cdd999246d17e301b2c24ae2fdf739132e580a0e94c94a87", size = 835085, upload-time = "2026-01-01T17:32:22.24Z" },
+ { url = "https://files.pythonhosted.org/packages/3e/d0/f301f83ac8dbe53442c5a43f6a39016f94f754d7a9815a875b65e218a307/pynacl-1.6.2-cp38-abi3-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8a66d6fb6ae7661c58995f9c6435bda2b1e68b54b598a6a10247bfcdadac996c", size = 1437614, upload-time = "2026-01-01T17:32:23.766Z" },
+ { url = "https://files.pythonhosted.org/packages/c4/58/fc6e649762b029315325ace1a8c6be66125e42f67416d3dbd47b69563d61/pynacl-1.6.2-cp38-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:26bfcd00dcf2cf160f122186af731ae30ab120c18e8375684ec2670dccd28130", size = 818251, upload-time = "2026-01-01T17:32:25.69Z" },
+ { url = "https://files.pythonhosted.org/packages/c9/a8/b917096b1accc9acd878819a49d3d84875731a41eb665f6ebc826b1af99e/pynacl-1.6.2-cp38-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:c8a231e36ec2cab018c4ad4358c386e36eede0319a0c41fed24f840b1dac59f6", size = 1402859, upload-time = "2026-01-01T17:32:27.215Z" },
+ { url = "https://files.pythonhosted.org/packages/85/42/fe60b5f4473e12c72f977548e4028156f4d340b884c635ec6b063fe7e9a5/pynacl-1.6.2-cp38-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:68be3a09455743ff9505491220b64440ced8973fe930f270c8e07ccfa25b1f9e", size = 791926, upload-time = "2026-01-01T17:32:29.314Z" },
+ { url = "https://files.pythonhosted.org/packages/fa/f9/e40e318c604259301cc091a2a63f237d9e7b424c4851cafaea4ea7c4834e/pynacl-1.6.2-cp38-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:8b097553b380236d51ed11356c953bf8ce36a29a3e596e934ecabe76c985a577", size = 1363101, upload-time = "2026-01-01T17:32:31.263Z" },
+ { url = "https://files.pythonhosted.org/packages/48/47/e761c254f410c023a469284a9bc210933e18588ca87706ae93002c05114c/pynacl-1.6.2-cp38-abi3-win32.whl", hash = "sha256:5811c72b473b2f38f7e2a3dc4f8642e3a3e9b5e7317266e4ced1fba85cae41aa", size = 227421, upload-time = "2026-01-01T17:32:33.076Z" },
+ { url = "https://files.pythonhosted.org/packages/41/ad/334600e8cacc7d86587fe5f565480fde569dfb487389c8e1be56ac21d8ac/pynacl-1.6.2-cp38-abi3-win_amd64.whl", hash = "sha256:62985f233210dee6548c223301b6c25440852e13d59a8b81490203c3227c5ba0", size = 239754, upload-time = "2026-01-01T17:32:34.557Z" },
+ { url = "https://files.pythonhosted.org/packages/29/7d/5945b5af29534641820d3bd7b00962abbbdfee84ec7e19f0d5b3175f9a31/pynacl-1.6.2-cp38-abi3-win_arm64.whl", hash = "sha256:834a43af110f743a754448463e8fd61259cd4ab5bbedcf70f9dabad1d28a394c", size = 184801, upload-time = "2026-01-01T17:32:36.309Z" },
]
[[package]]
@@ -4270,12 +4296,12 @@ name = "pytest"
version = "8.3.5"
source = { registry = "https://pypi.org/simple" }
dependencies = [
- { name = "colorama", marker = "sys_platform == 'win32' or (extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" },
- { name = "exceptiongroup", marker = "python_full_version < '3.11' or (extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" },
+ { name = "colorama", marker = "sys_platform == 'win32'" },
+ { name = "exceptiongroup", marker = "python_full_version < '3.11'" },
{ name = "iniconfig" },
{ name = "packaging" },
{ name = "pluggy" },
- { name = "tomli", marker = "python_full_version < '3.11' or (extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" },
+ { name = "tomli", marker = "python_full_version < '3.11'" },
]
sdist = { url = "https://files.pythonhosted.org/packages/ae/3c/c9d525a414d506893f0cd8a8d0de7706446213181570cdbd766691164e40/pytest-8.3.5.tar.gz", hash = "sha256:f4efe70cc14e511565ac476b57c279e12a855b11f48f212af1080ef2263d3845", size = 1450891, upload-time = "2025-03-02T12:54:54.503Z" }
wheels = [
@@ -4348,24 +4374,24 @@ wheels = [
[[package]]
name = "python-gitlab"
-version = "7.0.0"
+version = "7.1.0"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "requests" },
{ name = "requests-toolbelt" },
]
-sdist = { url = "https://files.pythonhosted.org/packages/5e/c4/0b613303b4f0fcda69b3d2e03d0a1fb1b6b079a7c7832e03a8d92461e9fe/python_gitlab-7.0.0.tar.gz", hash = "sha256:e4d934430f64efc09e6208b782c61cc0a3389527765e03ffbef17f4323dce441", size = 400568, upload-time = "2025-10-29T15:06:02.069Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/31/98/0b5d0a0367b90aec818298390b60ae65e6a08989cf5140271d0ee0206882/python_gitlab-7.1.0.tar.gz", hash = "sha256:1c34da3de40ad21675d788136f73d20a60649513e692f52c5a9720434db97c46", size = 401058, upload-time = "2025-12-28T01:27:01.369Z" }
wheels = [
- { url = "https://files.pythonhosted.org/packages/4f/9e/811edc46a15f8deb828cba7ef8aab3451dc11ca72d033f3df72a5af865d9/python_gitlab-7.0.0-py3-none-any.whl", hash = "sha256:712a6c8c5e79e7e66f6dabb25d8fe7831a6b238d4a5132f8231df6b3b890ceff", size = 144415, upload-time = "2025-10-29T15:06:00.232Z" },
+ { url = "https://files.pythonhosted.org/packages/14/44/70fa1e395731b6a4b1f249d5f7326f3bb6281e2cf94d6535f679239f4b93/python_gitlab-7.1.0-py3-none-any.whl", hash = "sha256:8e42030cf27674e7ec9ea1f6d2fedcaaef0a6210f5fa22c80721abaa3a4fec90", size = 144441, upload-time = "2025-12-28T01:26:59.726Z" },
]
[[package]]
name = "python-multipart"
-version = "0.0.20"
+version = "0.0.21"
source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/f3/87/f44d7c9f274c7ee665a29b885ec97089ec5dc034c7f3fafa03da9e39a09e/python_multipart-0.0.20.tar.gz", hash = "sha256:8dd0cab45b8e23064ae09147625994d090fa46f5b0d1e13af944c331a7fa9d13", size = 37158, upload-time = "2024-12-16T19:45:46.972Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/78/96/804520d0850c7db98e5ccb70282e29208723f0964e88ffd9d0da2f52ea09/python_multipart-0.0.21.tar.gz", hash = "sha256:7137ebd4d3bbf70ea1622998f902b97a29434a9e8dc40eb203bbcf7c2a2cba92", size = 37196, upload-time = "2025-12-17T09:24:22.446Z" }
wheels = [
- { url = "https://files.pythonhosted.org/packages/45/58/38b5afbc1a800eeea951b9285d3912613f2603bdf897a4ab0f4bd7f405fc/python_multipart-0.0.20-py3-none-any.whl", hash = "sha256:8a62d3a8335e06589fe01f2a3e178cdcc632f3fbe0d492ad9ee0ec35aab1f104", size = 24546, upload-time = "2024-12-16T19:45:44.423Z" },
+ { url = "https://files.pythonhosted.org/packages/aa/76/03af049af4dcee5d27442f71b6924f01f3efb5d2bd34f23fcd563f2cc5f5/python_multipart-0.0.21-py3-none-any.whl", hash = "sha256:cf7a6713e01c87aa35387f4774e812c4361150938d20d232800f75ffcf266090", size = 24541, upload-time = "2025-12-17T09:24:21.153Z" },
]
[[package]]
@@ -4489,7 +4515,7 @@ wheels = [
[[package]]
name = "ray"
-version = "2.51.2"
+version = "2.53.0"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "click" },
@@ -4502,21 +4528,21 @@ dependencies = [
{ name = "requests" },
]
wheels = [
- { url = "https://files.pythonhosted.org/packages/6a/ad/59270b7d1003152ef231b65c38c3721066fc970b2a2475314e7c8ee81990/ray-2.51.2-cp310-cp310-macosx_12_0_arm64.whl", hash = "sha256:eb9b995de9ba3110373f00e77dda86f6a55a80a58114b1eae5e6daf1f5697338", size = 68040029, upload-time = "2025-11-29T00:28:25.435Z" },
- { url = "https://files.pythonhosted.org/packages/bc/bf/43442642cf4f29ac9ef721d9b184512ed84436e65d8244f1867e31b1ecdb/ray-2.51.2-cp310-cp310-manylinux2014_aarch64.whl", hash = "sha256:983adacd9cecf2f74f7915560036f14c5d4fabdf6f65d959debc92820373729d", size = 70344819, upload-time = "2025-11-29T00:28:32.157Z" },
- { url = "https://files.pythonhosted.org/packages/57/78/79d8b884492b28c5d9ec99fd8750baaf30e311e79013e9f137dafee3b246/ray-2.51.2-cp310-cp310-manylinux2014_x86_64.whl", hash = "sha256:572d8f7e95e506d6264c7b916fe70e765e3367d5f1bc9755bc1d73c8607a2ac6", size = 71172369, upload-time = "2025-11-29T00:28:38.511Z" },
- { url = "https://files.pythonhosted.org/packages/6a/26/632c509eda0742f6c9e8c876ebe308cfdefdd2cdd414fcb4e65c37490995/ray-2.51.2-cp310-cp310-win_amd64.whl", hash = "sha256:05d1cdd0352f9da10555899cb6212ac9a2e783b05c20c2989cae09531c1b1969", size = 26696512, upload-time = "2025-11-29T00:28:42.955Z" },
- { url = "https://files.pythonhosted.org/packages/6d/fa/4ee6a516d9de9d5fa7ecd0e59888c9ab1a2bedaec06fe9c6b91d0f9523b2/ray-2.51.2-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:26100d25b0ca5162e7404d57247ad697514709c6f41db7efb3d312d78a5ef292", size = 68044847, upload-time = "2025-11-29T00:28:47.902Z" },
- { url = "https://files.pythonhosted.org/packages/92/ca/06b1b761e8c4398c2818f0ac04e14c2f2937fa79bf9be6ffc74d785641fb/ray-2.51.2-cp311-cp311-manylinux2014_aarch64.whl", hash = "sha256:1102471b4edb08605001be781f094c2291805d8e4a118ad8b59b833b12d4f13f", size = 70464861, upload-time = "2025-11-29T00:28:53.591Z" },
- { url = "https://files.pythonhosted.org/packages/7c/b0/7dda0bf542f3cf08fae67c57ec61422d4f8b3d0342d0d03057eefb93886e/ray-2.51.2-cp311-cp311-manylinux2014_x86_64.whl", hash = "sha256:ad6aafbb7f67d1edbe3cad72b9e33ee99b0ed31ca7210ee8c6af9db1d1c4d850", size = 71286437, upload-time = "2025-11-29T00:28:59.26Z" },
- { url = "https://files.pythonhosted.org/packages/57/c9/31289a53bf4418b9fe71be8f7780ee520ef5f76fb5a5cdd5dcff9e41fb0b/ray-2.51.2-cp311-cp311-win_amd64.whl", hash = "sha256:a48e3871cc2b526bca7de84527fdf56875115829fab518cc938dd4c64e0174b9", size = 26692167, upload-time = "2025-11-29T00:29:03.786Z" },
- { url = "https://files.pythonhosted.org/packages/70/54/66fcfebd26c9747d908e2ac24f3a8a5502e84f19ea1e7a9b7f4d4a12bc34/ray-2.51.2-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:461b0e711f73cebc68128bca7202bef8db2c0e14dc6d49140f96549e5e752eb1", size = 68030141, upload-time = "2025-11-29T00:29:08.67Z" },
- { url = "https://files.pythonhosted.org/packages/0e/9e/7add3c78a5a3d05f9c702d247da83a8a3e30d57eae153985f48ec3309c82/ray-2.51.2-cp312-cp312-manylinux2014_aarch64.whl", hash = "sha256:5c97f29574072e3568a2714a84e6948fb457ce09eefd251c919221584b2d458d", size = 70506728, upload-time = "2025-11-29T00:29:14.051Z" },
- { url = "https://files.pythonhosted.org/packages/b3/8e/5d1325619399d7eb9563e2f883f8e782fb26b39a122d6d629e54c8989a5a/ray-2.51.2-cp312-cp312-manylinux2014_x86_64.whl", hash = "sha256:7b2a842744a1d4b47af8f3c0665a319736139518dd2e26fb9e18114281d8f9ea", size = 71359570, upload-time = "2025-11-29T00:29:19.508Z" },
- { url = "https://files.pythonhosted.org/packages/ba/96/ec1ee03fb1731d9e09d94d7ba6d9e47fce886d7cc79aac47e8422fe9c528/ray-2.51.2-cp312-cp312-win_amd64.whl", hash = "sha256:6b04ca7dccf540da2ab07fd7073009dfe04d9d084d705e337572272fa3e56485", size = 26675734, upload-time = "2025-11-29T00:29:24.27Z" },
- { url = "https://files.pythonhosted.org/packages/70/89/255ac2a70928a1d439c98fca9f3437cabbbebd3ac767523df608cce39197/ray-2.51.2-cp313-cp313-macosx_12_0_arm64.whl", hash = "sha256:c9ed290667868c809eb467ad8830d887fdce10dac2c674b3d43d3b3b5f9c7b07", size = 67975149, upload-time = "2025-11-29T00:29:28.995Z" },
- { url = "https://files.pythonhosted.org/packages/d3/05/1e3bb04e263a2bc1eacd762b37a0013d18f76341de0a7199d84a5a00b372/ray-2.51.2-cp313-cp313-manylinux2014_aarch64.whl", hash = "sha256:554bd393e97bed9dfa5f73f47e4fbf42aa35d81b1228081aa93ccb7cdd5d4b34", size = 70414911, upload-time = "2025-11-29T00:29:34.286Z" },
- { url = "https://files.pythonhosted.org/packages/c4/85/f6994a74cf5e6fa6ebc959c27ff6f1f5352b78e71b947b4b302c6bb0a203/ray-2.51.2-cp313-cp313-manylinux2014_x86_64.whl", hash = "sha256:e3bf004ed23971ec5d324ed9748aed23f6645d56696a44cdbe35d331f66c4619", size = 71275062, upload-time = "2025-11-29T00:29:39.379Z" },
+ { url = "https://files.pythonhosted.org/packages/2f/99/21986c7f8135dafbf7c49229c52faaa9d2d365db7d86fffe978dde8ee967/ray-2.53.0-cp310-cp310-macosx_12_0_arm64.whl", hash = "sha256:4db914a0a6dd608fa49c066929a1282745a2dbd73caee67d7b80fe684ca65bdd", size = 69473649, upload-time = "2025-12-20T16:05:40.58Z" },
+ { url = "https://files.pythonhosted.org/packages/70/d9/58b5426a3f11993851db3c93841358cebdddd948153481d355b720f31f9d/ray-2.53.0-cp310-cp310-manylinux2014_aarch64.whl", hash = "sha256:4108280d8a1cb90d7d68e5c954c35e63b8bb9a4ba15f88c5e7da0e2025647712", size = 71342662, upload-time = "2025-12-20T16:05:46.936Z" },
+ { url = "https://files.pythonhosted.org/packages/c5/05/4aa32370b313481c2d1d41cb53ec786daebdb2ef665b01ef2ac43d9cf457/ray-2.53.0-cp310-cp310-manylinux2014_x86_64.whl", hash = "sha256:4dbb5fce1364763f29741055f50abe33cf726397141f9cc0e845dd3cc963e455", size = 72188620, upload-time = "2025-12-20T16:05:52.817Z" },
+ { url = "https://files.pythonhosted.org/packages/f7/c6/21efe5886898421df20078a333b0984eade7d7aa4bdc68a336f0c66db27e/ray-2.53.0-cp310-cp310-win_amd64.whl", hash = "sha256:90faf630d20b6abf3135997fb3edb5842134aff92e04ee709865db04816d97ef", size = 27200553, upload-time = "2025-12-20T16:05:57.655Z" },
+ { url = "https://files.pythonhosted.org/packages/bf/64/d5c29a4b014d8b9a624203a88b67630072c1d6960425dbf7a1f0fa5d6b74/ray-2.53.0-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:bd3ec4c342776ddac23ae2b108c64f5939f417ccc4875900d586c7c978463269", size = 69479296, upload-time = "2025-12-20T16:06:05.111Z" },
+ { url = "https://files.pythonhosted.org/packages/c6/41/9e19d1e5d9458a5ba157c36642e2874bcb22fddbd7c1e77b668e5afc3f3d/ray-2.53.0-cp311-cp311-manylinux2014_aarch64.whl", hash = "sha256:a0bbb98b0b0f25a3ee075ca10171e1260e70b6bc690cd509ecd7ce1228af854d", size = 71463449, upload-time = "2025-12-20T16:06:10.983Z" },
+ { url = "https://files.pythonhosted.org/packages/63/de/58c19906b0dd16ea06b4f2465b7327f5f180e6b6e1c8c9b610d7c589ea5f/ray-2.53.0-cp311-cp311-manylinux2014_x86_64.whl", hash = "sha256:eb000c17f7301071fdd15c44c4cd3ac0f7953bb4c7c227e61719fe7048195bcd", size = 72305102, upload-time = "2025-12-20T16:06:17.989Z" },
+ { url = "https://files.pythonhosted.org/packages/b1/43/72cc1cfe17d26abe62a793eab10445f9546dce24192b85a6cd0cdc47ed86/ray-2.53.0-cp311-cp311-win_amd64.whl", hash = "sha256:4a1bb3fe09ab4cd0d16ddc96b9f60c9ed83b3f93b87aa8506e0d3b746fd4e825", size = 27194174, upload-time = "2025-12-20T16:06:23.042Z" },
+ { url = "https://files.pythonhosted.org/packages/b2/44/562718a634e63e8ef7985285288a167d4af62bc2a7decce3300cf937776a/ray-2.53.0-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:d8b95d047d947493803fb8417aea31225dcacdab15afdc75b8a238901949d457", size = 69463763, upload-time = "2025-12-20T16:06:28.685Z" },
+ { url = "https://files.pythonhosted.org/packages/38/68/8e59b8413f3751fe7ce8b98ee8787d13964b47a4043587950790a9dd2151/ray-2.53.0-cp312-cp312-manylinux2014_aarch64.whl", hash = "sha256:65e2ce58d3dc6baa3cf45824d889c1968ebde565ee54dfd80a98af8f31af8e4a", size = 71504450, upload-time = "2025-12-20T16:06:34.922Z" },
+ { url = "https://files.pythonhosted.org/packages/2a/db/978a50d264565ca42e2a4bf115ec9a1f04f19ca5e620e6aa2f280747b644/ray-2.53.0-cp312-cp312-manylinux2014_x86_64.whl", hash = "sha256:14f46363e9b4cf0c1c8b4d8623ec337c5bd408377831b5e5b50067930137bbca", size = 72370424, upload-time = "2025-12-20T16:06:40.821Z" },
+ { url = "https://files.pythonhosted.org/packages/8d/6c/bba6f22a9d83ee8f236000ba315f0c197bdc79888b4fa42fd762f729cbbd/ray-2.53.0-cp312-cp312-win_amd64.whl", hash = "sha256:b828c147f9ff2f277b1d254e4fe9a746fdfaee7e313a93a97c7edf4dae9b81a4", size = 27178106, upload-time = "2025-12-20T16:06:45.594Z" },
+ { url = "https://files.pythonhosted.org/packages/3d/38/450cf9cf3c490fa4cc6d470597f819444da60f85579d2b34b95ee79fcb6f/ray-2.53.0-cp313-cp313-macosx_12_0_arm64.whl", hash = "sha256:85b472ab6fb8f1189f8cef81913fd91b24dd69b3fa7dcca7e144827bd924f6c0", size = 69409819, upload-time = "2025-12-20T16:06:50.668Z" },
+ { url = "https://files.pythonhosted.org/packages/71/5e/d452970b07174d5e4f8688abae889d01321b51ced827db1f1d1cb7d56d44/ray-2.53.0-cp313-cp313-manylinux2014_aarch64.whl", hash = "sha256:7196e5358dfcc8211be864f45e6dfe4827202df294af3c7a76ff8fbc080e0522", size = 71409529, upload-time = "2025-12-20T16:06:56.2Z" },
+ { url = "https://files.pythonhosted.org/packages/cb/84/50b317a125617a638a64694c12f56183edd5df01828a35fa4c55c7b13c66/ray-2.53.0-cp313-cp313-manylinux2014_x86_64.whl", hash = "sha256:73dbbaa7962a7f5e38aa8cf9483e0e9817205e989aa3dc859c738c2af1ae01df", size = 72283961, upload-time = "2025-12-20T16:07:05.831Z" },
]
[[package]]
@@ -4680,13 +4706,25 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/25/7a/b0178788f8dc6cafce37a212c99565fa1fe7872c70c6c9c1e1a372d9d88f/rich-14.2.0-py3-none-any.whl", hash = "sha256:76bc51fe2e57d2b1be1f96c524b890b816e334ab4c1e45888799bfaab0021edd", size = 243393, upload-time = "2025-10-09T14:16:51.245Z" },
]
+[[package]]
+name = "roman-numerals"
+version = "4.1.0"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/ae/f9/41dc953bbeb056c17d5f7a519f50fdf010bd0553be2d630bc69d1e022703/roman_numerals-4.1.0.tar.gz", hash = "sha256:1af8b147eb1405d5839e78aeb93131690495fe9da5c91856cb33ad55a7f1e5b2", size = 9077, upload-time = "2025-12-17T18:25:34.381Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/04/54/6f679c435d28e0a568d8e8a7c0a93a09010818634c3c3907fc98d8983770/roman_numerals-4.1.0-py3-none-any.whl", hash = "sha256:647ba99caddc2cc1e55a51e4360689115551bf4476d90e8162cf8c345fe233c7", size = 7676, upload-time = "2025-12-17T18:25:33.098Z" },
+]
+
[[package]]
name = "roman-numerals-py"
-version = "3.1.0"
+version = "4.1.0"
source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/30/76/48fd56d17c5bdbdf65609abbc67288728a98ed4c02919428d4f52d23b24b/roman_numerals_py-3.1.0.tar.gz", hash = "sha256:be4bf804f083a4ce001b5eb7e3c0862479d10f94c936f6c4e5f250aa5ff5bd2d", size = 9017, upload-time = "2025-02-22T07:34:54.333Z" }
+dependencies = [
+ { name = "roman-numerals", marker = "python_full_version >= '3.11' or (extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/cb/b5/de96fca640f4f656eb79bbee0e79aeec52e3e0e359f8a3e6a0d366378b64/roman_numerals_py-4.1.0.tar.gz", hash = "sha256:f5d7b2b4ca52dd855ef7ab8eb3590f428c0b1ea480736ce32b01fef2a5f8daf9", size = 4274, upload-time = "2025-12-17T18:25:41.153Z" }
wheels = [
- { url = "https://files.pythonhosted.org/packages/53/97/d2cbbaa10c9b826af0e10fdf836e1bf344d9f0abb873ebc34d1f49642d3f/roman_numerals_py-3.1.0-py3-none-any.whl", hash = "sha256:9da2ad2fb670bcf24e81070ceb3be72f6c11c440d73bd579fbeca1e9f330954c", size = 7742, upload-time = "2025-02-22T07:34:52.422Z" },
+ { url = "https://files.pythonhosted.org/packages/27/2c/daca29684cbe9fd4bc711f8246da3c10adca1ccc4d24436b17572eb2590e/roman_numerals_py-4.1.0-py3-none-any.whl", hash = "sha256:553114c1167141c1283a51743759723ecd05604a1b6b507225e91dc1a6df0780", size = 4547, upload-time = "2025-12-17T18:25:40.136Z" },
]
[[package]]
@@ -4951,7 +4989,7 @@ resolution-markers = [
"python_full_version == '3.11.*' and sys_platform != 'linux'",
]
dependencies = [
- { name = "numpy", version = "2.3.5", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" },
+ { name = "numpy", version = "2.4.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" },
]
sdist = { url = "https://files.pythonhosted.org/packages/0a/ca/d8ace4f98322d01abcd52d381134344bf7b431eba7ed8b42bdea5a3c2ac9/scipy-1.16.3.tar.gz", hash = "sha256:01e87659402762f43bd2fee13370553a17ada367d42e7487800bf2916535aecb", size = 30597883, upload-time = "2025-10-28T17:38:54.068Z" }
wheels = [
@@ -5083,15 +5121,15 @@ wheels = [
[[package]]
name = "sentry-sdk"
-version = "2.47.0"
+version = "2.48.0"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "certifi" },
{ name = "urllib3" },
]
-sdist = { url = "https://files.pythonhosted.org/packages/4a/2a/d225cbf87b6c8ecce5664db7bcecb82c317e448e3b24a2dcdaacb18ca9a7/sentry_sdk-2.47.0.tar.gz", hash = "sha256:8218891d5e41b4ea8d61d2aed62ed10c80e39d9f2959d6f939efbf056857e050", size = 381895, upload-time = "2025-12-03T14:06:36.846Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/40/f0/0e9dc590513d5e742d7799e2038df3a05167cba084c6ca4f3cdd75b55164/sentry_sdk-2.48.0.tar.gz", hash = "sha256:5213190977ff7fdff8a58b722fb807f8d5524a80488626ebeda1b5676c0c1473", size = 384828, upload-time = "2025-12-16T14:55:41.722Z" }
wheels = [
- { url = "https://files.pythonhosted.org/packages/bd/ac/d6286ea0d49e7b58847faf67b00e56bb4ba3d525281e2ac306e1f1f353da/sentry_sdk-2.47.0-py2.py3-none-any.whl", hash = "sha256:d72f8c61025b7d1d9e52510d03a6247b280094a327dd900d987717a4fce93412", size = 411088, upload-time = "2025-12-03T14:06:35.374Z" },
+ { url = "https://files.pythonhosted.org/packages/4d/19/8d77f9992e5cbfcaa9133c3bf63b4fbbb051248802e1e803fed5c552fbb2/sentry_sdk-2.48.0-py2.py3-none-any.whl", hash = "sha256:6b12ac256769d41825d9b7518444e57fa35b5642df4c7c5e322af4d2c8721172", size = 414555, upload-time = "2025-12-16T14:55:40.152Z" },
]
[[package]]
@@ -5173,7 +5211,7 @@ source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "cffi" },
{ name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version < '3.11' and extra == 'extra-13-megatron-core-dev') or (python_full_version < '3.11' and extra == 'extra-13-megatron-core-lts') or (extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" },
- { name = "numpy", version = "2.3.5", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version >= '3.11' and extra == 'extra-13-megatron-core-dev') or (python_full_version >= '3.11' and extra == 'extra-13-megatron-core-lts') or (extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" },
+ { name = "numpy", version = "2.4.0", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version >= '3.11' and extra == 'extra-13-megatron-core-dev') or (python_full_version >= '3.11' and extra == 'extra-13-megatron-core-lts') or (extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" },
]
sdist = { url = "https://files.pythonhosted.org/packages/e1/41/9b873a8c055582859b239be17902a85339bec6a30ad162f98c9b0288a2cc/soundfile-0.13.1.tar.gz", hash = "sha256:b2c68dab1e30297317080a5b43df57e302584c49e2942defdde0acccc53f0e5b", size = 46156, upload-time = "2025-01-25T09:17:04.831Z" }
wheels = [
@@ -5188,11 +5226,11 @@ wheels = [
[[package]]
name = "soupsieve"
-version = "2.8"
+version = "2.8.1"
source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/6d/e6/21ccce3262dd4889aa3332e5a119a3491a95e8f60939870a3a035aabac0d/soupsieve-2.8.tar.gz", hash = "sha256:e2dd4a40a628cb5f28f6d4b0db8800b8f581b65bb380b97de22ba5ca8d72572f", size = 103472, upload-time = "2025-08-27T15:39:51.78Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/89/23/adf3796d740536d63a6fbda113d07e60c734b6ed5d3058d1e47fc0495e47/soupsieve-2.8.1.tar.gz", hash = "sha256:4cf733bc50fa805f5df4b8ef4740fc0e0fa6218cf3006269afd3f9d6d80fd350", size = 117856, upload-time = "2025-12-18T13:50:34.655Z" }
wheels = [
- { url = "https://files.pythonhosted.org/packages/14/a0/bb38d3b76b8cae341dad93a2dd83ab7462e6dbcdd84d43f54ee60a8dc167/soupsieve-2.8-py3-none-any.whl", hash = "sha256:0cc76456a30e20f5d7f2e14a98a4ae2ee4e5abdc7c5ea0aafe795f344bc7984c", size = 36679, upload-time = "2025-08-27T15:39:50.179Z" },
+ { url = "https://files.pythonhosted.org/packages/48/f3/b67d6ea49ca9154453b6d70b34ea22f3996b9fa55da105a79d8732227adc/soupsieve-2.8.1-py3-none-any.whl", hash = "sha256:a11fe2a6f3d76ab3cf2de04eb339c1be5b506a8a47f2ceb6d139803177f85434", size = 36710, upload-time = "2025-12-18T13:50:33.267Z" },
]
[[package]]
@@ -5446,7 +5484,7 @@ dependencies = [
{ name = "grpcio" },
{ name = "markdown" },
{ name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11' or (extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" },
- { name = "numpy", version = "2.3.5", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11' or (extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" },
+ { name = "numpy", version = "2.4.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11' or (extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" },
{ name = "packaging" },
{ name = "pillow" },
{ name = "protobuf" },
@@ -5506,7 +5544,7 @@ wheels = [
[[package]]
name = "tensorstore"
-version = "0.1.79"
+version = "0.1.80"
source = { registry = "https://pypi.org/simple" }
resolution-markers = [
"python_full_version >= '3.14' and sys_platform == 'linux'",
@@ -5520,30 +5558,34 @@ resolution-markers = [
]
dependencies = [
{ name = "ml-dtypes", marker = "python_full_version >= '3.11'" },
- { name = "numpy", version = "2.3.5", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" },
-]
-sdist = { url = "https://files.pythonhosted.org/packages/26/2c/50ab489a0862ca88d2d766130a6fec45ccd5174f0e04081d8b7b07a8aedd/tensorstore-0.1.79.tar.gz", hash = "sha256:8dad44a8a7f2952a5d0030a8bd868b3cfdff048bd40ab53e7226f3d8b0881c5e", size = 7075782, upload-time = "2025-11-11T22:05:23.824Z" }
-wheels = [
- { url = "https://files.pythonhosted.org/packages/68/a9/1695d7ea197c4568c2f02f34b203eef702ec8080422331f00a65c6fb2a37/tensorstore-0.1.79-cp311-cp311-macosx_10_14_x86_64.whl", hash = "sha256:11a2c62694ea9c21770bc5a09938d3d15c4b9662b738ae6e1e513c26ed96251a", size = 16466511, upload-time = "2025-11-11T22:04:18.614Z" },
- { url = "https://files.pythonhosted.org/packages/db/0e/5ce8a615c7f9ad7cf8ed4ac6e182fe0ef46fd06fef89757e49ba84a6ba9e/tensorstore-0.1.79-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:5e152d334bf34fbabdfe8e5bc35b87d1f9947065924ff83c29e659308b36e948", size = 14499810, upload-time = "2025-11-11T22:04:21.725Z" },
- { url = "https://files.pythonhosted.org/packages/c0/29/2cb9552138fe84ab29421489121350e4af0502eafff31ccd9017490be0d8/tensorstore-0.1.79-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c4230b8fd29795e88e441f749d881973eca8dadf33c5262b367839fb8891f79b", size = 18937510, upload-time = "2025-11-11T22:04:24.221Z" },
- { url = "https://files.pythonhosted.org/packages/42/70/d2a672a93faebdd176cd8541405cd5614b14d3d8dc812fbeaf2cf46d390a/tensorstore-0.1.79-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:83072ee0e551d6dca582e154b64c8b8066d276ec0759784e3149c28212a61f18", size = 20910324, upload-time = "2025-11-11T22:04:26.769Z" },
- { url = "https://files.pythonhosted.org/packages/91/d5/7958cbfb614c4ffa5070ae9575874d46937067c0d81a7739e67fb1d62de5/tensorstore-0.1.79-cp311-cp311-win_amd64.whl", hash = "sha256:6c98c6b74c00e00eba7969292144e471d5c45d67088f0dc08e3a4c60a15ee191", size = 13206191, upload-time = "2025-11-11T22:04:29.254Z" },
- { url = "https://files.pythonhosted.org/packages/f1/a2/a77be16b4a882ace36da0748305795f35306bdad568472f208bd89b96b9d/tensorstore-0.1.79-cp312-cp312-macosx_10_14_x86_64.whl", hash = "sha256:71aa9b45436d888c37b965f7b71195916d15438119b7dccb66a3b0776bfba367", size = 16485740, upload-time = "2025-11-11T22:04:33.478Z" },
- { url = "https://files.pythonhosted.org/packages/7a/e4/7fe268ec41aa70b71a1c56b1ec83346fbcbf12f4bfbefc79d14fb9c03408/tensorstore-0.1.79-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:108c0e867aa2c87d4982cc6325a2de0c4f5bd63c2bea18adb193a370c40594ce", size = 14508736, upload-time = "2025-11-11T22:04:38.613Z" },
- { url = "https://files.pythonhosted.org/packages/5a/f1/b1248dae02598ce534834413e841f915a32ab185c36ecd05e4c67bdc8d19/tensorstore-0.1.79-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:debd435042c00be68ba1fb3cf59325a7babb3f4a3cf4744c87dde346802cbbb4", size = 18947817, upload-time = "2025-11-11T22:04:40.768Z" },
- { url = "https://files.pythonhosted.org/packages/87/4a/60e234147570e21bbab4ac70ab79dd794a5ef9a4945d36c34c1914a73205/tensorstore-0.1.79-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:608f7178ec6e4e4a3c26545b0a44f44bf83438d04bf2d960cd0e7699eaa99ef6", size = 20929832, upload-time = "2025-11-11T22:04:43.613Z" },
- { url = "https://files.pythonhosted.org/packages/f8/48/0531868bce12a2f520002e810d4200ec6f01ba33a2f27b6bd7289fbc197b/tensorstore-0.1.79-cp312-cp312-win_amd64.whl", hash = "sha256:a071c6c255b7e412957a6aa563bc4250242c7894edad06ae6358e3d30b7d88ce", size = 13211970, upload-time = "2025-11-11T22:04:46.179Z" },
- { url = "https://files.pythonhosted.org/packages/fa/0b/54a44e55836d8e8f576343134c0e3db71c6c837d39a0ac44699aba5b01df/tensorstore-0.1.79-cp313-cp313-macosx_10_14_x86_64.whl", hash = "sha256:1e8e2d098829919caac6a62cf568902e34789069ceddb28497d6e36ebcb95c0b", size = 16485855, upload-time = "2025-11-11T22:04:48.734Z" },
- { url = "https://files.pythonhosted.org/packages/04/59/cadb9a45896d480882476df4759cda1659c70669aff87a4d5a4a07ded084/tensorstore-0.1.79-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:29cf4336153af136ac8ac528e2ed46df19367edae7e14e37bca1a8b7c4848ef2", size = 14508277, upload-time = "2025-11-11T22:04:50.775Z" },
- { url = "https://files.pythonhosted.org/packages/e6/cb/3647bdd03c7692882ebc10c19df9ede49f290c216b2906f785edbdb53ef1/tensorstore-0.1.79-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:94d8fc9df1721b0287046aca7209fd5040889cad4202e7b73a1fdb77cd9b71c6", size = 18949307, upload-time = "2025-11-11T22:04:53.145Z" },
- { url = "https://files.pythonhosted.org/packages/20/a0/f91ac492cf2ee9f7541aefaaed4ad1258e73e33f3cd3e06cdce5859431db/tensorstore-0.1.79-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c9f2dc3342e4686af98f6e259dc9fb377f1bf657b649c247bf6647bbe4f98090", size = 20930427, upload-time = "2025-11-11T22:04:55.353Z" },
- { url = "https://files.pythonhosted.org/packages/69/a6/752fd11747eb9fead715b02d389da7fb180a56172b885de0b48b20237d1e/tensorstore-0.1.79-cp313-cp313-win_amd64.whl", hash = "sha256:0fd6165f3df49abc7c9de029b2b72d74bebd2ff2481a5ced003607eb61c56d3e", size = 13212196, upload-time = "2025-11-11T22:05:00.451Z" },
- { url = "https://files.pythonhosted.org/packages/46/57/1649019893accb3f195780fec55b8bf6793343faf140040bc73f1c28d6a5/tensorstore-0.1.79-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:6f8f5a940eab434a951c2dadcc7c0516c7bef6d8b7a7144054f7a0c56152b5f5", size = 16488849, upload-time = "2025-11-11T22:05:03.014Z" },
- { url = "https://files.pythonhosted.org/packages/bf/23/2668cb120e855a6a7a8a5eb0eba30e2e7020da932a4d3fa13c6ee3c41f9f/tensorstore-0.1.79-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:97756d2cba3c5ce21e15602c2af5a02521cc0ecda7f9fb6d18da2f3bd51827f4", size = 14511448, upload-time = "2025-11-11T22:05:05.58Z" },
- { url = "https://files.pythonhosted.org/packages/6a/0e/c38f079f3933cc284aab53d52976f6cb4f1ad43bb6a704ac27e0b710f176/tensorstore-0.1.79-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:847982652273fb7b2d694b789205747aaf3e50ae64738c5cb7b5eb03d86a9947", size = 18949282, upload-time = "2025-11-11T22:05:07.562Z" },
- { url = "https://files.pythonhosted.org/packages/6f/99/03479deea5bfd27a0d8a8c75d5f1d85417a7bbc9c6c7a90fb85b4a4e347a/tensorstore-0.1.79-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7af9422269c2bfcdecf9dd55309060665ab9c2d7f6c892377ed32c032400feea", size = 20931601, upload-time = "2025-11-11T22:05:10.098Z" },
- { url = "https://files.pythonhosted.org/packages/26/36/2617edf6c6d6fc73b3ff96d9d0b97332adf0d0c56fa2014a226bf4f7dfa6/tensorstore-0.1.79-cp314-cp314-win_amd64.whl", hash = "sha256:bbd8c1ab7d2e3c03ded3d40bb373ee9a67668e33a564484927865ce43b210386", size = 13599766, upload-time = "2025-11-11T22:05:12.265Z" },
+ { name = "numpy", version = "2.4.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/88/18/7b91daa9cf29dbb6bfdd603154f355c9069a9cd8c757038fe52b0f613611/tensorstore-0.1.80.tar.gz", hash = "sha256:4158fe76b96f62d12a37d7868150d836e089b5280b2bdd363c43c5d651f10e26", size = 7090032, upload-time = "2025-12-10T21:35:10.941Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/96/1f/902d822626a6c2774229236440c85c17e384f53afb4d2c6fa4118a30c53a/tensorstore-0.1.80-cp311-cp311-macosx_10_14_x86_64.whl", hash = "sha256:246641a8780ee5e04e88bc95c8e31faac6471bab1180d1f5cdc9804b29a77c04", size = 16519587, upload-time = "2025-12-10T21:34:05.758Z" },
+ { url = "https://files.pythonhosted.org/packages/21/c9/2ed6ed809946d7b0de08645800584937912c404b85900eea66361d5e2541/tensorstore-0.1.80-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:7451b30f99d9f31a2b9d70e6ef61815713dc782c58c6d817f91781341e4dac05", size = 14550336, upload-time = "2025-12-10T21:34:08.394Z" },
+ { url = "https://files.pythonhosted.org/packages/d6/50/d97acbc5a4d632590dd9053697181fa41cbcb09389e88acfa6958ab8ead5/tensorstore-0.1.80-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1113a6982fc0fa8dda8fcc0495715e647ac3360909a86ff13f2e04564f82d54a", size = 19004795, upload-time = "2025-12-10T21:34:11.14Z" },
+ { url = "https://files.pythonhosted.org/packages/a9/2d/fdbbf3cd6f08d41d3c1d8a2f6a67a4a2a07ac238fb6eeea852c2669184a3/tensorstore-0.1.80-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b193a7a1c4f455a61e60ed2dd67271a3daab0910ddb4bd9db51390d1b36d9996", size = 20996847, upload-time = "2025-12-10T21:34:14.031Z" },
+ { url = "https://files.pythonhosted.org/packages/b6/37/4570fe93f0c5c339843042556a841cfe0073d3e7fa4dae7ba31417eb4fd3/tensorstore-0.1.80-cp311-cp311-win_amd64.whl", hash = "sha256:9c088e8c9f67c266ef4dae3703bd617f7c0cb0fd98e99c4500692e38a4328140", size = 13258296, upload-time = "2025-12-10T21:34:16.764Z" },
+ { url = "https://files.pythonhosted.org/packages/c3/47/8733a99926caca2db6e8dbe22491c0623da2298a23bc649bfe6e6f645fa7/tensorstore-0.1.80-cp312-cp312-macosx_10_14_x86_64.whl", hash = "sha256:f65dfaf9e737a41389e29a5a2ea52ca5d14c8d6f48b402c723d800cd16d322b0", size = 16537887, upload-time = "2025-12-10T21:34:19.799Z" },
+ { url = "https://files.pythonhosted.org/packages/50/54/59a34fee963e46f9f401c54131bdc6a17d6cfb10e5a094d586d33ae273df/tensorstore-0.1.80-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:f8b51d7e685bbb63f6becd7d2ac8634d5ab67ec7e53038e597182e2db2c7aa90", size = 14551674, upload-time = "2025-12-10T21:34:22.171Z" },
+ { url = "https://files.pythonhosted.org/packages/87/15/0734521f8b648e2c43a00f1bc99a7195646c9e4e31f64ab22a15ac84e75c/tensorstore-0.1.80-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:acb8d52fadcefafef4ef8ecca3fc99b1d0e3c5c5a888766484c3e39f050be7f5", size = 19013402, upload-time = "2025-12-10T21:34:24.961Z" },
+ { url = "https://files.pythonhosted.org/packages/48/85/55addd16896343ea2731388028945576060139dda3c68a15d6b00158ef6f/tensorstore-0.1.80-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bc28a58c580253a526a4b6d239d18181ef96f1e285a502dbb03ff15eeec07a5b", size = 21007488, upload-time = "2025-12-10T21:34:28.093Z" },
+ { url = "https://files.pythonhosted.org/packages/c3/d2/5075cfea2ffd13c5bd2e91d76cdf87a355f617e40fa0b8fbfbbdc5e7bd23/tensorstore-0.1.80-cp312-cp312-win_amd64.whl", hash = "sha256:1b2b2ed0051dfab7e25295b14e6620520729e6e2ddf505f98c8d3917569614bf", size = 13263376, upload-time = "2025-12-10T21:34:30.797Z" },
+ { url = "https://files.pythonhosted.org/packages/79/3d/34e64ef1e4573419671b9aa72b69e927702d84e1d95bcef3cc98a8d63ad5/tensorstore-0.1.80-cp313-cp313-macosx_10_14_x86_64.whl", hash = "sha256:46136fe42ee6dd835d957db37073058aea0b78fdfbe2975941640131b7740824", size = 16537403, upload-time = "2025-12-10T21:34:33.404Z" },
+ { url = "https://files.pythonhosted.org/packages/94/03/19f45f6134bbb98d13f8de3160271aa4f49466e1a91000c6ab2eec7d9264/tensorstore-0.1.80-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:a92505189731fcb03f1c69a84ea4460abb24204bfac1f339448a0621e7def77c", size = 14551401, upload-time = "2025-12-10T21:34:36.041Z" },
+ { url = "https://files.pythonhosted.org/packages/f7/fa/d5de3f1b711773e33a329b5fe11de1265b77a13f2a2447fe685ee5d0c1bc/tensorstore-0.1.80-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:de63843706fdfe9565a45567238c5b1e55a0b28bbde6524200b31d29043a9a16", size = 19013246, upload-time = "2025-12-10T21:34:38.507Z" },
+ { url = "https://files.pythonhosted.org/packages/87/ee/e874b5a495a7aa14817772a91095971f3a965a4cef5b52ad06a8e15c924f/tensorstore-0.1.80-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6c8dbbdd31cbb28eccfb23dbbd4218fe67bfc32e9cb452875a485b81031c949d", size = 21008391, upload-time = "2025-12-10T21:34:41.332Z" },
+ { url = "https://files.pythonhosted.org/packages/2f/99/03bcc5da6a735ffa290f888af1f2c990edc9a375b373d04152d8b6fce3e8/tensorstore-0.1.80-cp313-cp313-win_amd64.whl", hash = "sha256:c0529afab3800749dd245843d3bf0d061a109a8edb77fb345f476e8bccda51b8", size = 13262770, upload-time = "2025-12-10T21:34:43.673Z" },
+ { url = "https://files.pythonhosted.org/packages/ef/57/75f65d8ba5829768e67aa978d4c0856956b9bacb279c96f0ee28564b6c41/tensorstore-0.1.80-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:04c29d979eb8b8ee48f873dc13d2701bfd49425500ffc5b848e4ec55b2548281", size = 16543698, upload-time = "2025-12-10T21:34:46.095Z" },
+ { url = "https://files.pythonhosted.org/packages/9c/92/17a18eac2cfdb019c36b4362d1a5c614d769a78d10cad0aae3d368fefa0e/tensorstore-0.1.80-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:189d924eaec394c9331e284a9c513ed583e336472a925823b5151cb26f41d091", size = 14552217, upload-time = "2025-12-10T21:34:48.539Z" },
+ { url = "https://files.pythonhosted.org/packages/b6/df/71f317633a0cd5270b85d185ac5ce91a749930fc076205d3fae4f1f043ed/tensorstore-0.1.80-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:07e4a84bacf70b78305831897068a9b5ad30326e63bbeb92c4bf7e565fcf5e9e", size = 19020675, upload-time = "2025-12-10T21:34:51.168Z" },
+ { url = "https://files.pythonhosted.org/packages/2b/35/f03cdb5edf8e009ff73e48c0c3d0f692a70a7ffc5e393f2ea1761eff89b5/tensorstore-0.1.80-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d2b353b0bd53fedd77fc5a12a1c1a91cacc3cf59e3dd785529c5a54b31d1c7b1", size = 21009171, upload-time = "2025-12-10T21:34:53.979Z" },
+ { url = "https://files.pythonhosted.org/packages/51/a9/6cf5675a7d4214ae7fd114c5c7bcf09aa71a57fce6648e187576e60c0c08/tensorstore-0.1.80-cp314-cp314-win_amd64.whl", hash = "sha256:53fd121ccd332bc4cc397f7af45889360c668b43dc3ff6bc3264df0f9886c11a", size = 13653134, upload-time = "2025-12-10T21:34:56.818Z" },
+ { url = "https://files.pythonhosted.org/packages/1d/d0/8cd2725c6691387438491d0c1fbbe07235439084722f968c20f07de4119d/tensorstore-0.1.80-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:4baee67fce95f29f593fbab4866119347115eaace887732aa92cfcbb9e6b0748", size = 16620211, upload-time = "2025-12-10T21:34:59.106Z" },
+ { url = "https://files.pythonhosted.org/packages/f7/c0/289b8979a08b477ce0622a6c13a59dbe8cda407e4c82c8b2ab0b4f8d1989/tensorstore-0.1.80-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:8cd11027b5a8b66db8d344085a31a1666c78621dac27039c4d571bc4974804a1", size = 14638072, upload-time = "2025-12-10T21:35:01.598Z" },
+ { url = "https://files.pythonhosted.org/packages/42/47/5c63024ced48e3f440c131babedef2f5398f48ab81c1aeee6c6193491d1c/tensorstore-0.1.80-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6b7c5dd434bba4ee08fe46bbbdb25c60dd3d47ccb4b8561a9751cf1526da52b8", size = 19024739, upload-time = "2025-12-10T21:35:04.324Z" },
+ { url = "https://files.pythonhosted.org/packages/6e/16/d08ade819949e0622f27e949c15b09f7b86ac18f8ac7c4d8bdfb4a711076/tensorstore-0.1.80-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e93df6d34ff5f0f6be245f4d29b99a7c1eef8ad91b50686adf57a5eeea99cb74", size = 21024449, upload-time = "2025-12-10T21:35:08.149Z" },
]
[[package]]
@@ -5609,27 +5651,32 @@ wheels = [
[[package]]
name = "tokenizers"
-version = "0.22.1"
+version = "0.22.2"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "huggingface-hub" },
]
-sdist = { url = "https://files.pythonhosted.org/packages/1c/46/fb6854cec3278fbfa4a75b50232c77622bc517ac886156e6afbfa4d8fc6e/tokenizers-0.22.1.tar.gz", hash = "sha256:61de6522785310a309b3407bac22d99c4db5dba349935e99e4d15ea2226af2d9", size = 363123, upload-time = "2025-09-19T09:49:23.424Z" }
-wheels = [
- { url = "https://files.pythonhosted.org/packages/bf/33/f4b2d94ada7ab297328fc671fed209368ddb82f965ec2224eb1892674c3a/tokenizers-0.22.1-cp39-abi3-macosx_10_12_x86_64.whl", hash = "sha256:59fdb013df17455e5f950b4b834a7b3ee2e0271e6378ccb33aa74d178b513c73", size = 3069318, upload-time = "2025-09-19T09:49:11.848Z" },
- { url = "https://files.pythonhosted.org/packages/1c/58/2aa8c874d02b974990e89ff95826a4852a8b2a273c7d1b4411cdd45a4565/tokenizers-0.22.1-cp39-abi3-macosx_11_0_arm64.whl", hash = "sha256:8d4e484f7b0827021ac5f9f71d4794aaef62b979ab7608593da22b1d2e3c4edc", size = 2926478, upload-time = "2025-09-19T09:49:09.759Z" },
- { url = "https://files.pythonhosted.org/packages/1e/3b/55e64befa1e7bfea963cf4b787b2cea1011362c4193f5477047532ce127e/tokenizers-0.22.1-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:19d2962dd28bc67c1f205ab180578a78eef89ac60ca7ef7cbe9635a46a56422a", size = 3256994, upload-time = "2025-09-19T09:48:56.701Z" },
- { url = "https://files.pythonhosted.org/packages/71/0b/fbfecf42f67d9b7b80fde4aabb2b3110a97fac6585c9470b5bff103a80cb/tokenizers-0.22.1-cp39-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:38201f15cdb1f8a6843e6563e6e79f4abd053394992b9bbdf5213ea3469b4ae7", size = 3153141, upload-time = "2025-09-19T09:48:59.749Z" },
- { url = "https://files.pythonhosted.org/packages/17/a9/b38f4e74e0817af8f8ef925507c63c6ae8171e3c4cb2d5d4624bf58fca69/tokenizers-0.22.1-cp39-abi3-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d1cbe5454c9a15df1b3443c726063d930c16f047a3cc724b9e6e1a91140e5a21", size = 3508049, upload-time = "2025-09-19T09:49:05.868Z" },
- { url = "https://files.pythonhosted.org/packages/d2/48/dd2b3dac46bb9134a88e35d72e1aa4869579eacc1a27238f1577270773ff/tokenizers-0.22.1-cp39-abi3-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e7d094ae6312d69cc2a872b54b91b309f4f6fbce871ef28eb27b52a98e4d0214", size = 3710730, upload-time = "2025-09-19T09:49:01.832Z" },
- { url = "https://files.pythonhosted.org/packages/93/0e/ccabc8d16ae4ba84a55d41345207c1e2ea88784651a5a487547d80851398/tokenizers-0.22.1-cp39-abi3-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:afd7594a56656ace95cdd6df4cca2e4059d294c5cfb1679c57824b605556cb2f", size = 3412560, upload-time = "2025-09-19T09:49:03.867Z" },
- { url = "https://files.pythonhosted.org/packages/d0/c6/dc3a0db5a6766416c32c034286d7c2d406da1f498e4de04ab1b8959edd00/tokenizers-0.22.1-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e2ef6063d7a84994129732b47e7915e8710f27f99f3a3260b8a38fc7ccd083f4", size = 3250221, upload-time = "2025-09-19T09:49:07.664Z" },
- { url = "https://files.pythonhosted.org/packages/d7/a6/2c8486eef79671601ff57b093889a345dd3d576713ef047776015dc66de7/tokenizers-0.22.1-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:ba0a64f450b9ef412c98f6bcd2a50c6df6e2443b560024a09fa6a03189726879", size = 9345569, upload-time = "2025-09-19T09:49:14.214Z" },
- { url = "https://files.pythonhosted.org/packages/6b/16/32ce667f14c35537f5f605fe9bea3e415ea1b0a646389d2295ec348d5657/tokenizers-0.22.1-cp39-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:331d6d149fa9c7d632cde4490fb8bbb12337fa3a0232e77892be656464f4b446", size = 9271599, upload-time = "2025-09-19T09:49:16.639Z" },
- { url = "https://files.pythonhosted.org/packages/51/7c/a5f7898a3f6baa3fc2685c705e04c98c1094c523051c805cdd9306b8f87e/tokenizers-0.22.1-cp39-abi3-musllinux_1_2_i686.whl", hash = "sha256:607989f2ea68a46cb1dfbaf3e3aabdf3f21d8748312dbeb6263d1b3b66c5010a", size = 9533862, upload-time = "2025-09-19T09:49:19.146Z" },
- { url = "https://files.pythonhosted.org/packages/36/65/7e75caea90bc73c1dd8d40438adf1a7bc26af3b8d0a6705ea190462506e1/tokenizers-0.22.1-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:a0f307d490295717726598ef6fa4f24af9d484809223bbc253b201c740a06390", size = 9681250, upload-time = "2025-09-19T09:49:21.501Z" },
- { url = "https://files.pythonhosted.org/packages/30/2c/959dddef581b46e6209da82df3b78471e96260e2bc463f89d23b1bf0e52a/tokenizers-0.22.1-cp39-abi3-win32.whl", hash = "sha256:b5120eed1442765cd90b903bb6cfef781fd8fe64e34ccaecbae4c619b7b12a82", size = 2472003, upload-time = "2025-09-19T09:49:27.089Z" },
- { url = "https://files.pythonhosted.org/packages/b3/46/e33a8c93907b631a99377ef4c5f817ab453d0b34f93529421f42ff559671/tokenizers-0.22.1-cp39-abi3-win_amd64.whl", hash = "sha256:65fd6e3fb11ca1e78a6a93602490f134d1fdeb13bcef99389d5102ea318ed138", size = 2674684, upload-time = "2025-09-19T09:49:24.953Z" },
+sdist = { url = "https://files.pythonhosted.org/packages/73/6f/f80cfef4a312e1fb34baf7d85c72d4411afde10978d4657f8cdd811d3ccc/tokenizers-0.22.2.tar.gz", hash = "sha256:473b83b915e547aa366d1eee11806deaf419e17be16310ac0a14077f1e28f917", size = 372115, upload-time = "2026-01-05T10:45:15.988Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/92/97/5dbfabf04c7e348e655e907ed27913e03db0923abb5dfdd120d7b25630e1/tokenizers-0.22.2-cp39-abi3-macosx_10_12_x86_64.whl", hash = "sha256:544dd704ae7238755d790de45ba8da072e9af3eea688f698b137915ae959281c", size = 3100275, upload-time = "2026-01-05T10:41:02.158Z" },
+ { url = "https://files.pythonhosted.org/packages/2e/47/174dca0502ef88b28f1c9e06b73ce33500eedfac7a7692108aec220464e7/tokenizers-0.22.2-cp39-abi3-macosx_11_0_arm64.whl", hash = "sha256:1e418a55456beedca4621dbab65a318981467a2b188e982a23e117f115ce5001", size = 2981472, upload-time = "2026-01-05T10:41:00.276Z" },
+ { url = "https://files.pythonhosted.org/packages/d6/84/7990e799f1309a8b87af6b948f31edaa12a3ed22d11b352eaf4f4b2e5753/tokenizers-0.22.2-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2249487018adec45d6e3554c71d46eb39fa8ea67156c640f7513eb26f318cec7", size = 3290736, upload-time = "2026-01-05T10:40:32.165Z" },
+ { url = "https://files.pythonhosted.org/packages/78/59/09d0d9ba94dcd5f4f1368d4858d24546b4bdc0231c2354aa31d6199f0399/tokenizers-0.22.2-cp39-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:25b85325d0815e86e0bac263506dd114578953b7b53d7de09a6485e4a160a7dd", size = 3168835, upload-time = "2026-01-05T10:40:38.847Z" },
+ { url = "https://files.pythonhosted.org/packages/47/50/b3ebb4243e7160bda8d34b731e54dd8ab8b133e50775872e7a434e524c28/tokenizers-0.22.2-cp39-abi3-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:bfb88f22a209ff7b40a576d5324bf8286b519d7358663db21d6246fb17eea2d5", size = 3521673, upload-time = "2026-01-05T10:40:56.614Z" },
+ { url = "https://files.pythonhosted.org/packages/e0/fa/89f4cb9e08df770b57adb96f8cbb7e22695a4cb6c2bd5f0c4f0ebcf33b66/tokenizers-0.22.2-cp39-abi3-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1c774b1276f71e1ef716e5486f21e76333464f47bece56bbd554485982a9e03e", size = 3724818, upload-time = "2026-01-05T10:40:44.507Z" },
+ { url = "https://files.pythonhosted.org/packages/64/04/ca2363f0bfbe3b3d36e95bf67e56a4c88c8e3362b658e616d1ac185d47f2/tokenizers-0.22.2-cp39-abi3-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:df6c4265b289083bf710dff49bc51ef252f9d5be33a45ee2bed151114a56207b", size = 3379195, upload-time = "2026-01-05T10:40:51.139Z" },
+ { url = "https://files.pythonhosted.org/packages/2e/76/932be4b50ef6ccedf9d3c6639b056a967a86258c6d9200643f01269211ca/tokenizers-0.22.2-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:369cc9fc8cc10cb24143873a0d95438bb8ee257bb80c71989e3ee290e8d72c67", size = 3274982, upload-time = "2026-01-05T10:40:58.331Z" },
+ { url = "https://files.pythonhosted.org/packages/1d/28/5f9f5a4cc211b69e89420980e483831bcc29dade307955cc9dc858a40f01/tokenizers-0.22.2-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:29c30b83d8dcd061078b05ae0cb94d3c710555fbb44861139f9f83dcca3dc3e4", size = 9478245, upload-time = "2026-01-05T10:41:04.053Z" },
+ { url = "https://files.pythonhosted.org/packages/6c/fb/66e2da4704d6aadebf8cb39f1d6d1957df667ab24cff2326b77cda0dcb85/tokenizers-0.22.2-cp39-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:37ae80a28c1d3265bb1f22464c856bd23c02a05bb211e56d0c5301a435be6c1a", size = 9560069, upload-time = "2026-01-05T10:45:10.673Z" },
+ { url = "https://files.pythonhosted.org/packages/16/04/fed398b05caa87ce9b1a1bb5166645e38196081b225059a6edaff6440fac/tokenizers-0.22.2-cp39-abi3-musllinux_1_2_i686.whl", hash = "sha256:791135ee325f2336f498590eb2f11dc5c295232f288e75c99a36c5dbce63088a", size = 9899263, upload-time = "2026-01-05T10:45:12.559Z" },
+ { url = "https://files.pythonhosted.org/packages/05/a1/d62dfe7376beaaf1394917e0f8e93ee5f67fea8fcf4107501db35996586b/tokenizers-0.22.2-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:38337540fbbddff8e999d59970f3c6f35a82de10053206a7562f1ea02d046fa5", size = 10033429, upload-time = "2026-01-05T10:45:14.333Z" },
+ { url = "https://files.pythonhosted.org/packages/fd/18/a545c4ea42af3df6effd7d13d250ba77a0a86fb20393143bbb9a92e434d4/tokenizers-0.22.2-cp39-abi3-win32.whl", hash = "sha256:a6bf3f88c554a2b653af81f3204491c818ae2ac6fbc09e76ef4773351292bc92", size = 2502363, upload-time = "2026-01-05T10:45:20.593Z" },
+ { url = "https://files.pythonhosted.org/packages/65/71/0670843133a43d43070abeb1949abfdef12a86d490bea9cd9e18e37c5ff7/tokenizers-0.22.2-cp39-abi3-win_amd64.whl", hash = "sha256:c9ea31edff2968b44a88f97d784c2f16dc0729b8b143ed004699ebca91f05c48", size = 2747786, upload-time = "2026-01-05T10:45:18.411Z" },
+ { url = "https://files.pythonhosted.org/packages/72/f4/0de46cfa12cdcbcd464cc59fde36912af405696f687e53a091fb432f694c/tokenizers-0.22.2-cp39-abi3-win_arm64.whl", hash = "sha256:9ce725d22864a1e965217204946f830c37876eee3b2ba6fc6255e8e903d5fcbc", size = 2612133, upload-time = "2026-01-05T10:45:17.232Z" },
+ { url = "https://files.pythonhosted.org/packages/84/04/655b79dbcc9b3ac5f1479f18e931a344af67e5b7d3b251d2dcdcd7558592/tokenizers-0.22.2-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:753d47ebd4542742ef9261d9da92cd545b2cacbb48349a1225466745bb866ec4", size = 3282301, upload-time = "2026-01-05T10:40:34.858Z" },
+ { url = "https://files.pythonhosted.org/packages/46/cd/e4851401f3d8f6f45d8480262ab6a5c8cb9c4302a790a35aa14eeed6d2fd/tokenizers-0.22.2-pp310-pypy310_pp73-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:e10bf9113d209be7cd046d40fbabbaf3278ff6d18eb4da4c500443185dc1896c", size = 3161308, upload-time = "2026-01-05T10:40:40.737Z" },
+ { url = "https://files.pythonhosted.org/packages/6f/6e/55553992a89982cd12d4a66dddb5e02126c58677ea3931efcbe601d419db/tokenizers-0.22.2-pp310-pypy310_pp73-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:64d94e84f6660764e64e7e0b22baa72f6cd942279fdbb21d46abd70d179f0195", size = 3718964, upload-time = "2026-01-05T10:40:46.56Z" },
+ { url = "https://files.pythonhosted.org/packages/59/8c/b1c87148aa15e099243ec9f0cf9d0e970cc2234c3257d558c25a2c5304e6/tokenizers-0.22.2-pp310-pypy310_pp73-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f01a9c019878532f98927d2bacb79bbb404b43d3437455522a00a30718cdedb5", size = 3373542, upload-time = "2026-01-05T10:40:52.803Z" },
]
[[package]]
@@ -5708,7 +5755,7 @@ dependencies = [
{ name = "fsspec" },
{ name = "jinja2" },
{ name = "networkx", version = "3.4.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11' or (extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" },
- { name = "networkx", version = "3.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11' or (extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" },
+ { name = "networkx", version = "3.6.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11' or (extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" },
{ name = "nvidia-cublas-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" },
{ name = "nvidia-cuda-cupti-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" },
{ name = "nvidia-cuda-nvrtc-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" },
@@ -5766,7 +5813,7 @@ version = "0.0.4"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version < '3.11' and extra == 'extra-13-megatron-core-dev') or (extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" },
- { name = "numpy", version = "2.3.5", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version >= '3.11' and extra == 'extra-13-megatron-core-dev') or (extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" },
+ { name = "numpy", version = "2.4.0", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version >= '3.11' and extra == 'extra-13-megatron-core-dev') or (extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" },
{ name = "torch", marker = "sys_platform == 'never'" },
{ name = "torchvision", marker = "sys_platform == 'never'" },
]
@@ -5781,7 +5828,7 @@ version = "0.24.1"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version < '3.11' and sys_platform != 'linux' and extra == 'extra-13-megatron-core-dev') or (python_full_version >= '3.11' and extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts') or (sys_platform == 'linux' and extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" },
- { name = "numpy", version = "2.3.5", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version >= '3.11' and sys_platform != 'linux' and extra == 'extra-13-megatron-core-dev') or (python_full_version < '3.11' and extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts') or (sys_platform == 'linux' and extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" },
+ { name = "numpy", version = "2.4.0", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version >= '3.11' and sys_platform != 'linux' and extra == 'extra-13-megatron-core-dev') or (python_full_version < '3.11' and extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts') or (sys_platform == 'linux' and extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" },
{ name = "pillow", marker = "sys_platform != 'linux'" },
{ name = "torch", marker = "sys_platform == 'never'" },
]
@@ -5850,8 +5897,8 @@ wheels = [
[[package]]
name = "transformer-engine"
-version = "2.10.0+769ed778"
-source = { git = "https://github.com/NVIDIA/TransformerEngine.git?rev=release_v2.10#769ed778341a32c8c593fda391700c0a80f65f1f" }
+version = "2.11.0+c188b533"
+source = { git = "https://github.com/NVIDIA/TransformerEngine.git?rev=release_v2.11#c188b533cc3721ca9c6bbfd26148f5cf60108c25" }
dependencies = [
{ name = "einops" },
{ name = "importlib-metadata" },
@@ -5870,7 +5917,7 @@ dependencies = [
{ name = "filelock" },
{ name = "huggingface-hub" },
{ name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11' or (extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" },
- { name = "numpy", version = "2.3.5", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11' or (extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" },
+ { name = "numpy", version = "2.4.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11' or (extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" },
{ name = "packaging" },
{ name = "pyyaml" },
{ name = "regex" },
@@ -5916,7 +5963,7 @@ wheels = [
[[package]]
name = "typer"
-version = "0.20.0"
+version = "0.21.1"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "click" },
@@ -5924,9 +5971,9 @@ dependencies = [
{ name = "shellingham" },
{ name = "typing-extensions" },
]
-sdist = { url = "https://files.pythonhosted.org/packages/8f/28/7c85c8032b91dbe79725b6f17d2fffc595dff06a35c7a30a37bef73a1ab4/typer-0.20.0.tar.gz", hash = "sha256:1aaf6494031793e4876fb0bacfa6a912b551cf43c1e63c800df8b1a866720c37", size = 106492, upload-time = "2025-10-20T17:03:49.445Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/36/bf/8825b5929afd84d0dabd606c67cd57b8388cb3ec385f7ef19c5cc2202069/typer-0.21.1.tar.gz", hash = "sha256:ea835607cd752343b6b2b7ce676893e5a0324082268b48f27aa058bdb7d2145d", size = 110371, upload-time = "2026-01-06T11:21:10.989Z" }
wheels = [
- { url = "https://files.pythonhosted.org/packages/78/64/7713ffe4b5983314e9d436a90d5bd4f63b6054e2aca783a3cfc44cb95bbf/typer-0.20.0-py3-none-any.whl", hash = "sha256:5b463df6793ec1dca6213a3cf4c0f03bc6e322ac5e16e13ddd622a889489784a", size = 47028, upload-time = "2025-10-20T17:03:47.617Z" },
+ { url = "https://files.pythonhosted.org/packages/a0/1d/d9257dd49ff2ca23ea5f132edf1281a0c4f9de8a762b9ae399b670a59235/typer-0.21.1-py3-none-any.whl", hash = "sha256:7985e89081c636b88d172c2ee0cfe33c253160994d47bdfdc302defd7d1f1d01", size = 47381, upload-time = "2026-01-06T11:21:09.824Z" },
]
[[package]]
@@ -5965,11 +6012,11 @@ wheels = [
[[package]]
name = "tzdata"
-version = "2025.2"
+version = "2025.3"
source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/95/32/1a225d6164441be760d75c2c42e2780dc0873fe382da3e98a2e1e48361e5/tzdata-2025.2.tar.gz", hash = "sha256:b60a638fcc0daffadf82fe0f57e53d06bdec2f36c4df66280ae79bce6bd6f2b9", size = 196380, upload-time = "2025-03-23T13:54:43.652Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/5e/a7/c202b344c5ca7daf398f3b8a477eeb205cf3b6f32e7ec3a6bac0629ca975/tzdata-2025.3.tar.gz", hash = "sha256:de39c2ca5dc7b0344f2eba86f49d614019d29f060fc4ebc8a417896a620b56a7", size = 196772, upload-time = "2025-12-13T17:45:35.667Z" }
wheels = [
- { url = "https://files.pythonhosted.org/packages/5c/23/c7abc0ca0a1526a0774eca151daeb8de62ec457e77262b66b359c3c7679e/tzdata-2025.2-py2.py3-none-any.whl", hash = "sha256:1a403fada01ff9221ca8044d701868fa132215d84beb92242d9acd2147f667a8", size = 347839, upload-time = "2025-03-23T13:54:41.845Z" },
+ { url = "https://files.pythonhosted.org/packages/c7/b0/003792df09decd6849a5e39c28b513c06e84436a54440380862b5aeff25d/tzdata-2025.3-py2.py3-none-any.whl", hash = "sha256:06a47e5700f3081aab02b2e513160914ff0694bce9947d6b76ebd6bf57cfc5d1", size = 348521, upload-time = "2025-12-13T17:45:33.889Z" },
]
[[package]]
@@ -5983,16 +6030,16 @@ wheels = [
[[package]]
name = "uvicorn"
-version = "0.38.0"
+version = "0.40.0"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "click" },
{ name = "h11" },
{ name = "typing-extensions", marker = "python_full_version < '3.11' or (extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" },
]
-sdist = { url = "https://files.pythonhosted.org/packages/cb/ce/f06b84e2697fef4688ca63bdb2fdf113ca0a3be33f94488f2cadb690b0cf/uvicorn-0.38.0.tar.gz", hash = "sha256:fd97093bdd120a2609fc0d3afe931d4d4ad688b6e75f0f929fde1bc36fe0e91d", size = 80605, upload-time = "2025-10-18T13:46:44.63Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/c3/d1/8f3c683c9561a4e6689dd3b1d345c815f10f86acd044ee1fb9a4dcd0b8c5/uvicorn-0.40.0.tar.gz", hash = "sha256:839676675e87e73694518b5574fd0f24c9d97b46bea16df7b8c05ea1a51071ea", size = 81761, upload-time = "2025-12-21T14:16:22.45Z" }
wheels = [
- { url = "https://files.pythonhosted.org/packages/ee/d9/d88e73ca598f4f6ff671fb5fde8a32925c2e08a637303a1d12883c7305fa/uvicorn-0.38.0-py3-none-any.whl", hash = "sha256:48c0afd214ceb59340075b4a052ea1ee91c16fbc2a9b1469cca0e54566977b02", size = 68109, upload-time = "2025-10-18T13:46:42.958Z" },
+ { url = "https://files.pythonhosted.org/packages/3d/d8/2083a1daa7439a66f3a48589a57d576aa117726762618f6bb09fe3798796/uvicorn-0.40.0-py3-none-any.whl", hash = "sha256:c6c8f55bc8bf13eb6fa9ff87ad62308bbbc33d0b67f84293151efe87e0d5f2ee", size = 68502, upload-time = "2025-12-21T14:16:21.041Z" },
]
[[package]]
@@ -6155,7 +6202,7 @@ source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "braceexpand" },
{ name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version < '3.11' and extra == 'extra-13-megatron-core-dev') or (python_full_version < '3.11' and extra == 'extra-13-megatron-core-lts') or (extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" },
- { name = "numpy", version = "2.3.5", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version >= '3.11' and extra == 'extra-13-megatron-core-dev') or (python_full_version >= '3.11' and extra == 'extra-13-megatron-core-lts') or (extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" },
+ { name = "numpy", version = "2.4.0", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version >= '3.11' and extra == 'extra-13-megatron-core-dev') or (python_full_version >= '3.11' and extra == 'extra-13-megatron-core-lts') or (extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" },
{ name = "pyyaml" },
]
sdist = { url = "https://files.pythonhosted.org/packages/5a/3a/68800d92e065cf4750ebecf973b13979c0c929b439e1293012938862038d/webdataset-1.0.2.tar.gz", hash = "sha256:7f0498be827cfa46cc5430a58768a24e2c6a410676a61be1838f53d61afdaab4", size = 80090, upload-time = "2025-06-19T23:26:21.945Z" }