cannot unpack non-iterable NoneType object

[CYWYKKK]

I have a exception problem "cannot unpack non-iterable NoneType object"，this problem may be involved in the following code，plz help me:
def generate_embeddings(self, text: str) -> FullEmbeddings:
all_layer_embeddings: Dict[int, LayerEmbeddings] = {}

    def split_embedding(embedding: List[float], chunk_size: int) -> List[List[float]]
        return [embedding[i:i+chunk_size] for i in range(0, len(embedding), chunk_size)]

    with torch.no_grad():  

        batch_dict = self._tokenizer([text], padding=True, return_tensors="pt")
        input_ids = batch_dict["input_ids"].to(self._model.device)
        hidden_states = self._model.embed_tokens(input_ids)  


        for layer_idx, layer in enumerate(self._model.layers):

            hidden_states = layer(hidden_states)[0]


            if layer_idx in self.target_layers:
                o_proj_input = layer.self_attn.o_proj.last_input
                if o_proj_input is not None:

                    last_token_o_input = o_proj_input[0][-1]

                    layer_emb = LayerEmbeddings(split_embedding(last_token_o_input.tolist(), 128))
                    all_layer_embeddings[layer_idx] = layer_emb


        standard_emb = self._model.norm(hidden_states)[0][-1].tolist()

    return FullEmbeddings(standard_embedding=standard_emb, layer_embeddings=all_layer_embeddings)

[rgersten]

Good afternoon,

Can you provide a little bit more context? For example, please let us know, what kind of command you are trying to execute and what embedding model you are using? Or are you using the default one (Salesforce/SFR-Embedding-Mistral)? What is the exact error message including the source file and line number?

Thank you in advance.

[CYWYKKK]

Thank you very much，here is my code:

import torch
import traceback
from typing import Optional, List, Dict, Set
from dataclasses import dataclass
from transformers import AutoTokenizer, AutoModel
import argparse
from fastapi import HTTPException

------------------------------

Auxiliary class definitions (storing embedding results)

------------------------------

Embedding = list[float]

@DataClass(frozen=True)
class LayerEmbeddings:
"""
Data class to store the attention heads of a layer.
"""
attention_heads: list[Embedding]

@classmethod
def from_dict(cls, emb_dict: dict):
    return cls(**emb_dict)

@DataClass(frozen=True)
class FullEmbeddings:
"""
Data class to store the standard embedding as well as embeddings from the
different attention heads for the various layers.
"""
standard_embedding: Embedding
layer_embeddings: dict[int, LayerEmbeddings]

@classmethod
def from_dict(cls, emb_dict: dict) -> 'FullEmbeddings':
    layer_embeddings: dict[int, LayerEmbeddings] = {}
    for layer_idx, l_emb_dict in emb_dict["layers"].items():
        layer_embeddings[int(layer_idx)] = LayerEmbeddings.from_dict(l_emb_dict)
    return cls(
        emb_dict["standard"],
        layer_embeddings
    )

------------------------------

Core embedding model class

------------------------------

class EmbeddingModel:
"""Wraps the Qwen/Qwen3-Embedding-8B model, supporting caching of attention layer inputs"""
class CachingModule(torch.nn.Module):
"""Custom Torch module for caching module inputs"""
def init(self, module: torch.nn.Module):
super().init()
self._module = module
self.last_input: Optional[torch.Tensor] = None # Cache input

    def forward(self, x: torch.Tensor) -> torch.Tensor:
        self.last_input = x  # Cache input
        return self._module(x)  # Normal forward pass

def __init__(self, target_layers: Optional[Set[int]], device: str, model_name_or_path: str = "Qwen/Qwen3-Embedding-8B"):
    """
    Initialize the embedding model
    :param target_layers: Layer indices for which to extract attention heads (default: last layer)
    :param device: Device to run the model on (cuda/cpu)
    :param model_name_or_path: Model name or local path
    """
    self._tokenizer = AutoTokenizer.from_pretrained(model_name_or_path)
    self._model = AutoModel.from_pretrained(
        model_name_or_path,
        device_map=device  # Auto assign device
    )
    self.target_layers = target_layers or {len(self._model.layers) - 1}  # Default: last layer

    # Add caching to the output projection layer of each self-attention layer
    for layer in self._model.layers:
        layer.self_attn.o_proj = self.CachingModule(layer.self_attn.o_proj)

def generate_embeddings(self, text: str) -> FullEmbeddings:
    """
    Generate text embeddings (standard embedding + target layer attention head embeddings)
    :param text: Input text
    :return: Complete embedding results (FullEmbeddings)
    """
    all_layer_embeddings: Dict[int, LayerEmbeddings] = {}  # Store attention head embeddings per layer

    def split_embedding(embedding: List[float], chunk_size: int) -> List[List[float]]:
        """Split a flat embedding vector into chunks of specified size (corresponding to attention heads)"""
        return [embedding[i:i+chunk_size] for i in range(0, len(embedding), chunk_size)]

    with torch.no_grad():  # Disable gradient calculation
        # 1. Tokenization and initial encoding
        batch_dict = self._tokenizer([text], padding=True, return_tensors="pt")
        input_ids = batch_dict["input_ids"].to(self._model.device)
        hidden_states = self._model.embed_tokens(input_ids)  # Token embeddings

        # 2. Forward pass through layers, extract target layer attention head outputs
        for layer_idx, layer in enumerate(self._model.layers):
            # Execute layer computation (output is a tuple, first element is updated hidden_states)
            hidden_states = layer(hidden_states)[0]

            # If current layer is a target layer, extract cached self-attention output
            if layer_idx in self.target_layers:
                o_proj_input = layer.self_attn.o_proj.last_input
                if o_proj_input is not None:
                    # Take output of the last token (similar to [CLS] but using last token)
                    last_token_o_input = o_proj_input[0][-1]
                    # Split embedding (assume 128 dimensions per attention head output)
                    layer_emb = LayerEmbeddings(split_embedding(last_token_o_input.tolist(), 128))
                    all_layer_embeddings[layer_idx] = layer_emb

        # 3. Compute standard sentence embedding (last token of RMSNorm output from last layer)
        standard_emb = self._model.norm(hidden_states)[0][-1].tolist()

    return FullEmbeddings(standard_embedding=standard_emb, layer_embeddings=all_layer_embeddings)

------------------------------

Singleton model management

------------------------------

_embedding_model: Optional[EmbeddingModel] = None # Global singleton instance

def get_embedding_model(model_name_or_path: str = "Qwen/Qwen3-Embedding-8B") -> EmbeddingModel:
"""
Get the singleton instance of the embedding model (initialized on first call)
:param model_name_or_path: Model name or local path
:return: Instance of the embedding model
"""
global _embedding_model
if _embedding_model is None:
try:
device = "cuda" if torch.cuda.is_available() else "cpu"
# Default target layer: last layer (adjust based on model depth, e.g., 31 for Qwen3-Embedding-8B's 32 layers)
target_layers = {len(AutoModel.from_pretrained(model_name_or_path).layers) - 1}
_embedding_model = EmbeddingModel(
target_layers=target_layers,
device=device,
model_name_or_path=model_name_or_path
)
print(f"✅ Embedding model loaded successfully (device: {device})")
except Exception as e:
raise HTTPException(
status_code=500,
detail=f"Model loading failed: {str(e)}"
) from e
return _embedding_model

------------------------------

Main function (for testing)

------------------------------

def main():
# 1. Initialize the model (singleton)
try:
embed_model = get_embedding_model() # Pass custom model path if needed: "your/model/path"
except HTTPException as e:
print(f"❌ Error: {e.detail}")
return

# 2. Generate embeddings
input_text = "Agent debugging test"
try:
    full_emb = embed_model.generate_embeddings(input_text)
    print(f"✅ Text embeddings generated successfully: '{input_text}'")
except Exception as e:
    print(f"❌ Failed to generate embeddings: {str(e)}")
    traceback.print_exc()
    return

# 3. Output result verification
print(f"🔢 Standard embedding dimension: {len(full_emb.standard_embedding)}")
print(f"🌰 First 10 dimensions of standard embedding: {full_emb.standard_embedding[:10]}")

# Print example first-layer attention head embeddings (first 2 heads, first 5 dimensions)
if full_emb.layer_embeddings:
    first_layer_emb = next(iter(full_emb.layer_embeddings.values()))
    print(f"🧠 First layer attention head embeddings (total {len(first_layer_emb.attention_heads)} heads):")
    for head_idx, head_emb in enumerate(first_layer_emb.attention_heads[:2]):
        print(f"  Head {head_idx+1}: {head_emb[:5]}...")
else:
    print("⚠️ No attention head embeddings found")

if name == "main":
# Parse command-line arguments (example: can extend model path parameter)
# parser = argparse.ArgumentParser(description="Qwen Embedding Model Test")
# args = parser.parse_args()

# Pass command-line arguments to the singleton (optional)
global _embedding_model
if _embedding_model is None:
    try:
        device = "cuda" if torch.cuda.is_available() else "cpu"
        target_layers = {len(AutoModel.from_pretrained("Qwen/Qwen3-Embedding-8B").layers) - 1}
        _embedding_model = EmbeddingModel(
            target_layers=target_layers,
            device=device,
            model_name_or_path="Qwen/Qwen3-Embedding-8B"
        )
        print("Model loaded successfully")
    except Exception as e:
        raise RuntimeError("Model loading failed") from e

main()

====================================================================================
my error is:

Traceback (most recent call last):
File "/tmp/ipykernel_120/487577833.py", line 164, in main
full_emb = embed_model.generate_embeddings(input_text)
File "/tmp/ipykernel_120/487577833.py", line 101, in generate_embeddings
hidden_states = layer(hidden_states)[0]
File "/opt/conda/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1553, in _wrapped_call_impl
return self._call_impl(*args, **kwargs)
File "/opt/conda/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1562, in _call_impl
return forward_call(*args, **kwargs)
File "/opt/conda/lib/python3.10/site-packages/transformers/models/qwen3/modeling_qwen3.py", line 289, in forward
hidden_states, self_attn_weights = self.self_attn(
File "/opt/conda/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1553, in _wrapped_call_impl
return self._call_impl(*args, **kwargs)
File "/opt/conda/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1562, in _call_impl
return forward_call(*args, **kwargs)
File "/opt/conda/lib/python3.10/site-packages/transformers/models/qwen3/modeling_qwen3.py", line 221, in forward
cos, sin = position_embeddings
TypeError: cannot unpack non-iterable NoneType object

[CYWYKKK]

the model what I use are Qwen3-Embedding-8B and SFR-Embedding-Mistral，both two embedding model are the same error