Image Description Support for DOCX

[ameetpal]

Hi, I recently came across a use case involving ingesting image descriptions into a vector database. We currently use markitdown for both PPTX and DOCX files. While PPTX support for image extraction is available, I noticed that similar support is not present for DOCX.

Are there any plans to add image description support for DOCX in the near future?

[stefan-rink]

i created a custom converter "annotated_docx_converter.py"

from markitdown import StreamInfo, DocumentConverterResult
from markitdown.converters import DocxConverter
import mammoth
import mammoth.html as html
from mammoth.images import data_uri

from typing import Any, BinaryIO, Callable, Union


__plugin_interface_version__ = 1


class AnnotatedDocxConverter(DocxConverter):
    """
    A converter that annotates images with LLM.
    """

    def llm_caption(
        self, llm_client, llm_model, llm_prompt=None, data_uri=None
    ) -> Union[None, str]:
        """
        Captions an image with an LLM.
        """

        messages = [
            {
                "role": "system",
                "content": llm_prompt  # Markitdown prompt handling does not work
            },
            {
                "role": "user",
                "content": [
                    {
                        "type": "image_url",
                        "image_url": {
                            "url": data_uri,
                        },
                    },
                ],
            }
        ]

        # Call the OpenAI API
        response = llm_client.chat.completions.create(
            model=llm_model, messages=messages)
        return response.choices[0].message.content

    def get_image_converter(self, kwargs) -> Callable[[mammoth.documents.Image], html.Element]:
        """
        Returns a handler that replaces each image with a paragraph block containing alt-text.
        """

        llm_client = kwargs.get("llm_client")
        llm_model = kwargs.get("llm_model")
        llm_prompt = kwargs.get("llm_prompt")

        if llm_client is None or llm_model is None or llm_prompt is None:
            return data_uri
        else:
            def handler(image: mammoth.documents.Image) -> html.Element:
                # get the data uri of the image
                uri = data_uri(image)[0].attributes['src']

                llm_response = self.llm_caption(
                    data_uri=uri, llm_client=llm_client, llm_model=llm_model, llm_prompt=llm_prompt)

                # Wrap the alt-text in a paragraph element
                return [html.element("p", {}, [html.text(llm_response)])]

            return handler

    def convert(
        self,
        file_stream: BinaryIO,
        stream_info: StreamInfo,
        **kwargs: Any,
    ) -> DocumentConverterResult:
        style_map = kwargs.get("style_map")

        # Set up debugging around Mammoth conversion
        image_converter = self.get_image_converter(kwargs=kwargs)
        try:
            result = mammoth.convert_to_html(
                file_stream,
                style_map=style_map,
                convert_image=image_converter,
            )
        except Exception as e:
            print(
                f"[AnnotatedDocxConverter] ERROR during mammoth.convert_to_html: {e}")
            raise

        html = result.value
        return self._html_converter.convert_string(html)

and registered it like this in my runner:

from markitdown import MarkItDown, PRIORITY_SPECIFIC_FILE_FORMAT

# ....

        md = MarkItDown(llm_client=llm_client, llm_model=model,
                        llm_prompt=image_annotation_prompt)
        md.register_converter(AnnotatedDocxConverter(),
                              priority=PRIORITY_SPECIFIC_FILE_FORMAT)

If I find time i'll create a PR but maybe this helps