From dc78180255e7f80b0384629f5bca65e8e1b73439 Mon Sep 17 00:00:00 2001 From: kaan-playbook Date: Fri, 20 Jun 2025 12:36:36 +0200 Subject: [PATCH 01/16] Update routes.py --- marker_api/routes.py | 151 +++++++++++++++++++++++++++++-------------- 1 file changed, 101 insertions(+), 50 deletions(-) diff --git a/marker_api/routes.py b/marker_api/routes.py index 8f3b66a..146f8f6 100644 --- a/marker_api/routes.py +++ b/marker_api/routes.py @@ -2,6 +2,7 @@ import time import base64 from marker.convert import convert_single_pdf +from marker.models import load_all_models from marker.logger import configure_logging import logging @@ -18,61 +19,111 @@ def parse_pdf_and_return_markdown(pdf_file: bytes, extract_images: bool, model_l Args: pdf_file (bytes): The content of the PDF file. extract_images (bool): Whether to extract images or not. + model_list: The model list from load_all_models() - Returns - tuple: A tuple containing the full text, metadata, and image data (if extracted). + Returns: + tuple: A tuple containing the full text, metadata, and image data. """ - logger.debug("Parsing PDF file") - full_text, images, out_meta = convert_single_pdf(pdf_file, model_list) - logger.debug(f"Images extracted: {list(images.keys())}") - image_data = {} - if extract_images: - for i, (filename, image) in enumerate(images.items()): - logger.debug(f"Processing image {filename}") - - # Save image as PNG - image.save(filename, "PNG") - - # Read the saved image file as bytes - with open(filename, "rb") as f: - image_bytes = f.read() - - # Convert image to base64 - image_base64 = base64.b64encode(image_bytes).decode("utf-8") - image_data[f"{filename}"] = image_base64 + try: + start_time = time.time() + + # Convert PDF using marker + full_text, images, out_meta = convert_single_pdf( + pdf_file, + model_list, + max_pages=None, + langs=None, + batch_multiplier=1, + start_page=None + ) + + end_time = time.time() + processing_time = end_time - start_time + + logger.info(f"PDF processing completed in {processing_time:.2f} seconds") + + # Process images if extraction is enabled + image_data = {} + if extract_images and images: + for i, image in enumerate(images): + if image is not None: + try: + # Check if image is already a PIL Image or needs conversion + from PIL import Image as PILImage + import io + + if isinstance(image, str): + # If it's a string, it might be a base64 encoded image or file path + logger.warning(f"Image {i} is a string: {type(image)}") + continue + elif hasattr(image, 'save'): + # It's a PIL Image object + buffer = io.BytesIO() + image.save(buffer, format='PNG') + img_str = base64.b64encode(buffer.getvalue()).decode() + image_data[f"image_{i}"] = img_str + else: + logger.warning(f"Unknown image type for image {i}: {type(image)}") + continue + except Exception as img_error: + logger.error(f"Error processing image {i}: {str(img_error)}") + continue + + return full_text, out_meta, image_data, processing_time + + except Exception as e: + logger.error(f"Error processing PDF: {str(e)}") + raise - # Remove the temporary image file - os.remove(filename) - return full_text, out_meta, image_data - - -# Function to process a single PDF file -def process_pdf_file(file_content: bytes, filename: str, model_list): +def process_pdf_file(pdf_file: bytes, filename: str = None, model_list=None, extract_images: bool = True): """ - Function to process a single PDF file. - + Process a PDF file and return markdown text with metadata. + Args: - file_content (bytes): The content of the PDF file. - filename (str): The name of the PDF file. - model_list: The list of loaded models. - + pdf_file (bytes): The PDF file content + filename (str): The original filename (optional) + model_list: Pre-loaded models + extract_images (bool): Whether to extract images + Returns: - dict: A dictionary containing the filename, markdown text, metadata, image data, status, and processing time. + dict: Processing results matching PDFConversionResult schema """ - entry_time = time.time() - logger.info(f"Entry time for {filename}: {entry_time}") - markdown_text, metadata, image_data = parse_pdf_and_return_markdown( - file_content, extract_images=True, model_list=model_list - ) - completion_time = time.time() - logger.info(f"Model processes complete time for {filename}: {completion_time}") - time_difference = completion_time - entry_time - return { - "filename": filename, - "markdown": markdown_text, - "metadata": metadata, - "images": image_data, - "status": "ok", - "time": time_difference, - } + try: + full_text, metadata, images, processing_time = parse_pdf_and_return_markdown( + pdf_file, extract_images, model_list + ) + + # Convert metadata to match GeneralMetadata schema + general_metadata = { + "languages": metadata.get("languages", None), + "toc": metadata.get("toc", None), + "pages": metadata.get("pages", None), + "custom_metadata": { + "processing_time": processing_time, + **metadata + } + } + + return { + "filename": filename or "document.pdf", + "markdown": full_text, + "metadata": general_metadata, + "images": images, + "status": "success" + } + + except Exception as e: + logger.error(f"Error in process_pdf_file: {str(e)}") + return { + "filename": filename or "document.pdf", + "markdown": "", + "metadata": { + "languages": None, + "toc": None, + "pages": None, + "custom_metadata": {"error": str(e)} + }, + "images": {}, + "status": "error" + } From d65f1f602cb00433d55cef29f9983a814bcf1be9 Mon Sep 17 00:00:00 2001 From: kaan-playbook Date: Fri, 20 Jun 2025 12:59:58 +0200 Subject: [PATCH 02/16] Update server.py --- server.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/server.py b/server.py index 81cd9df..a20fceb 100644 --- a/server.py +++ b/server.py @@ -13,14 +13,14 @@ from marker_api.utils import print_markerapi_text_art from contextlib import asynccontextmanager import logging -import gradio as gr +# import gradio as gr from marker_api.model.schema import ( BatchConversionResponse, ConversionResponse, HealthResponse, ServerType, ) -from marker_api.demo import demo_ui +# from marker_api.demo import demo_ui # Initialize logging configure_logging() @@ -52,16 +52,23 @@ async def lifespan(app: FastAPI): allow_credentials=True, ) -app = gr.mount_gradio_app(app, demo_ui, path="") +# app = gr.mount_gradio_app(app, demo_ui, path="") @app.get("/health", response_model=HealthResponse) -def server(): +def health_check(): """ Root endpoint to check server status. """ return HealthResponse(message="Welcome to Marker-api", type=ServerType.simple) +@app.get("/") +def root(): + """ + Root endpoint. + """ + return {"message": "Marker API is running", "status": "ok"} + # Endpoint to convert a single PDF to markdown @app.post("/convert", response_model=ConversionResponse) From 5310519dcaf8f497b9024010c45d86242612654f Mon Sep 17 00:00:00 2001 From: kaan-playbook Date: Fri, 20 Jun 2025 17:51:44 +0200 Subject: [PATCH 03/16] upgrade the internal marker to 1.6.2 and 1.7.5 --- distributed_server.py | 58 ++++++-- docker/Dockerfile.cpu.distributed-server | 53 +++++++- docker/Dockerfile.cpu.server | 78 ++++++++++- docker/Dockerfile.cpu.v1.7.5 | 84 ++++++++++++ docker/Dockerfile.gpu.distributed-server | 53 ++++++-- docker/Dockerfile.gpu.server | 53 ++++++-- marker_api/model/schema.py | 14 +- marker_api/routes.py | 149 ++++++++++++-------- marker_api/tasks.py | 166 +++++++++++++++++++++++ pyproject.toml | 3 +- server.py | 146 ++++++++++++++------ 11 files changed, 723 insertions(+), 134 deletions(-) create mode 100644 docker/Dockerfile.cpu.v1.7.5 create mode 100644 marker_api/tasks.py diff --git a/distributed_server.py b/distributed_server.py index 07ff93b..85864df 100644 --- a/distributed_server.py +++ b/distributed_server.py @@ -1,7 +1,7 @@ import argparse import uvicorn import logging -from fastapi import FastAPI, UploadFile, File +from fastapi import FastAPI, UploadFile, File, Form, HTTPException from celery.exceptions import TimeoutError from fastapi.middleware.cors import CORSMiddleware from marker_api.celery_worker import celery_app @@ -24,15 +24,21 @@ ConversionResponse, HealthResponse, ServerType, + PDFConversionResult, ) from typing import List +import os # Initialize logging configure_logging() logger = logging.getLogger(__name__) # Global variable to hold model list -app = FastAPI() +app = FastAPI( + title="Marker API Distributed Server", + description="Distributed API for converting documents (PDF, DOCX, PPTX, XLSX, HTML, EPUB) to Markdown using marker", + version="1.6.2" +) logger.info("Configuring CORS middleware") app.add_middleware( @@ -58,6 +64,7 @@ def server(): message="Welcome to Marker-api", type=server_type, workers=worker_count if server_type == ServerType.distributed else None, + supported_formats=["PDF", "DOCX", "PPTX", "XLSX", "HTML", "EPUB"] ) @@ -73,26 +80,61 @@ def is_celery_alive() -> bool: return False +def validate_file(file: UploadFile) -> str: + """Validate uploaded file and return file extension""" + # Check file size (limit to 50MB) + if file.size and file.size > 50 * 1024 * 1024: + raise HTTPException(status_code=413, detail="File too large. Maximum size is 50MB.") + + # Check file type + allowed_extensions = {'.pdf', '.docx', '.pptx', '.xlsx', '.html', '.epub'} + file_extension = os.path.splitext(file.filename)[1].lower() if file.filename else '' + + if file_extension not in allowed_extensions: + raise HTTPException( + status_code=400, + detail=f"Unsupported file type: {file_extension}. Supported types: {', '.join(allowed_extensions)}" + ) + + return file_extension + + def setup_routes(app: FastAPI, celery_live: bool): logger.info("Setting up routes") if celery_live: logger.info("Adding Celery routes") @app.post("/convert", response_model=ConversionResponse) - async def convert_pdf(pdf_file: UploadFile = File(...)): - return await celery_convert_pdf_concurrent_await(pdf_file) + async def convert_document( + file: UploadFile = File(...), + extract_images: bool = Form(True) + ): + """Convert a document (PDF, DOCX, PPTX, XLSX, HTML, EPUB) to markdown""" + validate_file(file) + return await celery_convert_pdf_concurrent_await(file, extract_images) @app.post("/celery/convert", response_model=CeleryTaskResponse) - async def celery_convert(pdf_file: UploadFile = File(...)): - return await celery_convert_pdf(pdf_file) + async def celery_convert_document( + file: UploadFile = File(...), + extract_images: bool = Form(True) + ): + """Submit a document conversion task to Celery queue""" + validate_file(file) + return await celery_convert_pdf(file, extract_images) @app.get("/celery/result/{task_id}", response_model=CeleryResultResponse) async def get_celery_result(task_id: str): return await celery_result(task_id) @app.post("/batch_convert", response_model=BatchConversionResponse) - async def batch_convert(pdf_files: List[UploadFile] = File(...)): - return await celery_batch_convert(pdf_files) + async def batch_convert( + files: List[UploadFile] = File(...), + extract_images: bool = Form(True) + ): + """Convert multiple documents to markdown""" + for file in files: + validate_file(file) + return await celery_batch_convert(files, extract_images) @app.get("/batch_convert/result/{task_id}", response_model=BatchResultResponse) async def get_batch_result(task_id: str): diff --git a/docker/Dockerfile.cpu.distributed-server b/docker/Dockerfile.cpu.distributed-server index 09e94ed..2d08006 100644 --- a/docker/Dockerfile.cpu.distributed-server +++ b/docker/Dockerfile.cpu.distributed-server @@ -1,21 +1,64 @@ # Use a base image with Python FROM python:3.11-slim -# Install system dependencies required for libGL +# Install system dependencies required for marker-pdf 1.6.2 and WeasyPrint RUN apt-get update && apt-get install -y \ + # Core system dependencies libgl1-mesa-glx \ libglib2.0-0 \ + libgomp1 \ + libgcc-s1 \ + # WeasyPrint dependencies for DOCX support + libgobject-2.0-0 \ + libcairo2 \ + libpango-1.0-0 \ + libpangocairo-1.0-0 \ + libgdk-pixbuf2.0-0 \ + libffi-dev \ + shared-mime-info \ + # LibreOffice for PDF generation (optional) + libreoffice \ + # Additional utilities + curl \ + wget \ && rm -rf /var/lib/apt/lists/* # Set the working directory WORKDIR /app # Copy the project files into the Docker image -COPY ../ ./ +COPY . ./ -# Install Python dependencies -RUN pip install -e . +# Install Python dependencies with updated marker-pdf version +RUN pip install --no-cache-dir \ + fastapi==0.115.0 \ + uvicorn==0.31.1 \ + "celery[redis]==5.3.6" \ + flower==2.0.1 \ + hf-transfer==0.1.8 \ + huggingface-hub==0.25.1 \ + locust==2.31.8 \ + python-multipart==0.0.12 \ + redis==5.1.1 \ + requests==2.32.3 \ + rich==13.9.2 \ + pynvml==11.5.3 \ + art==6.3 \ + marker-pdf==1.6.2 \ + "transformers>=4.36.0,<4.45.0" \ + "gradio>=4.44.0,<5.0.0" \ + pillow \ + torch \ + torchvision \ + # WeasyPrint and dependencies for DOCX support + weasyprint \ + cffi \ + pycairo \ + # Additional dependencies for marker-pdf 1.6.2 + pypdf \ + pdfplumber -RUN python -c 'from marker.models import load_all_models; load_all_models()' +# Pre-load models to speed up container startup using new API +RUN python -c 'from marker.models import create_model_dict; create_model_dict()' EXPOSE 8080 diff --git a/docker/Dockerfile.cpu.server b/docker/Dockerfile.cpu.server index 879eb04..ab57a58 100644 --- a/docker/Dockerfile.cpu.server +++ b/docker/Dockerfile.cpu.server @@ -1,22 +1,90 @@ # Use a base image with Python FROM python:3.11-slim -# Install system dependencies required for libGL +# Install system dependencies required for marker-pdf 1.6.2 and WeasyPrint RUN apt-get update && apt-get install -y \ + # Core system dependencies libgl1-mesa-glx \ libglib2.0-0 \ + libgomp1 \ + # WeasyPrint dependencies for DOCX support + libglib2.0-dev \ + libcairo2-dev \ + libpango1.0-dev \ + libgdk-pixbuf2.0-dev \ + libffi-dev \ + shared-mime-info \ + # Additional WeasyPrint runtime dependencies + libpango-1.0-0 \ + libharfbuzz0b \ + libpangoft2-1.0-0 \ + libfontconfig1 \ + libcairo2 \ + libgdk-pixbuf2.0-0 \ + fonts-liberation \ + # LibreOffice for PDF generation (optional) + libreoffice \ + # Additional utilities + curl \ + wget \ + build-essential \ + pkg-config \ && rm -rf /var/lib/apt/lists/* # Set the working directory WORKDIR /app # Copy the project files into the Docker image -COPY ../ ./ +COPY . ./ -# Install Python dependencies -RUN pip install -e . +# Install Python dependencies with updated marker-pdf version +RUN pip install --no-cache-dir \ + fastapi==0.115.0 \ + uvicorn==0.31.1 \ + "celery[redis]==5.3.6" \ + flower==2.0.1 \ + hf-transfer==0.1.8 \ + huggingface-hub==0.25.1 \ + locust==2.31.8 \ + python-multipart==0.0.12 \ + redis==5.1.1 \ + requests==2.32.3 \ + rich==13.9.2 \ + pynvml==11.5.3 \ + art==6.3 \ + marker-pdf==1.6.2 \ + "transformers>=4.45.2,<5.0.0" \ + pillow \ + torch \ + torchvision \ + # WeasyPrint and dependencies for DOCX support + weasyprint \ + cffi \ + pycairo \ + # DOCX processing dependency + mammoth \ + # Additional dependencies for marker-pdf 1.6.2 + pypdf \ + pdfplumber -RUN python -c 'from marker.models import load_all_models; load_all_models()' +# Install additional dependencies that might be needed +RUN pip install --no-cache-dir \ + billiard \ + click-didyoumean \ + click-plugins \ + click-repl \ + kombu \ + python-dateutil \ + tzdata \ + vine \ + starlette \ + humanize \ + prometheus-client \ + pytz \ + tornado + +# Pre-load models to speed up container startup using new API +RUN python -c 'from marker.models import create_model_dict; create_model_dict()' EXPOSE 8080 diff --git a/docker/Dockerfile.cpu.v1.7.5 b/docker/Dockerfile.cpu.v1.7.5 new file mode 100644 index 0000000..4a152a3 --- /dev/null +++ b/docker/Dockerfile.cpu.v1.7.5 @@ -0,0 +1,84 @@ +# Use a base image with Python +FROM python:3.11-slim + +# Install system dependencies required for marker-pdf 1.7.5 and WeasyPrint +RUN apt-get update && apt-get install -y \ + libgl1-mesa-glx \ + libglib2.0-0 \ + libgomp1 \ + libglib2.0-dev \ + libcairo2-dev \ + libpango1.0-dev \ + libgdk-pixbuf2.0-dev \ + libffi-dev \ + shared-mime-info \ + libpango-1.0-0 \ + libharfbuzz0b \ + libpangoft2-1.0-0 \ + libfontconfig1 \ + libcairo2 \ + libgdk-pixbuf2.0-0 \ + fonts-liberation \ + libreoffice \ + curl \ + wget \ + build-essential \ + pkg-config \ + && rm -rf /var/lib/apt/lists/* + +# Set the working directory +WORKDIR /app + +# Copy the project files into the Docker image +COPY . ./ + +# Install Python dependencies with marker-pdf 1.7.5 +RUN pip install --no-cache-dir \ + fastapi==0.115.0 \ + uvicorn==0.31.1 \ + python-multipart==0.0.12 \ + requests==2.32.3 \ + "marker-pdf[full]==1.7.5" \ + pillow \ + torch \ + torchvision \ + weasyprint \ + cffi \ + pycairo \ + mammoth \ + pypdf \ + pdfplumber \ + streamlit \ + streamlit-ace + +# Install additional dependencies that might be needed +RUN pip install --no-cache-dir \ + pydantic \ + pydantic-settings \ + python-dotenv \ + rich \ + pynvml \ + psutil \ + # Dependencies from working v1.6.2 Dockerfile + art \ + billiard \ + click-didyoumean \ + click-plugins \ + click-repl \ + kombu \ + python-dateutil \ + tzdata \ + vine \ + starlette \ + humanize \ + prometheus-client \ + pytz \ + tornado + +# Pre-load models to speed up container startup using new API +RUN python -c 'from marker.models import create_model_dict; create_model_dict()' + +EXPOSE 8080 + +# Command to run the server +CMD ["python", "server.py", "--host", "0.0.0.0", "--port", "8080"] \ No newline at end of file diff --git a/docker/Dockerfile.gpu.distributed-server b/docker/Dockerfile.gpu.distributed-server index 403ece8..b602d78 100644 --- a/docker/Dockerfile.gpu.distributed-server +++ b/docker/Dockerfile.gpu.distributed-server @@ -32,20 +32,32 @@ RUN apt-get update && \ git \ python3 \ python3-pip \ + python3-dev \ + # Core system dependencies libgl1 \ libglib2.0-0 \ - curl \ + libgomp1 \ + libgcc-s1 \ + # WeasyPrint dependencies for DOCX support + libgobject-2.0-0 \ + libcairo2 \ + libpango-1.0-0 \ + libpangocairo-1.0-0 \ + libgdk-pixbuf2.0-0 \ + libffi-dev \ + shared-mime-info \ + # LibreOffice for PDF generation (optional) + libreoffice \ + # Additional utilities gnupg2 \ ca-certificates \ apt-transport-https \ software-properties-common \ - libreoffice \ ffmpeg \ git-lfs \ xvfb \ + python3-packaging \ && ln -s /usr/bin/python3 /usr/bin/python \ - && apt-get update \ - && apt install python3-packaging \ && rm -rf /var/lib/apt/lists/* RUN pip3 install --no-cache-dir torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cu118 @@ -53,11 +65,36 @@ RUN pip3 install --no-cache-dir torch torchvision torchaudio --index-url https:/ WORKDIR /app # Copy the project files into the Docker image -COPY ../ ./ +COPY . ./ -# Install Python dependencies -RUN pip3 install --no-cache-dir -e . +# Install Python dependencies with updated marker-pdf version +RUN pip3 install --no-cache-dir \ + fastapi==0.115.0 \ + uvicorn==0.31.1 \ + "celery[redis]==5.3.6" \ + flower==2.0.1 \ + hf-transfer==0.1.8 \ + huggingface-hub==0.25.1 \ + locust==2.31.8 \ + python-multipart==0.0.12 \ + redis==5.1.1 \ + requests==2.32.3 \ + rich==13.9.2 \ + pynvml==11.5.3 \ + art==6.3 \ + marker-pdf==1.6.2 \ + "transformers>=4.36.0,<4.45.0" \ + "gradio>=4.44.0,<5.0.0" \ + pillow \ + # WeasyPrint and dependencies for DOCX support + weasyprint \ + cffi \ + pycairo \ + # Additional dependencies for marker-pdf 1.6.2 + pypdf \ + pdfplumber -RUN python -c 'from marker.models import load_all_models; load_all_models()' +# Pre-load models to speed up container startup using new API +RUN python -c 'from marker.models import create_model_dict; create_model_dict()' EXPOSE 8080 diff --git a/docker/Dockerfile.gpu.server b/docker/Dockerfile.gpu.server index d63d9ea..b689c8f 100644 --- a/docker/Dockerfile.gpu.server +++ b/docker/Dockerfile.gpu.server @@ -32,20 +32,32 @@ RUN apt-get update && \ git \ python3 \ python3-pip \ + python3-dev \ + # Core system dependencies libgl1 \ libglib2.0-0 \ - curl \ + libgomp1 \ + libgcc-s1 \ + # WeasyPrint dependencies for DOCX support + libgobject-2.0-0 \ + libcairo2 \ + libpango-1.0-0 \ + libpangocairo-1.0-0 \ + libgdk-pixbuf2.0-0 \ + libffi-dev \ + shared-mime-info \ + # LibreOffice for PDF generation (optional) + libreoffice \ + # Additional utilities gnupg2 \ ca-certificates \ apt-transport-https \ software-properties-common \ - libreoffice \ ffmpeg \ git-lfs \ xvfb \ + python3-packaging \ && ln -s /usr/bin/python3 /usr/bin/python \ - && apt-get update \ - && apt install python3-packaging \ && rm -rf /var/lib/apt/lists/* RUN pip3 install --no-cache-dir torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cu118 @@ -53,12 +65,37 @@ RUN pip3 install --no-cache-dir torch torchvision torchaudio --index-url https:/ WORKDIR /app # Copy the project files into the Docker image -COPY ../ ./ +COPY . ./ -# Install Python dependencies -RUN pip3 install --no-cache-dir -e . +# Install Python dependencies with updated marker-pdf version +RUN pip3 install --no-cache-dir \ + fastapi==0.115.0 \ + uvicorn==0.31.1 \ + "celery[redis]==5.3.6" \ + flower==2.0.1 \ + hf-transfer==0.1.8 \ + huggingface-hub==0.25.1 \ + locust==2.31.8 \ + python-multipart==0.0.12 \ + redis==5.1.1 \ + requests==2.32.3 \ + rich==13.9.2 \ + pynvml==11.5.3 \ + art==6.3 \ + marker-pdf==1.6.2 \ + "transformers>=4.36.0,<4.45.0" \ + "gradio>=4.44.0,<5.0.0" \ + pillow \ + # WeasyPrint and dependencies for DOCX support + weasyprint \ + cffi \ + pycairo \ + # Additional dependencies for marker-pdf 1.6.2 + pypdf \ + pdfplumber -RUN python -c 'from marker.models import load_all_models; load_all_models()' +# Pre-load models to speed up container startup using new API +RUN python -c 'from marker.models import create_model_dict; create_model_dict()' EXPOSE 8080 diff --git a/marker_api/model/schema.py b/marker_api/model/schema.py index d090515..52bd7a8 100644 --- a/marker_api/model/schema.py +++ b/marker_api/model/schema.py @@ -68,8 +68,7 @@ class CeleryResultResponse(BaseModel): class BatchConversionResponse(BaseModel): - task_id: str - status: str + results: List[PDFConversionResult] class BatchResultResponse(BaseModel): @@ -79,3 +78,14 @@ class BatchResultResponse(BaseModel): completed: Optional[int] = None total: Optional[int] = None progress: Optional[str] = None + + +class TaskResponse(BaseModel): + task_id: str + status: str + + +class TaskStatusResponse(BaseModel): + task_id: str + status: str + result: Optional[Union[PDFConversionResult, str, Dict[str, Any]]] = None diff --git a/marker_api/routes.py b/marker_api/routes.py index 146f8f6..fb39743 100644 --- a/marker_api/routes.py +++ b/marker_api/routes.py @@ -1,8 +1,10 @@ import os import time import base64 -from marker.convert import convert_single_pdf -from marker.models import load_all_models +import tempfile +from marker.converters.pdf import PdfConverter +from marker.models import create_model_dict +from marker.output import text_from_rendered from marker.logger import configure_logging import logging @@ -11,15 +13,15 @@ logger = logging.getLogger(__name__) -# Function to parse PDF and return markdown, metadata, and image data -def parse_pdf_and_return_markdown(pdf_file: bytes, extract_images: bool, model_list): +def parse_document_and_return_markdown(file_bytes: bytes, filename: str, extract_images: bool, model_dict): """ - Function to parse a PDF and extract text and images. + Function to parse a document (PDF, DOCX, etc.) and extract text and images. Args: - pdf_file (bytes): The content of the PDF file. + file_bytes (bytes): The content of the document file. + filename (str): Original filename to determine file type extract_images (bool): Whether to extract images or not. - model_list: The model list from load_all_models() + model_dict: The model dictionary from create_model_dict() Returns: tuple: A tuple containing the full text, metadata, and image data. @@ -27,71 +29,87 @@ def parse_pdf_and_return_markdown(pdf_file: bytes, extract_images: bool, model_l try: start_time = time.time() - # Convert PDF using marker - full_text, images, out_meta = convert_single_pdf( - pdf_file, - model_list, - max_pages=None, - langs=None, - batch_multiplier=1, - start_page=None - ) - - end_time = time.time() - processing_time = end_time - start_time + # Create a temporary file with the correct extension + file_extension = os.path.splitext(filename)[1].lower() if filename else '.pdf' - logger.info(f"PDF processing completed in {processing_time:.2f} seconds") + with tempfile.NamedTemporaryFile(suffix=file_extension, delete=False) as temp_file: + temp_file.write(file_bytes) + temp_file_path = temp_file.name - # Process images if extraction is enabled - image_data = {} - if extract_images and images: - for i, image in enumerate(images): - if image is not None: - try: - # Check if image is already a PIL Image or needs conversion - from PIL import Image as PILImage - import io - - if isinstance(image, str): - # If it's a string, it might be a base64 encoded image or file path - logger.warning(f"Image {i} is a string: {type(image)}") + try: + # Create converter with model dictionary - PdfConverter handles all formats in 1.6.2 + converter = PdfConverter( + artifact_dict=model_dict, + ) + + # Convert document using the new API - pass the file path directly + rendered = converter(temp_file_path) + + # Extract text and metadata from rendered output + full_text, images, out_meta = text_from_rendered(rendered) + + end_time = time.time() + processing_time = end_time - start_time + + logger.info(f"Document processing completed in {processing_time:.2f} seconds") + logger.info(f"Extracted text length: {len(full_text)} characters") + logger.info(f"Images extracted: {len(images) if images else 0}") + + # Process images if extraction is enabled + image_data = {} + if extract_images and images: + # Handle different types of images return value + if isinstance(images, dict): + for image_name, image_data_bytes in images.items(): + try: + if isinstance(image_data_bytes, bytes): + img_str = base64.b64encode(image_data_bytes).decode() + image_data[image_name] = img_str + else: + logger.warning(f"Unexpected image data type for {image_name}: {type(image_data_bytes)}") + except Exception as img_error: + logger.error(f"Error processing image {image_name}: {str(img_error)}") continue - elif hasattr(image, 'save'): - # It's a PIL Image object - buffer = io.BytesIO() - image.save(buffer, format='PNG') - img_str = base64.b64encode(buffer.getvalue()).decode() - image_data[f"image_{i}"] = img_str - else: - logger.warning(f"Unknown image type for image {i}: {type(image)}") - continue - except Exception as img_error: - logger.error(f"Error processing image {i}: {str(img_error)}") - continue - - return full_text, out_meta, image_data, processing_time + else: + logger.info(f"Images returned as {type(images)}: {images}") + + return full_text, out_meta, image_data, processing_time + + finally: + # Clean up temporary file + if os.path.exists(temp_file_path): + os.unlink(temp_file_path) except Exception as e: - logger.error(f"Error processing PDF: {str(e)}") + logger.error(f"Error processing document: {str(e)}") raise -def process_pdf_file(pdf_file: bytes, filename: str = None, model_list=None, extract_images: bool = True): +def process_document_file(file_bytes: bytes, filename: str = None, model_dict=None, extract_images: bool = True): """ - Process a PDF file and return markdown text with metadata. + Process a document file (PDF, DOCX, etc.) and return markdown text with metadata. Args: - pdf_file (bytes): The PDF file content + file_bytes (bytes): The document file content filename (str): The original filename (optional) - model_list: Pre-loaded models + model_dict: Pre-loaded model dictionary extract_images (bool): Whether to extract images Returns: dict: Processing results matching PDFConversionResult schema """ try: - full_text, metadata, images, processing_time = parse_pdf_and_return_markdown( - pdf_file, extract_images, model_list + # Determine file type from filename + file_extension = os.path.splitext(filename)[1].lower() if filename else '.pdf' + supported_extensions = ['.pdf', '.docx', '.pptx', '.xlsx', '.html', '.epub'] + + if file_extension not in supported_extensions: + raise ValueError(f"Unsupported file format: {file_extension}. Supported formats: {supported_extensions}") + + logger.info(f"Processing {file_extension} file: {filename}") + + full_text, metadata, images, processing_time = parse_document_and_return_markdown( + file_bytes, filename, extract_images, model_dict ) # Convert metadata to match GeneralMetadata schema @@ -101,20 +119,26 @@ def process_pdf_file(pdf_file: bytes, filename: str = None, model_list=None, ext "pages": metadata.get("pages", None), "custom_metadata": { "processing_time": processing_time, + "file_type": file_extension, + "text_length": len(full_text), + "images_count": len(images), **metadata } } - return { - "filename": filename or "document.pdf", + result = { + "filename": filename or f"document{file_extension}", "markdown": full_text, "metadata": general_metadata, "images": images, "status": "success" } + logger.info(f"Successfully processed {filename}: {len(full_text)} chars, {len(images)} images") + return result + except Exception as e: - logger.error(f"Error in process_pdf_file: {str(e)}") + logger.error(f"Error in process_document_file: {str(e)}") return { "filename": filename or "document.pdf", "markdown": "", @@ -122,8 +146,17 @@ def process_pdf_file(pdf_file: bytes, filename: str = None, model_list=None, ext "languages": None, "toc": None, "pages": None, - "custom_metadata": {"error": str(e)} + "custom_metadata": {"error": str(e), "file_type": file_extension if filename else ".pdf"} }, "images": {}, "status": "error" } + + +# Keep the old function for backward compatibility +def process_pdf_file(pdf_file: bytes, filename: str = None, model_dict=None, extract_images: bool = True): + """ + Process a PDF file and return markdown text with metadata. + Wrapper around process_document_file for backward compatibility. + """ + return process_document_file(pdf_file, filename, model_dict, extract_images) diff --git a/marker_api/tasks.py b/marker_api/tasks.py new file mode 100644 index 0000000..11d020e --- /dev/null +++ b/marker_api/tasks.py @@ -0,0 +1,166 @@ +import os +from celery import Celery +from marker.models import create_model_dict +from marker_api.routes import process_document_file +import logging + +# Initialize logging +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) + +# Get Redis URL from environment variable +REDIS_URL = os.getenv("REDIS_URL", "redis://localhost:6379/0") + +# Initialize Celery +celery_app = Celery( + "marker_api", + broker=REDIS_URL, + backend=REDIS_URL +) + +# Celery configuration +celery_app.conf.update( + task_serializer="json", + accept_content=["json"], + result_serializer="json", + timezone="UTC", + enable_utc=True, + task_track_started=True, + task_time_limit=30 * 60, # 30 minutes + task_soft_time_limit=25 * 60, # 25 minutes + worker_prefetch_multiplier=1, + worker_max_tasks_per_child=1000, +) + +# Global variable to hold model dictionary +model_dict = None + +def load_models(): + """Load marker models once per worker""" + global model_dict + if model_dict is None: + logger.info("Loading marker models...") + model_dict = create_model_dict() + logger.info("Models loaded successfully!") + return model_dict + +@celery_app.task(bind=True) +def process_document_task(self, file_content: bytes, filename: str): + """ + Celery task to process a document (PDF, DOCX, etc.) and convert it to markdown. + + Args: + file_content (bytes): The document file content + filename (str): The original filename + + Returns: + dict: Processing results + """ + try: + # Update task state to PROGRESS + self.update_state( + state="PROGRESS", + meta={"status": "Loading models and processing document..."} + ) + + # Load models if not already loaded + models = load_models() + + # Update task state + self.update_state( + state="PROGRESS", + meta={"status": f"Converting {filename}..."} + ) + + # Process the document + result = process_document_file( + file_content, + filename=filename, + model_dict=models, + extract_images=True + ) + + logger.info(f"Successfully processed document: {filename}") + return result + + except Exception as e: + logger.error(f"Error processing document {filename}: {str(e)}") + self.update_state( + state="FAILURE", + meta={"error": str(e), "filename": filename} + ) + raise + +@celery_app.task(bind=True) +def process_batch_documents_task(self, files_data: list): + """ + Celery task to process multiple documents in batch. + + Args: + files_data (list): List of dictionaries with 'content' and 'filename' keys + + Returns: + list: List of processing results + """ + try: + # Update task state to PROGRESS + self.update_state( + state="PROGRESS", + meta={"status": "Loading models..."} + ) + + # Load models if not already loaded + models = load_models() + + results = [] + total_files = len(files_data) + + for i, file_data in enumerate(files_data): + try: + # Update progress + self.update_state( + state="PROGRESS", + meta={ + "status": f"Processing file {i+1}/{total_files}: {file_data['filename']}", + "current": i+1, + "total": total_files + } + ) + + # Process the document + result = process_document_file( + file_data['content'], + filename=file_data['filename'], + model_dict=models, + extract_images=True + ) + + results.append(result) + logger.info(f"Successfully processed document {i+1}/{total_files}: {file_data['filename']}") + + except Exception as e: + logger.error(f"Error processing document {file_data['filename']}: {str(e)}") + error_result = { + "filename": file_data['filename'], + "markdown": "", + "metadata": { + "languages": None, + "toc": None, + "pages": None, + "custom_metadata": {"error": str(e)} + }, + "images": {}, + "status": "error" + } + results.append(error_result) + + logger.info(f"Batch processing completed. Processed {len(results)} files.") + return results + + except Exception as e: + logger.error(f"Error in batch processing: {str(e)}") + self.update_state( + state="FAILURE", + meta={"error": str(e)} + ) + raise \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index 8d19e0e..c7720a8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -19,10 +19,9 @@ python-multipart = "^0.0.12" redis = "^5.1.1" requests = "^2.32.3" rich = "^13.9.2" -marker-pdf = "^0.2.17" +marker-pdf = "1.6.2" pynvml = "^11.5.3" art = "^6.3" -gradio = "^5.1.0" diff --git a/server.py b/server.py index a20fceb..faebaff 100644 --- a/server.py +++ b/server.py @@ -1,14 +1,14 @@ import os import asyncio import argparse -from fastapi import FastAPI, UploadFile, File +from fastapi import FastAPI, UploadFile, File, Form, HTTPException from fastapi.middleware.cors import CORSMiddleware from typing import List import concurrent.futures from marker.logger import configure_logging # Import logging configuration -from marker.models import load_all_models # Import function to load models +from marker.models import create_model_dict # Import function to create model dict from marker_api.routes import ( - process_pdf_file, + process_document_file, # Updated to handle all document types ) from marker_api.utils import print_markerapi_text_art from contextlib import asynccontextmanager @@ -19,6 +19,7 @@ ConversionResponse, HealthResponse, ServerType, + PDFConversionResult, ) # from marker_api.demo import demo_ui @@ -26,22 +27,29 @@ configure_logging() logger = logging.getLogger(__name__) -# Global variable to hold model list -model_list = None +# Global variable to hold model dict +model_dict = None # Event that runs on startup to load all models @asynccontextmanager async def lifespan(app: FastAPI): - global model_list + global model_dict logger.debug("--------------------- Loading OCR Model -----------------------") print_markerapi_text_art() - model_list = load_all_models() + model_dict = create_model_dict() yield # Initialize FastAPI app -app = FastAPI(lifespan=lifespan) +app = FastAPI( + title="Marker API", + description="Convert PDF to Markdown with marker-pdf", + version="1.7.5", + docs_url="/docs", + redoc_url="/redoc", + lifespan=lifespan +) # Add CORS middleware to allow cross-origin requests app.add_middleware( @@ -55,56 +63,118 @@ async def lifespan(app: FastAPI): # app = gr.mount_gradio_app(app, demo_ui, path="") -@app.get("/health", response_model=HealthResponse) -def health_check(): - """ - Root endpoint to check server status. - """ - return HealthResponse(message="Welcome to Marker-api", type=ServerType.simple) +@app.get("/health") +async def health_check(): + return {"message": "Marker API is running", "status": "ok", "version": "1.7.5"} @app.get("/") -def root(): - """ - Root endpoint. - """ - return {"message": "Marker API is running", "status": "ok"} +async def root(): + return {"message": "Welcome to Marker-api", "version": "1.7.5"} -# Endpoint to convert a single PDF to markdown -@app.post("/convert", response_model=ConversionResponse) -async def convert_pdf_to_markdown(pdf_file: UploadFile): +# Endpoint to convert a single document to markdown +@app.post("/convert", response_model=PDFConversionResult) +async def convert_document_to_markdown( + file: UploadFile = File(...), + extract_images: bool = Form(True) +): """ - Endpoint to convert a single PDF to markdown. + Endpoint to convert a document (PDF, DOCX, PPTX, XLSX, HTML, EPUB) to markdown. """ - logger.debug(f"Received file: {pdf_file.filename}") - file = await pdf_file.read() - response = process_pdf_file(file, pdf_file.filename, model_list) - return ConversionResponse(status="Success", result=response) - - -# Endpoint to convert multiple PDFs to markdown + global model_dict + + if model_dict is None: + raise HTTPException(status_code=503, detail="Models not loaded yet") + + logger.debug(f"Received file: {file.filename}") + + # Check file size (limit to 50MB) + if file.size and file.size > 50 * 1024 * 1024: + raise HTTPException(status_code=413, detail="File too large. Maximum size is 50MB.") + + # Check file type + allowed_extensions = {'.pdf', '.docx', '.pptx', '.xlsx', '.html', '.epub'} + file_extension = os.path.splitext(file.filename)[1].lower() if file.filename else '' + + if file_extension not in allowed_extensions: + raise HTTPException( + status_code=400, + detail=f"Unsupported file type: {file_extension}. Supported types: {', '.join(allowed_extensions)}" + ) + + try: + file_content = await file.read() + response = process_document_file( + file_content, + filename=file.filename, + model_dict=model_dict, + extract_images=extract_images + ) + return PDFConversionResult(**response) + except Exception as e: + logger.error(f"Error processing document: {str(e)}") + raise HTTPException(status_code=500, detail=f"Error processing document: {str(e)}") + + +# Endpoint to convert multiple documents to markdown @app.post("/batch_convert", response_model=BatchConversionResponse) -async def convert_pdfs_to_markdown(pdf_files: List[UploadFile] = File(...)): +async def convert_documents_to_markdown( + files: List[UploadFile] = File(...), + extract_images: bool = Form(True) +): """ - Endpoint to convert multiple PDFs to markdown. + Endpoint to convert multiple documents to markdown. """ - logger.debug(f"Received {len(pdf_files)} files for batch conversion") - - async def process_files(files): + global model_dict + + if model_dict is None: + raise HTTPException(status_code=503, detail="Models not loaded yet") + + logger.debug(f"Received {len(files)} files for batch conversion") + + async def process_files(file_list): loop = asyncio.get_event_loop() with concurrent.futures.ThreadPoolExecutor(max_workers=2) as pool: coroutines = [ loop.run_in_executor( - pool, process_pdf_file, await file.read(), file.filename, model_list + pool, + process_document_file, + await file.read(), + file.filename, + model_dict, + extract_images ) - for file in files + for file in file_list ] return await asyncio.gather(*coroutines) - responses = await process_files(pdf_files) + responses = await process_files(files) return BatchConversionResponse(results=responses) +@app.post("/extract_structured") +async def extract_structured_data( + file: UploadFile = File(...), + schema: str = Form(None) +): + """ + Extract structured data from documents using Pydantic schemas (v1.7.5 feature) + """ + try: + # For now, return a placeholder response indicating the feature is available + # In a full implementation, this would use marker-pdf 1.7.5's structured extraction + return { + "message": "Structured extraction endpoint available in v1.7.5", + "filename": file.filename, + "feature": "structured_extraction", + "status": "placeholder_implementation", + "note": "This would extract structured data using Pydantic schemas in the full implementation" + } + except Exception as e: + logger.error(f"Structured extraction failed: {str(e)}") + raise HTTPException(status_code=500, detail=f"Structured extraction failed: {str(e)}") + + # Main function to run the server def main(): parser = argparse.ArgumentParser(description="Run the marker-api server.") From 8839dafe2ef4d742a6ba9b8adb0abb72d23d2619 Mon Sep 17 00:00:00 2001 From: kaan-playbook Date: Thu, 26 Jun 2025 18:36:16 +0200 Subject: [PATCH 04/16] working setup with marker --- distributed_server.py | 31 ++- docker-compose.distributed.yml | 78 ++++++ docker-compose.original.yml | 12 + docker/Dockerfile.base | 51 ++++ ...Dockerfile.original.cpu.distributed-server | 9 + docker/Dockerfile.original.cpu.server | 43 ++++ marker_api/celery_routes.py | 49 +--- marker_api/celery_tasks.py | 222 ++++++++++++++---- marker_api/celery_worker.py | 23 +- marker_api/routes.py | 43 ++-- pyproject.toml | 36 ++- server.py | 64 ++--- 12 files changed, 487 insertions(+), 174 deletions(-) create mode 100644 docker-compose.distributed.yml create mode 100644 docker-compose.original.yml create mode 100644 docker/Dockerfile.base create mode 100644 docker/Dockerfile.original.cpu.distributed-server create mode 100644 docker/Dockerfile.original.cpu.server diff --git a/distributed_server.py b/distributed_server.py index 85864df..6b3c375 100644 --- a/distributed_server.py +++ b/distributed_server.py @@ -14,8 +14,8 @@ celery_batch_convert, celery_batch_result, ) -import gradio as gr -from marker_api.demo import demo_ui +# import gradio as gr +# from marker_api.demo import demo_ui from marker_api.model.schema import ( BatchConversionResponse, BatchResultResponse, @@ -70,14 +70,23 @@ def server(): def is_celery_alive() -> bool: logger.debug("Checking if Celery is alive") - try: - result = celery_app.send_task("celery.ping") - result.get(timeout=3) - logger.info("Celery is alive") - return True - except (TimeoutError, Exception) as e: - logger.warning(f"Celery is not responding: {str(e)}") - return False + max_retries = 3 + timeout = 10 # seconds + + for attempt in range(max_retries): + try: + result = celery_app.send_task("celery.ping") + result.get(timeout=timeout) + logger.info("Celery is alive") + return True + except (TimeoutError, Exception) as e: + logger.warning(f"Celery connection attempt {attempt + 1}/{max_retries} failed: {str(e)}") + if attempt < max_retries - 1: + import time + time.sleep(2) # Wait 2 seconds before retrying + + logger.error("All Celery connection attempts failed") + return False def validate_file(file: UploadFile) -> str: @@ -143,7 +152,7 @@ async def get_batch_result(task_id: str): logger.info("Adding real-time conversion route") else: logger.warning("Celery routes not added as Celery is not alive") - app = gr.mount_gradio_app(app, demo_ui, path="") + # app = gr.mount_gradio_app(app, demo_ui, path="") def parse_args(): diff --git a/docker-compose.distributed.yml b/docker-compose.distributed.yml new file mode 100644 index 0000000..7590cc8 --- /dev/null +++ b/docker-compose.distributed.yml @@ -0,0 +1,78 @@ +version: "3.8" + +services: + base: + image: marker-api-base:latest + build: + context: . + dockerfile: docker/Dockerfile.base + + celery_worker: + build: + context: . + dockerfile: docker/Dockerfile.original.cpu.distributed-server + image: marker-api-distributed-cpu + command: celery -A marker_api.celery_worker.celery_app worker --pool=solo -n worker@%h --loglevel=info + volumes: + - .:/app + - model_cache:/home/marker/.cache/huggingface + environment: + - REDIS_HOST=redis + depends_on: + - redis + - base + + api: + container_name: marker-api-distributed + build: + context: . + dockerfile: docker/Dockerfile.original.cpu.distributed-server + command: python distributed_server.py --host 0.0.0.0 --port 8080 + ports: + - "8086:8080" + volumes: + - .:/app + - model_cache:/home/marker/.cache/huggingface + environment: + - REDIS_HOST=redis + depends_on: + - redis + - base + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:8080/health"] + interval: 30s + timeout: 10s + retries: 3 + + redis: + image: redis:7-alpine + ports: + - "6379:6379" + healthcheck: + test: ["CMD", "redis-cli", "ping"] + interval: 5s + timeout: 3s + retries: 3 + + flower: + build: + context: . + dockerfile: docker/Dockerfile.original.cpu.distributed-server + command: celery -A marker_api.celery_worker.celery_app flower --port=5555 + ports: + - "5555:5555" + volumes: + - .:/app + - model_cache:/home/marker/.cache/huggingface + environment: + - REDIS_HOST=redis + depends_on: + - redis + - base + +volumes: + model_cache: + +networks: + default: + name: marker-api-network \ No newline at end of file diff --git a/docker-compose.original.yml b/docker-compose.original.yml new file mode 100644 index 0000000..b70709b --- /dev/null +++ b/docker-compose.original.yml @@ -0,0 +1,12 @@ +version: "3.8" + +services: + app: + container_name: marker-api-cpu-original + build: + context: . + dockerfile: docker/Dockerfile.original.cpu.server + ports: + - "8080:8080" + volumes: + - .:/app diff --git a/docker/Dockerfile.base b/docker/Dockerfile.base new file mode 100644 index 0000000..84903ef --- /dev/null +++ b/docker/Dockerfile.base @@ -0,0 +1,51 @@ +# Use a base image with Python +FROM python:3.11-slim + +# Install system dependencies required for libGL and DOCX conversion +RUN apt-get update && apt-get install -y \ + libgl1-mesa-glx \ + libglib2.0-0 \ + libglib2.0-dev \ + libcairo2-dev \ + libpango1.0-dev \ + libgdk-pixbuf2.0-dev \ + libffi-dev \ + shared-mime-info \ + libpango-1.0-0 \ + libharfbuzz0b \ + libpangoft2-1.0-0 \ + libfontconfig1 \ + libcairo2 \ + libgdk-pixbuf2.0-0 \ + fonts-liberation \ + build-essential \ + pkg-config \ + gcc \ + && rm -rf /var/lib/apt/lists/* + +# Create a non-root user +RUN useradd -m -u 1000 marker + +# Create font directory for marker user +RUN mkdir -p /home/marker/.local/share/fonts && \ + chown -R marker:marker /home/marker/.local + +# Set the working directory +WORKDIR /app + +# Copy the entire project +COPY . . + +# Install Python dependencies with distributed extras +RUN pip install -e ".[distributed]" + +USER marker + +# Set font path for marker user +ENV FONT_PATH=/home/marker/.local/share/fonts/font.ttf + +# Pre-load models and save them to a shared location +RUN python -c 'from marker.models import create_model_dict; create_model_dict()' + +# The models will be cached in /home/marker/.cache/huggingface +# This directory will be used as a volume in the distributed setup \ No newline at end of file diff --git a/docker/Dockerfile.original.cpu.distributed-server b/docker/Dockerfile.original.cpu.distributed-server new file mode 100644 index 0000000..06c4443 --- /dev/null +++ b/docker/Dockerfile.original.cpu.distributed-server @@ -0,0 +1,9 @@ +# Use the base image that has models pre-loaded +FROM marker-api-base:latest + +# Copy the rest of the project files +COPY . . + +# No need to install dependencies or pre-load models as they're in the base image +# Just expose the port +EXPOSE 8080 \ No newline at end of file diff --git a/docker/Dockerfile.original.cpu.server b/docker/Dockerfile.original.cpu.server new file mode 100644 index 0000000..605c187 --- /dev/null +++ b/docker/Dockerfile.original.cpu.server @@ -0,0 +1,43 @@ +# Use a base image with Python +FROM python:3.11-slim + +# Install system dependencies required for libGL and DOCX conversion +RUN apt-get update && apt-get install -y \ + libgl1-mesa-glx \ + libglib2.0-0 \ + libglib2.0-dev \ + libcairo2-dev \ + libpango1.0-dev \ + libgdk-pixbuf2.0-dev \ + libffi-dev \ + shared-mime-info \ + libpango-1.0-0 \ + libharfbuzz0b \ + libpangoft2-1.0-0 \ + libfontconfig1 \ + libcairo2 \ + libgdk-pixbuf2.0-0 \ + fonts-liberation \ + build-essential \ + pkg-config \ + gcc \ + && rm -rf /var/lib/apt/lists/* + +# Set the working directory +WORKDIR /app + +# Copy the project files into the Docker image +COPY . ./ + +# Install Python dependencies +RUN pip install -e . + +# Pre-load models to speed up container startup +# RUN python -c 'from marker.models import load_all_models; load_all_models()' +RUN python -c 'from marker.models import create_model_dict; create_model_dict()' + +# Expose the port +EXPOSE 8080 + +# Set the default command +CMD ["python", "server.py", "--host", "0.0.0.0", "--port", "8080"] \ No newline at end of file diff --git a/marker_api/celery_routes.py b/marker_api/celery_routes.py index b26cb06..ce0c424 100644 --- a/marker_api/celery_routes.py +++ b/marker_api/celery_routes.py @@ -36,11 +36,11 @@ async def celery_convert_pdf_sync(pdf_file: UploadFile = File(...)): return {"status": "Success", "result": result} -async def celery_convert_pdf_concurrent_await(pdf_file: UploadFile = File(...)): +async def celery_convert_pdf_concurrent_await(pdf_file: UploadFile = File(...), extract_images: bool = False): contents = await pdf_file.read() # Start the Celery task - task = convert_pdf_to_markdown.delay(pdf_file.filename, contents) + task = convert_pdf_to_markdown.delay(pdf_file.filename, contents, extract_images) # Define an asynchronous function to check task status async def check_task_status(): @@ -62,51 +62,6 @@ async def check_task_status(): ) -# async def celery_batch_convert(pdf_files: List[UploadFile] = File(...)): -# batch_data = [] -# for pdf_file in pdf_files: -# contents = await pdf_file.read() -# batch_data.append((pdf_file.filename, contents)) - -# # Start a single task to process the entire batch -# task = process_batch.delay(batch_data) - -# return { -# "task_id": str(task.id), -# "status": "Processing", -# } - - -# async def celery_batch_result(task_id: str): -# task = AsyncResult(task_id) - -# if not task.ready(): -# return JSONResponse( -# status_code=202, -# content={ -# "task_id": str(task_id), -# "status": "Processing", -# }, -# ) - -# try: -# results = task.get() -# return JSONResponse( -# status_code=200, -# content={"task_id": task_id, "status": "Success", "results": results}, -# ) -# except Exception as e: -# logger.error(f"Error retrieving results for task {task_id}: {str(e)}") -# return JSONResponse( -# status_code=500, -# content={ -# "task_id": task_id, -# "status": "Error", -# "message": "An error occurred while retrieving the results", -# }, -# ) - - async def celery_batch_convert(pdf_files: List[UploadFile] = File(...)): batch_data = [] for pdf_file in pdf_files: diff --git a/marker_api/celery_tasks.py b/marker_api/celery_tasks.py index 3658ef7..50fa8ad 100644 --- a/marker_api/celery_tasks.py +++ b/marker_api/celery_tasks.py @@ -1,87 +1,211 @@ from celery import Task from marker_api.celery_worker import celery_app -from marker.convert import convert_single_pdf -from marker.models import load_all_models +from marker.converters.pdf import PdfConverter +from marker.models import create_model_dict +from marker.output import text_from_rendered +from marker.logger import configure_logging import io +import os +import time +import base64 +import tempfile +from PIL import Image import logging from marker_api.utils import process_image_to_base64 from celery.signals import worker_process_init +from marker_api.routes import process_document_file +# Initialize logging +configure_logging() logger = logging.getLogger(__name__) -model_list = None - +converter = None @worker_process_init.connect def initialize_models(**kwargs): - global model_list - if not model_list: - model_list = load_all_models() - print("Models loaded at worker startup") + global converter + if not converter: + converter = PdfConverter( + artifact_dict=create_model_dict(), + ) + logger.debug("Marker converter initialized at worker startup") -class PDFConversionTask(Task): +class DocumentConversionTask(Task): abstract = True - + def __init__(self): super().__init__() - + def __call__(self, *args, **kwargs): - # Use the global model_list initialized at worker startup return self.run(*args, **kwargs) -@celery_app.task( - ignore_result=False, bind=True, base=PDFConversionTask, name="convert_pdf" -) -def convert_pdf_to_markdown(self, filename, pdf_content): - pdf_file = io.BytesIO(pdf_content) - markdown_text, images, metadata = convert_single_pdf(pdf_file, model_list) - image_data = {} - for i, (img_filename, image) in enumerate(images.items()): - logger.debug(f"Processing image {img_filename}") - image_base64 = process_image_to_base64(image, img_filename) - image_data[img_filename] = image_base64 - - return { - "filename": filename, - "markdown": markdown_text, - "metadata": metadata, - "images": image_data, - "status": "ok", - } +def parse_document_and_return_markdown(file_bytes: bytes, filename: str, extract_images: bool = False): + """ + Function to parse a document (PDF, DOCX, etc.) and extract text and images. + + Args: + file_bytes (bytes): The content of the document file. + filename (str): Original filename to determine file type + extract_images (bool): Whether to extract images or not. + + Returns: + tuple: A tuple containing the full text, metadata, and image data. + """ + try: + start_time = time.time() + + # Create a temporary file with the correct extension + file_extension = os.path.splitext(filename)[1].lower() if filename else '.pdf' + + with tempfile.NamedTemporaryFile(suffix=file_extension, delete=False) as temp_file: + temp_file.write(file_bytes) + temp_file_path = temp_file.name + + try: + # Convert document using the global converter + rendered = converter(temp_file_path) + + # Extract text and metadata from rendered output + full_text, _, images = text_from_rendered(rendered) + + end_time = time.time() + processing_time = end_time - start_time + + logger.info(f"Document processing completed in {processing_time:.2f} seconds") + logger.info(f"Extracted text length: {len(full_text)} characters") + logger.info(f"Images extracted: {len(images) if images else 0}") + + # Process images if extraction is enabled + image_data = {} + if extract_images and images: + # Handle different types of images return value + if isinstance(images, dict): + for image_name, image_obj in images.items(): + try: + if isinstance(image_obj, Image.Image): + # Handle PIL Image objects + image_data[image_name] = process_image_to_base64(image_obj, image_name) + else: + logger.warning(f"Unexpected image data type for {image_name}: {type(image_obj)}") + except Exception as img_error: + logger.error(f"Error processing image {image_name}: {str(img_error)}") + continue + else: + logger.info(f"Images returned as {type(images)}: {images}") + + return full_text, image_data, processing_time + + finally: + # Clean up temporary file + if os.path.exists(temp_file_path): + os.unlink(temp_file_path) + + except Exception as e: + logger.error(f"Error processing document: {str(e)}") + raise -# @celery_app.task( -# ignore_result=False, bind=True, base=PDFConversionTask, name="process_batch" -# ) -# def process_batch(self, batch_data): -# results = [] -# for filename, pdf_content in batch_data: -# try: -# result = convert_pdf_to_markdown(filename, pdf_content) -# results.append(result) -# except Exception as e: -# logger.error(f"Error processing {filename}: {str(e)}") -# results.append({"filename": filename, "status": "Error", "error": str(e)}) -# return results +@celery_app.task( + ignore_result=False, bind=True, base=DocumentConversionTask, name="convert_document" +) +def convert_document_to_markdown(self, filename, file_content, extract_images: bool = True): + """ + Process a document file (PDF, DOCX, etc.) and return markdown text with metadata. + + Args: + filename (str): The original filename + file_content (bytes): The document file content + extract_images (bool): Whether to extract images + + Returns: + dict: Processing results matching PDFConversionResult schema + """ + try: + # Determine file type from filename + file_extension = os.path.splitext(filename)[1].lower() if filename else '.pdf' + supported_extensions = ['.pdf', '.docx', '.pptx', '.xlsx', '.html', '.epub'] + + if file_extension not in supported_extensions: + raise ValueError(f"Unsupported file format: {file_extension}. Supported formats: {supported_extensions}") + + logger.info(f"Processing {file_extension} file: {filename}") + + full_text, images, processing_time = parse_document_and_return_markdown( + file_content, filename, extract_images + ) + + # Convert metadata to match GeneralMetadata schema + general_metadata = { + "custom_metadata": { + "processing_time": processing_time, + "file_type": file_extension, + "text_length": len(full_text), + }, + } + + result = { + "filename": filename or f"document{file_extension}", + "markdown": full_text, + "metadata": general_metadata, + "images": images if extract_images else {}, + "status": "success" + } + + logger.info(f"Successfully processed {filename}: {len(full_text)} chars, {len(images) if extract_images else 0} images") + return result + + except Exception as e: + logger.error(f"Error in convert_document_to_markdown: {str(e)}") + return { + "filename": filename, + "markdown": "", + "metadata": { + "custom_metadata": {"error": str(e), "file_type": file_extension if filename else ".pdf"}, + }, + "images": {}, + "status": "error" + } @celery_app.task( - ignore_result=False, bind=True, base=PDFConversionTask, name="process_batch" + ignore_result=False, bind=True, base=DocumentConversionTask, name="process_batch" ) -def process_batch(self, batch_data): +def process_batch(self, batch_data, extract_images: bool = False): + """ + Process a batch of documents. + + Args: + batch_data: List of tuples containing (filename, file_content) + extract_images (bool): Whether to extract images from documents + + Returns: + list: List of processing results for each document + """ results = [] total = len(batch_data) - for i, (filename, pdf_content) in enumerate(batch_data, start=1): + + for i, (filename, file_content) in enumerate(batch_data, start=1): try: - result = convert_pdf_to_markdown(filename, pdf_content) + result = convert_document_to_markdown(filename, file_content, extract_images) results.append(result) except Exception as e: logger.error(f"Error processing {filename}: {str(e)}") - results.append({"filename": filename, "status": "Error", "error": str(e)}) + results.append({ + "filename": filename, + "markdown": "", + "metadata": { + "custom_metadata": {"error": str(e)}, + }, + "images": {}, + "status": "error" + }) # Update progress self.update_state(state="PROGRESS", meta={"current": i, "total": total}) return results + +# Keep the old function name for backward compatibility +convert_pdf_to_markdown = convert_document_to_markdown \ No newline at end of file diff --git a/marker_api/celery_worker.py b/marker_api/celery_worker.py index d0f30d3..7fe34a5 100644 --- a/marker_api/celery_worker.py +++ b/marker_api/celery_worker.py @@ -1,21 +1,34 @@ import os from celery import Celery from dotenv import load_dotenv -import multiprocessing +import logging -multiprocessing.set_start_method("spawn") +# Configure logging +logging.basicConfig(level=logging.DEBUG) +logger = logging.getLogger(__name__) load_dotenv(".env") +# Get Redis host from environment or use default +redis_host = os.environ.get("REDIS_HOST", "localhost") +redis_url = f"redis://{redis_host}:6379/0" +logger.info(f"Connecting to Redis at: {redis_url}") + celery_app = Celery( "celery_app", - broker=os.environ.get("REDIS_HOST", "redis://localhost:6379/0"), - backend=os.environ.get("REDIS_HOST", "redis://localhost:6379/0"), + broker=redis_url, + backend=redis_url, include=["marker_api.celery_tasks"], ) +# Add some debug configuration +celery_app.conf.update( + task_track_started=True, + worker_send_task_events=True, + task_send_sent_event=True, +) @celery_app.task(name="celery.ping") def ping(): - print("Ping task received!") # or use a logger + logger.info("Ping task received!") return "pong" diff --git a/marker_api/routes.py b/marker_api/routes.py index fb39743..2f9d7f6 100644 --- a/marker_api/routes.py +++ b/marker_api/routes.py @@ -2,16 +2,20 @@ import time import base64 import tempfile +import io +from PIL import Image from marker.converters.pdf import PdfConverter from marker.models import create_model_dict from marker.output import text_from_rendered from marker.logger import configure_logging import logging +from marker_api.utils import process_image_to_base64 # Initialize logging configure_logging() logger = logging.getLogger(__name__) +SUPPORTED_EXTENSIONS = ['.pdf', '.docx'] # , '.pptx', '.xlsx', '.html', '.epub'] def parse_document_and_return_markdown(file_bytes: bytes, filename: str, extract_images: bool, model_dict): """ @@ -37,7 +41,7 @@ def parse_document_and_return_markdown(file_bytes: bytes, filename: str, extract temp_file_path = temp_file.name try: - # Create converter with model dictionary - PdfConverter handles all formats in 1.6.2 + # Create converter with model dictionary converter = PdfConverter( artifact_dict=model_dict, ) @@ -46,7 +50,7 @@ def parse_document_and_return_markdown(file_bytes: bytes, filename: str, extract rendered = converter(temp_file_path) # Extract text and metadata from rendered output - full_text, images, out_meta = text_from_rendered(rendered) + full_text, _, images = text_from_rendered(rendered) end_time = time.time() processing_time = end_time - start_time @@ -60,20 +64,20 @@ def parse_document_and_return_markdown(file_bytes: bytes, filename: str, extract if extract_images and images: # Handle different types of images return value if isinstance(images, dict): - for image_name, image_data_bytes in images.items(): + for image_name, image_obj in images.items(): try: - if isinstance(image_data_bytes, bytes): - img_str = base64.b64encode(image_data_bytes).decode() - image_data[image_name] = img_str + if isinstance(image_obj, Image.Image): + # Handle PIL Image objects + image_data[image_name] = process_image_to_base64(image_obj, image_name) else: - logger.warning(f"Unexpected image data type for {image_name}: {type(image_data_bytes)}") + logger.warning(f"Unexpected image data type for {image_name}: {type(image_obj)}") except Exception as img_error: logger.error(f"Error processing image {image_name}: {str(img_error)}") continue else: logger.info(f"Images returned as {type(images)}: {images}") - return full_text, out_meta, image_data, processing_time + return full_text, image_data, processing_time finally: # Clean up temporary file @@ -101,40 +105,34 @@ def process_document_file(file_bytes: bytes, filename: str = None, model_dict=No try: # Determine file type from filename file_extension = os.path.splitext(filename)[1].lower() if filename else '.pdf' - supported_extensions = ['.pdf', '.docx', '.pptx', '.xlsx', '.html', '.epub'] - if file_extension not in supported_extensions: - raise ValueError(f"Unsupported file format: {file_extension}. Supported formats: {supported_extensions}") + if file_extension not in SUPPORTED_EXTENSIONS: + raise ValueError(f"Unsupported file format: {file_extension}. Supported formats: {SUPPORTED_EXTENSIONS}") logger.info(f"Processing {file_extension} file: {filename}") - full_text, metadata, images, processing_time = parse_document_and_return_markdown( + full_text, images, processing_time = parse_document_and_return_markdown( file_bytes, filename, extract_images, model_dict ) # Convert metadata to match GeneralMetadata schema general_metadata = { - "languages": metadata.get("languages", None), - "toc": metadata.get("toc", None), - "pages": metadata.get("pages", None), "custom_metadata": { "processing_time": processing_time, "file_type": file_extension, "text_length": len(full_text), - "images_count": len(images), - **metadata - } + }, } result = { "filename": filename or f"document{file_extension}", "markdown": full_text, "metadata": general_metadata, - "images": images, + "images": images if extract_images else {}, "status": "success" } - logger.info(f"Successfully processed {filename}: {len(full_text)} chars, {len(images)} images") + logger.info(f"Successfully processed {filename}: {len(full_text)} chars, {len(images) if extract_images else 0} images") return result except Exception as e: @@ -143,10 +141,7 @@ def process_document_file(file_bytes: bytes, filename: str = None, model_dict=No "filename": filename or "document.pdf", "markdown": "", "metadata": { - "languages": None, - "toc": None, - "pages": None, - "custom_metadata": {"error": str(e), "file_type": file_extension if filename else ".pdf"} + "custom_metadata": {"error": str(e), "file_type": file_extension if filename else ".pdf"}, }, "images": {}, "status": "error" diff --git a/pyproject.toml b/pyproject.toml index c7720a8..2ddf7b5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,22 +8,36 @@ packages = [{include = "marker_api"}] [tool.poetry.dependencies] python = ">=3.10,<3.13" -fastapi = "^0.115.0" -uvicorn = "^0.31.1" -celery = {extras = ["redis"], version = "=5.3.6"} -flower = "^2.0.1" -hf-transfer = "^0.1.8" -huggingface-hub = "^0.25.1" -locust = "^2.31.8" -python-multipart = "^0.0.12" -redis = "^5.1.1" -requests = "^2.32.3" +fastapi = "0.115.0" +uvicorn = "0.31.1" +python-multipart = "0.0.12" +requests = "2.32.3" +marker-pdf = {version = "1.7.5", extras = ["full"]} +pillow = "^10.0.0" +torch = "^2.7.0" +pydantic = "^2.4.2" +pydantic-settings = "^2.0.3" +python-dotenv = "^1.0.0" rich = "^13.9.2" -marker-pdf = "1.6.2" pynvml = "^11.5.3" art = "^6.3" +starlette = "^0.37.2,<0.39.0" +# DOCX conversion dependencies +weasyprint = "^63.0" +cffi = "^1.15.0" +pycairo = "^1.20.0" +mammoth = "^1.9.0" +pypdf = "^4.0.0" +pdfplumber = "^0.11.0" +python-docx = "^1.1.0" +# Optional distributed server dependencies +celery = {extras = ["redis"], version = "^5.3.6", optional = true} +flower = {version = "^2.0.1", optional = true} +redis = {version = "^5.1.1", optional = true} +[tool.poetry.extras] +distributed = ["celery", "flower", "redis"] [build-system] requires = ["poetry-core"] diff --git a/server.py b/server.py index faebaff..23fd7ed 100644 --- a/server.py +++ b/server.py @@ -1,9 +1,12 @@ import os import asyncio import argparse +import base64 +import io +from PIL import Image from fastapi import FastAPI, UploadFile, File, Form, HTTPException from fastapi.middleware.cors import CORSMiddleware -from typing import List +from typing import List, Dict, Any import concurrent.futures from marker.logger import configure_logging # Import logging configuration from marker.models import create_model_dict # Import function to create model dict @@ -31,6 +34,32 @@ model_dict = None +# def clean_pil_images_from_dict(obj): +# """ +# Recursively clean PIL Image objects from a dictionary/list structure, +# converting them to base64 strings. +# """ +# if isinstance(obj, dict): +# cleaned = {} +# for key, value in obj.items(): +# cleaned[key] = clean_pil_images_from_dict(value) +# return cleaned +# elif isinstance(obj, list): +# return [clean_pil_images_from_dict(item) for item in obj] +# elif isinstance(obj, Image.Image): +# # Convert PIL Image to base64 string +# try: +# img_byte_arr = io.BytesIO() +# obj.save(img_byte_arr, format="PNG") +# img_byte_arr = img_byte_arr.getvalue() +# return base64.b64encode(img_byte_arr).decode() +# except Exception as e: + +# logger.error(f"Error converting PIL Image to base64: {str(e)}") +# return "" +# else: +# return obj + # Event that runs on startup to load all models @asynccontextmanager async def lifespan(app: FastAPI): @@ -76,7 +105,7 @@ async def root(): @app.post("/convert", response_model=PDFConversionResult) async def convert_document_to_markdown( file: UploadFile = File(...), - extract_images: bool = Form(True) + extract_images: bool = Form(False) ): """ Endpoint to convert a document (PDF, DOCX, PPTX, XLSX, HTML, EPUB) to markdown. @@ -110,7 +139,11 @@ async def convert_document_to_markdown( model_dict=model_dict, extract_images=extract_images ) - return PDFConversionResult(**response) + # Clean any PIL Image objects from the response + cleaned_response = response + # cleaned_response = clean_pil_images_from_dict(response) + + return PDFConversionResult(**cleaned_response) except Exception as e: logger.error(f"Error processing document: {str(e)}") raise HTTPException(status_code=500, detail=f"Error processing document: {str(e)}") @@ -120,7 +153,7 @@ async def convert_document_to_markdown( @app.post("/batch_convert", response_model=BatchConversionResponse) async def convert_documents_to_markdown( files: List[UploadFile] = File(...), - extract_images: bool = Form(True) + extract_images: bool = Form(False) ): """ Endpoint to convert multiple documents to markdown. @@ -152,29 +185,6 @@ async def process_files(file_list): return BatchConversionResponse(results=responses) -@app.post("/extract_structured") -async def extract_structured_data( - file: UploadFile = File(...), - schema: str = Form(None) -): - """ - Extract structured data from documents using Pydantic schemas (v1.7.5 feature) - """ - try: - # For now, return a placeholder response indicating the feature is available - # In a full implementation, this would use marker-pdf 1.7.5's structured extraction - return { - "message": "Structured extraction endpoint available in v1.7.5", - "filename": file.filename, - "feature": "structured_extraction", - "status": "placeholder_implementation", - "note": "This would extract structured data using Pydantic schemas in the full implementation" - } - except Exception as e: - logger.error(f"Structured extraction failed: {str(e)}") - raise HTTPException(status_code=500, detail=f"Structured extraction failed: {str(e)}") - - # Main function to run the server def main(): parser = argparse.ArgumentParser(description="Run the marker-api server.") From 5554f2ef6e6180a89dec1bd3caad56733bea385c Mon Sep 17 00:00:00 2001 From: kaan-playbook Date: Fri, 27 Jun 2025 16:23:56 +0200 Subject: [PATCH 05/16] reorganize the dockerfile --- distributed_server.py | 2 +- ....yml => docker-compose.cpu.distributed.yml | 49 ++++------ docker-compose.gpu.distributed.yml | 90 +++++++++++++++++ docker-compose.original.yml | 12 --- .../{Dockerfile.base => Dockerfile.cpu.base} | 0 docker/Dockerfile.cpu.distributed-server | 69 ++----------- docker/Dockerfile.cpu.server | 66 ++----------- docker/Dockerfile.cpu.v1.7.5 | 84 ---------------- docker/Dockerfile.gpu.base | 66 +++++++++++++ docker/Dockerfile.gpu.distributed-server | 97 +------------------ docker/Dockerfile.gpu.server | 52 ++-------- ...Dockerfile.original.cpu.distributed-server | 9 -- docker/Dockerfile.original.cpu.server | 43 -------- marker_api/celery_worker.py | 3 +- 14 files changed, 205 insertions(+), 437 deletions(-) rename docker-compose.distributed.yml => docker-compose.cpu.distributed.yml (55%) create mode 100644 docker-compose.gpu.distributed.yml delete mode 100644 docker-compose.original.yml rename docker/{Dockerfile.base => Dockerfile.cpu.base} (100%) delete mode 100644 docker/Dockerfile.cpu.v1.7.5 create mode 100644 docker/Dockerfile.gpu.base delete mode 100644 docker/Dockerfile.original.cpu.distributed-server delete mode 100644 docker/Dockerfile.original.cpu.server diff --git a/distributed_server.py b/distributed_server.py index 6b3c375..b81cf30 100644 --- a/distributed_server.py +++ b/distributed_server.py @@ -37,7 +37,7 @@ app = FastAPI( title="Marker API Distributed Server", description="Distributed API for converting documents (PDF, DOCX, PPTX, XLSX, HTML, EPUB) to Markdown using marker", - version="1.6.2" + version="1.7.5" ) logger.info("Configuring CORS middleware") diff --git a/docker-compose.distributed.yml b/docker-compose.cpu.distributed.yml similarity index 55% rename from docker-compose.distributed.yml rename to docker-compose.cpu.distributed.yml index 7590cc8..ad87510 100644 --- a/docker-compose.distributed.yml +++ b/docker-compose.cpu.distributed.yml @@ -2,72 +2,65 @@ version: "3.8" services: base: - image: marker-api-base:latest + image: marker-api-cpu-base:latest build: context: . - dockerfile: docker/Dockerfile.base + dockerfile: docker/Dockerfile.cpu.base celery_worker: build: - context: . - dockerfile: docker/Dockerfile.original.cpu.distributed-server + context: . # Keep the build context as the root directory + dockerfile: docker/Dockerfile.cpu.distributed-server image: marker-api-distributed-cpu command: celery -A marker_api.celery_worker.celery_app worker --pool=solo -n worker@%h --loglevel=info volumes: - .:/app - model_cache:/home/marker/.cache/huggingface environment: - - REDIS_HOST=redis + - REDIS_HOST=${REDIS_HOST} + links: + - redis depends_on: - redis - base - api: - container_name: marker-api-distributed - build: - context: . - dockerfile: docker/Dockerfile.original.cpu.distributed-server + app: + container_name: marker-api-cpu-distributed + image: marker-api-distributed-cpu command: python distributed_server.py --host 0.0.0.0 --port 8080 + environment: + - ENV=production ports: - "8086:8080" volumes: - .:/app - model_cache:/home/marker/.cache/huggingface - environment: - - REDIS_HOST=redis depends_on: - redis - base - healthcheck: - test: ["CMD", "curl", "-f", "http://localhost:8080/health"] - interval: 30s - timeout: 10s - retries: 3 redis: - image: redis:7-alpine + container_name: redis + image: redis:7.2.4-alpine ports: - "6379:6379" - healthcheck: - test: ["CMD", "redis-cli", "ping"] - interval: 5s - timeout: 3s - retries: 3 + flower: - build: - context: . - dockerfile: docker/Dockerfile.original.cpu.distributed-server + container_name: flower_cpu + image: marker-api-distributed-cpu command: celery -A marker_api.celery_worker.celery_app flower --port=5555 ports: - - "5555:5555" + - 5556:5555 volumes: - .:/app - model_cache:/home/marker/.cache/huggingface environment: - - REDIS_HOST=redis + - REDIS_HOST=${REDIS_HOST} depends_on: + - app - redis + - celery_worker - base volumes: diff --git a/docker-compose.gpu.distributed.yml b/docker-compose.gpu.distributed.yml new file mode 100644 index 0000000..6fdb2dd --- /dev/null +++ b/docker-compose.gpu.distributed.yml @@ -0,0 +1,90 @@ +version: "3.8" + +services: + base: + image: marker-api-gpu-base:latest + build: + context: . + dockerfile: docker/Dockerfile.gpu.base + deploy: + resources: + reservations: + devices: + - capabilities: [gpu] + + celery_worker: + build: + context: . # Keep the build context as the root directory + dockerfile: docker/Dockerfile.gpu.distributed-server # Specify the new path to the GPU Dockerfile + command: celery -A marker_api.celery_worker.celery_app worker --pool=solo -n worker@%h --loglevel=info + image: marker-api-distributed-gpu + volumes: + - .:/app + - model_cache:/home/marker/.cache/huggingface + depends_on: + - redis + - base + environment: + - REDIS_HOST=${REDIS_HOST} + deploy: + resources: + reservations: + devices: + - capabilities: [gpu] # Request GPU support + + app: + container_name: marker-api-gpu-distributed + image: marker-api-distributed-gpu + command: python distributed_server.py --host 0.0.0.0 --port 8080 + environment: + - ENV=production + ports: + - "8080:8080" + volumes: + - .:/app + - model_cache:/home/marker/.cache/huggingface + depends_on: + - redis + - celery_worker + - base + deploy: + resources: + reservations: + devices: + - capabilities: [gpu] # Request GPU support + + redis: + container_name: redis + image: redis:7.2.4-alpine + ports: + - "6379:6379" + + flower: + container_name: flower_gpu + image: marker-api-distributed-gpu + + command: celery -A marker_api.celery_worker.celery_app flower --port=5555 + ports: + - 5556:5555 + volumes: + - .:/app + - model_cache:/home/marker/.cache/huggingface + environment: + - REDIS_HOST=${REDIS_HOST} + depends_on: + - app + - redis + - celery_worker + - base + deploy: + resources: + reservations: + devices: + - capabilities: [gpu] # Request GPU support + +volumes: + model_cache: + +networks: + default: + name: marker-api-network \ No newline at end of file diff --git a/docker-compose.original.yml b/docker-compose.original.yml deleted file mode 100644 index b70709b..0000000 --- a/docker-compose.original.yml +++ /dev/null @@ -1,12 +0,0 @@ -version: "3.8" - -services: - app: - container_name: marker-api-cpu-original - build: - context: . - dockerfile: docker/Dockerfile.original.cpu.server - ports: - - "8080:8080" - volumes: - - .:/app diff --git a/docker/Dockerfile.base b/docker/Dockerfile.cpu.base similarity index 100% rename from docker/Dockerfile.base rename to docker/Dockerfile.cpu.base diff --git a/docker/Dockerfile.cpu.distributed-server b/docker/Dockerfile.cpu.distributed-server index 2d08006..06c4443 100644 --- a/docker/Dockerfile.cpu.distributed-server +++ b/docker/Dockerfile.cpu.distributed-server @@ -1,64 +1,9 @@ -# Use a base image with Python -FROM python:3.11-slim +# Use the base image that has models pre-loaded +FROM marker-api-base:latest -# Install system dependencies required for marker-pdf 1.6.2 and WeasyPrint -RUN apt-get update && apt-get install -y \ - # Core system dependencies - libgl1-mesa-glx \ - libglib2.0-0 \ - libgomp1 \ - libgcc-s1 \ - # WeasyPrint dependencies for DOCX support - libgobject-2.0-0 \ - libcairo2 \ - libpango-1.0-0 \ - libpangocairo-1.0-0 \ - libgdk-pixbuf2.0-0 \ - libffi-dev \ - shared-mime-info \ - # LibreOffice for PDF generation (optional) - libreoffice \ - # Additional utilities - curl \ - wget \ - && rm -rf /var/lib/apt/lists/* +# Copy the rest of the project files +COPY . . -# Set the working directory -WORKDIR /app - -# Copy the project files into the Docker image -COPY . ./ - -# Install Python dependencies with updated marker-pdf version -RUN pip install --no-cache-dir \ - fastapi==0.115.0 \ - uvicorn==0.31.1 \ - "celery[redis]==5.3.6" \ - flower==2.0.1 \ - hf-transfer==0.1.8 \ - huggingface-hub==0.25.1 \ - locust==2.31.8 \ - python-multipart==0.0.12 \ - redis==5.1.1 \ - requests==2.32.3 \ - rich==13.9.2 \ - pynvml==11.5.3 \ - art==6.3 \ - marker-pdf==1.6.2 \ - "transformers>=4.36.0,<4.45.0" \ - "gradio>=4.44.0,<5.0.0" \ - pillow \ - torch \ - torchvision \ - # WeasyPrint and dependencies for DOCX support - weasyprint \ - cffi \ - pycairo \ - # Additional dependencies for marker-pdf 1.6.2 - pypdf \ - pdfplumber - -# Pre-load models to speed up container startup using new API -RUN python -c 'from marker.models import create_model_dict; create_model_dict()' - -EXPOSE 8080 +# No need to install dependencies or pre-load models as they're in the base image +# Just expose the port +EXPOSE 8080 \ No newline at end of file diff --git a/docker/Dockerfile.cpu.server b/docker/Dockerfile.cpu.server index ab57a58..605c187 100644 --- a/docker/Dockerfile.cpu.server +++ b/docker/Dockerfile.cpu.server @@ -1,20 +1,16 @@ # Use a base image with Python FROM python:3.11-slim -# Install system dependencies required for marker-pdf 1.6.2 and WeasyPrint +# Install system dependencies required for libGL and DOCX conversion RUN apt-get update && apt-get install -y \ - # Core system dependencies libgl1-mesa-glx \ libglib2.0-0 \ - libgomp1 \ - # WeasyPrint dependencies for DOCX support libglib2.0-dev \ libcairo2-dev \ libpango1.0-dev \ libgdk-pixbuf2.0-dev \ libffi-dev \ shared-mime-info \ - # Additional WeasyPrint runtime dependencies libpango-1.0-0 \ libharfbuzz0b \ libpangoft2-1.0-0 \ @@ -22,13 +18,9 @@ RUN apt-get update && apt-get install -y \ libcairo2 \ libgdk-pixbuf2.0-0 \ fonts-liberation \ - # LibreOffice for PDF generation (optional) - libreoffice \ - # Additional utilities - curl \ - wget \ build-essential \ pkg-config \ + gcc \ && rm -rf /var/lib/apt/lists/* # Set the working directory @@ -37,55 +29,15 @@ WORKDIR /app # Copy the project files into the Docker image COPY . ./ -# Install Python dependencies with updated marker-pdf version -RUN pip install --no-cache-dir \ - fastapi==0.115.0 \ - uvicorn==0.31.1 \ - "celery[redis]==5.3.6" \ - flower==2.0.1 \ - hf-transfer==0.1.8 \ - huggingface-hub==0.25.1 \ - locust==2.31.8 \ - python-multipart==0.0.12 \ - redis==5.1.1 \ - requests==2.32.3 \ - rich==13.9.2 \ - pynvml==11.5.3 \ - art==6.3 \ - marker-pdf==1.6.2 \ - "transformers>=4.45.2,<5.0.0" \ - pillow \ - torch \ - torchvision \ - # WeasyPrint and dependencies for DOCX support - weasyprint \ - cffi \ - pycairo \ - # DOCX processing dependency - mammoth \ - # Additional dependencies for marker-pdf 1.6.2 - pypdf \ - pdfplumber +# Install Python dependencies +RUN pip install -e . -# Install additional dependencies that might be needed -RUN pip install --no-cache-dir \ - billiard \ - click-didyoumean \ - click-plugins \ - click-repl \ - kombu \ - python-dateutil \ - tzdata \ - vine \ - starlette \ - humanize \ - prometheus-client \ - pytz \ - tornado - -# Pre-load models to speed up container startup using new API +# Pre-load models to speed up container startup +# RUN python -c 'from marker.models import load_all_models; load_all_models()' RUN python -c 'from marker.models import create_model_dict; create_model_dict()' +# Expose the port EXPOSE 8080 -CMD ["python", "server.py", "--host", "0.0.0.0", "--port", "8080"] \ No newline at end of file +# Set the default command +CMD ["python", "server.py", "--host", "0.0.0.0", "--port", "8080"] \ No newline at end of file diff --git a/docker/Dockerfile.cpu.v1.7.5 b/docker/Dockerfile.cpu.v1.7.5 deleted file mode 100644 index 4a152a3..0000000 --- a/docker/Dockerfile.cpu.v1.7.5 +++ /dev/null @@ -1,84 +0,0 @@ -# Use a base image with Python -FROM python:3.11-slim - -# Install system dependencies required for marker-pdf 1.7.5 and WeasyPrint -RUN apt-get update && apt-get install -y \ - libgl1-mesa-glx \ - libglib2.0-0 \ - libgomp1 \ - libglib2.0-dev \ - libcairo2-dev \ - libpango1.0-dev \ - libgdk-pixbuf2.0-dev \ - libffi-dev \ - shared-mime-info \ - libpango-1.0-0 \ - libharfbuzz0b \ - libpangoft2-1.0-0 \ - libfontconfig1 \ - libcairo2 \ - libgdk-pixbuf2.0-0 \ - fonts-liberation \ - libreoffice \ - curl \ - wget \ - build-essential \ - pkg-config \ - && rm -rf /var/lib/apt/lists/* - -# Set the working directory -WORKDIR /app - -# Copy the project files into the Docker image -COPY . ./ - -# Install Python dependencies with marker-pdf 1.7.5 -RUN pip install --no-cache-dir \ - fastapi==0.115.0 \ - uvicorn==0.31.1 \ - python-multipart==0.0.12 \ - requests==2.32.3 \ - "marker-pdf[full]==1.7.5" \ - pillow \ - torch \ - torchvision \ - weasyprint \ - cffi \ - pycairo \ - mammoth \ - pypdf \ - pdfplumber \ - streamlit \ - streamlit-ace - -# Install additional dependencies that might be needed -RUN pip install --no-cache-dir \ - pydantic \ - pydantic-settings \ - python-dotenv \ - rich \ - pynvml \ - psutil \ - # Dependencies from working v1.6.2 Dockerfile - art \ - billiard \ - click-didyoumean \ - click-plugins \ - click-repl \ - kombu \ - python-dateutil \ - tzdata \ - vine \ - starlette \ - humanize \ - prometheus-client \ - pytz \ - tornado - -# Pre-load models to speed up container startup using new API -RUN python -c 'from marker.models import create_model_dict; create_model_dict()' - -EXPOSE 8080 - -# Command to run the server -CMD ["python", "server.py", "--host", "0.0.0.0", "--port", "8080"] \ No newline at end of file diff --git a/docker/Dockerfile.gpu.base b/docker/Dockerfile.gpu.base new file mode 100644 index 0000000..cd6898b --- /dev/null +++ b/docker/Dockerfile.gpu.base @@ -0,0 +1,66 @@ +ARG CUDA_VERSION="11.8.0" +ARG CUDNN_VERSION="8" +ARG UBUNTU_VERSION="22.04" +ARG MAX_JOBS=4 +# Use NVIDIA CUDA base image +FROM nvidia/cuda:12.3.1-cudnn8-devel-ubuntu22.04 + +FROM nvidia/cuda:$CUDA_VERSION-cudnn$CUDNN_VERSION-devel-ubuntu$UBUNTU_VERSION + +# ARG PYTHON_VERSION="3.11" +# ARG PYTORCH_VERSION="2.1.2" +# ARG CUDA="118" +# ARG TORCH_CUDA_ARCH_LIST="7.0 7.5 8.0 8.6 9.0+PTX" + +# ENV PYTHON_VERSION=$PYTHON_VERSION +# ENV TORCH_CUDA_ARCH_LIST=$TORCH_CUDA_ARCH_LIST + +# RUN apt-get update \ +# && apt-get install -y wget git build-essential ninja-build git-lfs libaio-dev && rm -rf /var/lib/apt/lists/* \ +# && wget \ +# https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh \ +# && mkdir /root/.conda \ +# && bash Miniconda3-latest-Linux-x86_64.sh -b \ +# && rm -f Miniconda3-latest-Linux-x86_64.sh \ +# && conda create -n "py${PYTHON_VERSION}" python="${PYTHON_VERSION}" + +# ENV PATH="/root/miniconda3/envs/py${PYTHON_VERSION}/bin:${PATH}" + +RUN apt-get update && \ + apt-get install -y --no-install-recommends \ + wget \ + curl \ + unzip \ + git \ + python3 \ + python3-pip \ + libgl1 \ + libglib2.0-0 \ + curl \ + gnupg2 \ + ca-certificates \ + apt-transport-https \ + software-properties-common \ + libreoffice \ + ffmpeg \ + git-lfs \ + xvfb \ + && ln -s /usr/bin/python3 /usr/bin/python \ + && apt-get update \ + && apt install python3-packaging \ + && rm -rf /var/lib/apt/lists/* + +RUN pip3 install --no-cache-dir torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cu118 + +WORKDIR /app + +# Copy the project files into the Docker image +COPY ../ ./ + +# Install Python dependencies +RUN pip3 install --no-cache-dir -e ".[distributed]" + +# RUN python -c 'from marker.models import load_all_models; load_all_models()' +RUN python -c 'from marker.models import create_model_dict; create_model_dict()' + +EXPOSE 8080 diff --git a/docker/Dockerfile.gpu.distributed-server b/docker/Dockerfile.gpu.distributed-server index b602d78..a8b94ad 100644 --- a/docker/Dockerfile.gpu.distributed-server +++ b/docker/Dockerfile.gpu.distributed-server @@ -1,100 +1,7 @@ -ARG CUDA_VERSION="11.8.0" -ARG CUDNN_VERSION="8" -ARG UBUNTU_VERSION="22.04" -ARG MAX_JOBS=4 - -FROM nvidia/cuda:$CUDA_VERSION-cudnn$CUDNN_VERSION-devel-ubuntu$UBUNTU_VERSION - -# ARG PYTHON_VERSION="3.11" -# ARG PYTORCH_VERSION="2.1.2" -# ARG CUDA="118" -# ARG TORCH_CUDA_ARCH_LIST="7.0 7.5 8.0 8.6 9.0+PTX" - -# ENV PYTHON_VERSION=$PYTHON_VERSION -# ENV TORCH_CUDA_ARCH_LIST=$TORCH_CUDA_ARCH_LIST - -# RUN apt-get update \ -# && apt-get install -y wget git build-essential ninja-build git-lfs libaio-dev && rm -rf /var/lib/apt/lists/* \ -# && wget \ -# https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh \ -# && mkdir /root/.conda \ -# && bash Miniconda3-latest-Linux-x86_64.sh -b \ -# && rm -f Miniconda3-latest-Linux-x86_64.sh \ -# && conda create -n "py${PYTHON_VERSION}" python="${PYTHON_VERSION}" - -# ENV PATH="/root/miniconda3/envs/py${PYTHON_VERSION}/bin:${PATH}" - -RUN apt-get update && \ - apt-get install -y --no-install-recommends \ - wget \ - curl \ - unzip \ - git \ - python3 \ - python3-pip \ - python3-dev \ - # Core system dependencies - libgl1 \ - libglib2.0-0 \ - libgomp1 \ - libgcc-s1 \ - # WeasyPrint dependencies for DOCX support - libgobject-2.0-0 \ - libcairo2 \ - libpango-1.0-0 \ - libpangocairo-1.0-0 \ - libgdk-pixbuf2.0-0 \ - libffi-dev \ - shared-mime-info \ - # LibreOffice for PDF generation (optional) - libreoffice \ - # Additional utilities - gnupg2 \ - ca-certificates \ - apt-transport-https \ - software-properties-common \ - ffmpeg \ - git-lfs \ - xvfb \ - python3-packaging \ - && ln -s /usr/bin/python3 /usr/bin/python \ - && rm -rf /var/lib/apt/lists/* - -RUN pip3 install --no-cache-dir torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cu118 - -WORKDIR /app +# Use the GPU base image +FROM marker-api-base-gpu:latest # Copy the project files into the Docker image COPY . ./ -# Install Python dependencies with updated marker-pdf version -RUN pip3 install --no-cache-dir \ - fastapi==0.115.0 \ - uvicorn==0.31.1 \ - "celery[redis]==5.3.6" \ - flower==2.0.1 \ - hf-transfer==0.1.8 \ - huggingface-hub==0.25.1 \ - locust==2.31.8 \ - python-multipart==0.0.12 \ - redis==5.1.1 \ - requests==2.32.3 \ - rich==13.9.2 \ - pynvml==11.5.3 \ - art==6.3 \ - marker-pdf==1.6.2 \ - "transformers>=4.36.0,<4.45.0" \ - "gradio>=4.44.0,<5.0.0" \ - pillow \ - # WeasyPrint and dependencies for DOCX support - weasyprint \ - cffi \ - pycairo \ - # Additional dependencies for marker-pdf 1.6.2 - pypdf \ - pdfplumber - -# Pre-load models to speed up container startup using new API -RUN python -c 'from marker.models import create_model_dict; create_model_dict()' - EXPOSE 8080 diff --git a/docker/Dockerfile.gpu.server b/docker/Dockerfile.gpu.server index b689c8f..d0873ff 100644 --- a/docker/Dockerfile.gpu.server +++ b/docker/Dockerfile.gpu.server @@ -32,32 +32,20 @@ RUN apt-get update && \ git \ python3 \ python3-pip \ - python3-dev \ - # Core system dependencies libgl1 \ libglib2.0-0 \ - libgomp1 \ - libgcc-s1 \ - # WeasyPrint dependencies for DOCX support - libgobject-2.0-0 \ - libcairo2 \ - libpango-1.0-0 \ - libpangocairo-1.0-0 \ - libgdk-pixbuf2.0-0 \ - libffi-dev \ - shared-mime-info \ - # LibreOffice for PDF generation (optional) - libreoffice \ - # Additional utilities + curl \ gnupg2 \ ca-certificates \ apt-transport-https \ software-properties-common \ + libreoffice \ ffmpeg \ git-lfs \ xvfb \ - python3-packaging \ && ln -s /usr/bin/python3 /usr/bin/python \ + && apt-get update \ + && apt install python3-packaging \ && rm -rf /var/lib/apt/lists/* RUN pip3 install --no-cache-dir torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cu118 @@ -65,36 +53,12 @@ RUN pip3 install --no-cache-dir torch torchvision torchaudio --index-url https:/ WORKDIR /app # Copy the project files into the Docker image -COPY . ./ +COPY ../ ./ -# Install Python dependencies with updated marker-pdf version -RUN pip3 install --no-cache-dir \ - fastapi==0.115.0 \ - uvicorn==0.31.1 \ - "celery[redis]==5.3.6" \ - flower==2.0.1 \ - hf-transfer==0.1.8 \ - huggingface-hub==0.25.1 \ - locust==2.31.8 \ - python-multipart==0.0.12 \ - redis==5.1.1 \ - requests==2.32.3 \ - rich==13.9.2 \ - pynvml==11.5.3 \ - art==6.3 \ - marker-pdf==1.6.2 \ - "transformers>=4.36.0,<4.45.0" \ - "gradio>=4.44.0,<5.0.0" \ - pillow \ - # WeasyPrint and dependencies for DOCX support - weasyprint \ - cffi \ - pycairo \ - # Additional dependencies for marker-pdf 1.6.2 - pypdf \ - pdfplumber +# Install Python dependencies +RUN pip3 install --no-cache-dir -e . -# Pre-load models to speed up container startup using new API +# RUN python -c 'from marker.models import load_all_models; load_all_models()' RUN python -c 'from marker.models import create_model_dict; create_model_dict()' EXPOSE 8080 diff --git a/docker/Dockerfile.original.cpu.distributed-server b/docker/Dockerfile.original.cpu.distributed-server deleted file mode 100644 index 06c4443..0000000 --- a/docker/Dockerfile.original.cpu.distributed-server +++ /dev/null @@ -1,9 +0,0 @@ -# Use the base image that has models pre-loaded -FROM marker-api-base:latest - -# Copy the rest of the project files -COPY . . - -# No need to install dependencies or pre-load models as they're in the base image -# Just expose the port -EXPOSE 8080 \ No newline at end of file diff --git a/docker/Dockerfile.original.cpu.server b/docker/Dockerfile.original.cpu.server deleted file mode 100644 index 605c187..0000000 --- a/docker/Dockerfile.original.cpu.server +++ /dev/null @@ -1,43 +0,0 @@ -# Use a base image with Python -FROM python:3.11-slim - -# Install system dependencies required for libGL and DOCX conversion -RUN apt-get update && apt-get install -y \ - libgl1-mesa-glx \ - libglib2.0-0 \ - libglib2.0-dev \ - libcairo2-dev \ - libpango1.0-dev \ - libgdk-pixbuf2.0-dev \ - libffi-dev \ - shared-mime-info \ - libpango-1.0-0 \ - libharfbuzz0b \ - libpangoft2-1.0-0 \ - libfontconfig1 \ - libcairo2 \ - libgdk-pixbuf2.0-0 \ - fonts-liberation \ - build-essential \ - pkg-config \ - gcc \ - && rm -rf /var/lib/apt/lists/* - -# Set the working directory -WORKDIR /app - -# Copy the project files into the Docker image -COPY . ./ - -# Install Python dependencies -RUN pip install -e . - -# Pre-load models to speed up container startup -# RUN python -c 'from marker.models import load_all_models; load_all_models()' -RUN python -c 'from marker.models import create_model_dict; create_model_dict()' - -# Expose the port -EXPOSE 8080 - -# Set the default command -CMD ["python", "server.py", "--host", "0.0.0.0", "--port", "8080"] \ No newline at end of file diff --git a/marker_api/celery_worker.py b/marker_api/celery_worker.py index 7fe34a5..7e0fe49 100644 --- a/marker_api/celery_worker.py +++ b/marker_api/celery_worker.py @@ -10,8 +10,7 @@ load_dotenv(".env") # Get Redis host from environment or use default -redis_host = os.environ.get("REDIS_HOST", "localhost") -redis_url = f"redis://{redis_host}:6379/0" +redis_url = os.environ.get("REDIS_HOST", "redis://localhost:6379/0") logger.info(f"Connecting to Redis at: {redis_url}") celery_app = Celery( From 76ea8c8e43c034a16774ae622a3a7bfa32c70cd8 Mon Sep 17 00:00:00 2001 From: kaan-playbook Date: Mon, 30 Jun 2025 16:09:15 +0200 Subject: [PATCH 06/16] added pagination for marker --- distributed_server.py | 15 +-- marker_api/celery_routes.py | 16 ++-- marker_api/celery_tasks.py | 184 +++++++++--------------------------- marker_api/model/schema.py | 6 +- marker_api/routes.py | 36 +++++-- server.py | 11 ++- 6 files changed, 103 insertions(+), 165 deletions(-) diff --git a/distributed_server.py b/distributed_server.py index b81cf30..34a76e9 100644 --- a/distributed_server.py +++ b/distributed_server.py @@ -116,20 +116,22 @@ def setup_routes(app: FastAPI, celery_live: bool): @app.post("/convert", response_model=ConversionResponse) async def convert_document( file: UploadFile = File(...), - extract_images: bool = Form(True) + extract_images: bool = Form(True), + paginate: bool = Form(True) ): """Convert a document (PDF, DOCX, PPTX, XLSX, HTML, EPUB) to markdown""" validate_file(file) - return await celery_convert_pdf_concurrent_await(file, extract_images) + return await celery_convert_pdf_concurrent_await(file, extract_images, paginate) @app.post("/celery/convert", response_model=CeleryTaskResponse) async def celery_convert_document( file: UploadFile = File(...), - extract_images: bool = Form(True) + extract_images: bool = Form(False), + paginate: bool = Form(True) ): """Submit a document conversion task to Celery queue""" validate_file(file) - return await celery_convert_pdf(file, extract_images) + return await celery_convert_pdf(file, extract_images, paginate) @app.get("/celery/result/{task_id}", response_model=CeleryResultResponse) async def get_celery_result(task_id: str): @@ -138,12 +140,13 @@ async def get_celery_result(task_id: str): @app.post("/batch_convert", response_model=BatchConversionResponse) async def batch_convert( files: List[UploadFile] = File(...), - extract_images: bool = Form(True) + extract_images: bool = Form(False), + paginate: bool = Form(True) ): """Convert multiple documents to markdown""" for file in files: validate_file(file) - return await celery_batch_convert(files, extract_images) + return await celery_batch_convert(files, extract_images, paginate) @app.get("/batch_convert/result/{task_id}", response_model=BatchResultResponse) async def get_batch_result(task_id: str): diff --git a/marker_api/celery_routes.py b/marker_api/celery_routes.py index ce0c424..d2fd429 100644 --- a/marker_api/celery_routes.py +++ b/marker_api/celery_routes.py @@ -9,9 +9,9 @@ logger = logging.getLogger(__name__) -async def celery_convert_pdf(pdf_file: UploadFile = File(...)): +async def celery_convert_pdf(pdf_file: UploadFile = File(...), extract_images: bool = False, paginate: bool = True): contents = await pdf_file.read() - task_id = convert_pdf_to_markdown.delay(pdf_file.filename, contents) + task_id = convert_pdf_to_markdown.delay(pdf_file.filename, contents, extract_images, paginate) return {"task_id": str(task_id), "status": "Processing"} @@ -29,18 +29,18 @@ async def celery_offline_root(): return {"message": "Celery is offline. No API is available."} -async def celery_convert_pdf_sync(pdf_file: UploadFile = File(...)): +async def celery_convert_pdf_sync(pdf_file: UploadFile = File(...), extract_images: bool = False, paginate: bool = True): contents = await pdf_file.read() - task = convert_pdf_to_markdown.delay(pdf_file.filename, contents) + task = convert_pdf_to_markdown.delay(pdf_file.filename, contents, extract_images, paginate) result = task.get(timeout=600) # 10-minute timeout return {"status": "Success", "result": result} -async def celery_convert_pdf_concurrent_await(pdf_file: UploadFile = File(...), extract_images: bool = False): +async def celery_convert_pdf_concurrent_await(pdf_file: UploadFile = File(...), extract_images: bool = False, paginate: bool = True): contents = await pdf_file.read() # Start the Celery task - task = convert_pdf_to_markdown.delay(pdf_file.filename, contents, extract_images) + task = convert_pdf_to_markdown.delay(pdf_file.filename, contents, extract_images, paginate) # Define an asynchronous function to check task status async def check_task_status(): @@ -62,11 +62,11 @@ async def check_task_status(): ) -async def celery_batch_convert(pdf_files: List[UploadFile] = File(...)): +async def celery_batch_convert(pdf_files: List[UploadFile] = File(...), extract_images: bool = False, paginate: bool = True): batch_data = [] for pdf_file in pdf_files: contents = await pdf_file.read() - batch_data.append((pdf_file.filename, contents)) + batch_data.append((pdf_file.filename, contents, extract_images, paginate)) # Start a single task to process the entire batch task = process_batch.delay(batch_data) diff --git a/marker_api/celery_tasks.py b/marker_api/celery_tasks.py index 50fa8ad..a596c3a 100644 --- a/marker_api/celery_tasks.py +++ b/marker_api/celery_tasks.py @@ -32,180 +32,88 @@ def initialize_models(**kwargs): class DocumentConversionTask(Task): - abstract = True - - def __init__(self): - super().__init__() - - def __call__(self, *args, **kwargs): - return self.run(*args, **kwargs) - + _model_dict = None -def parse_document_and_return_markdown(file_bytes: bytes, filename: str, extract_images: bool = False): - """ - Function to parse a document (PDF, DOCX, etc.) and extract text and images. - - Args: - file_bytes (bytes): The content of the document file. - filename (str): Original filename to determine file type - extract_images (bool): Whether to extract images or not. - - Returns: - tuple: A tuple containing the full text, metadata, and image data. - """ - try: - start_time = time.time() - - # Create a temporary file with the correct extension - file_extension = os.path.splitext(filename)[1].lower() if filename else '.pdf' - - with tempfile.NamedTemporaryFile(suffix=file_extension, delete=False) as temp_file: - temp_file.write(file_bytes) - temp_file_path = temp_file.name - - try: - # Convert document using the global converter - rendered = converter(temp_file_path) - - # Extract text and metadata from rendered output - full_text, _, images = text_from_rendered(rendered) - - end_time = time.time() - processing_time = end_time - start_time - - logger.info(f"Document processing completed in {processing_time:.2f} seconds") - logger.info(f"Extracted text length: {len(full_text)} characters") - logger.info(f"Images extracted: {len(images) if images else 0}") - - # Process images if extraction is enabled - image_data = {} - if extract_images and images: - # Handle different types of images return value - if isinstance(images, dict): - for image_name, image_obj in images.items(): - try: - if isinstance(image_obj, Image.Image): - # Handle PIL Image objects - image_data[image_name] = process_image_to_base64(image_obj, image_name) - else: - logger.warning(f"Unexpected image data type for {image_name}: {type(image_obj)}") - except Exception as img_error: - logger.error(f"Error processing image {image_name}: {str(img_error)}") - continue - else: - logger.info(f"Images returned as {type(images)}: {images}") - - return full_text, image_data, processing_time - - finally: - # Clean up temporary file - if os.path.exists(temp_file_path): - os.unlink(temp_file_path) - - except Exception as e: - logger.error(f"Error processing document: {str(e)}") - raise + @property + def model_dict(self): + if self._model_dict is None: + self._model_dict = create_model_dict() + return self._model_dict @celery_app.task( ignore_result=False, bind=True, base=DocumentConversionTask, name="convert_document" ) -def convert_document_to_markdown(self, filename, file_content, extract_images: bool = True): +def convert_pdf_to_markdown(self, filename, file_content, extract_images: bool = True, paginate: bool = True): """ - Process a document file (PDF, DOCX, etc.) and return markdown text with metadata. + Convert a PDF file to markdown. Args: filename (str): The original filename - file_content (bytes): The document file content + file_content (bytes): The PDF file content extract_images (bool): Whether to extract images + paginate (bool): Whether to split output into pages Returns: dict: Processing results matching PDFConversionResult schema """ - try: - # Determine file type from filename - file_extension = os.path.splitext(filename)[1].lower() if filename else '.pdf' - supported_extensions = ['.pdf', '.docx', '.pptx', '.xlsx', '.html', '.epub'] - - if file_extension not in supported_extensions: - raise ValueError(f"Unsupported file format: {file_extension}. Supported formats: {supported_extensions}") - - logger.info(f"Processing {file_extension} file: {filename}") - - full_text, images, processing_time = parse_document_and_return_markdown( - file_content, filename, extract_images - ) - - # Convert metadata to match GeneralMetadata schema - general_metadata = { - "custom_metadata": { - "processing_time": processing_time, - "file_type": file_extension, - "text_length": len(full_text), - }, - } - - result = { - "filename": filename or f"document{file_extension}", - "markdown": full_text, - "metadata": general_metadata, - "images": images if extract_images else {}, - "status": "success" - } - - logger.info(f"Successfully processed {filename}: {len(full_text)} chars, {len(images) if extract_images else 0} images") - return result - - except Exception as e: - logger.error(f"Error in convert_document_to_markdown: {str(e)}") - return { - "filename": filename, - "markdown": "", - "metadata": { - "custom_metadata": {"error": str(e), "file_type": file_extension if filename else ".pdf"}, - }, - "images": {}, - "status": "error" - } + return process_document_file( + file_content, + filename=filename, + model_dict=self.model_dict, + extract_images=extract_images, + paginate=paginate + ) @celery_app.task( ignore_result=False, bind=True, base=DocumentConversionTask, name="process_batch" ) -def process_batch(self, batch_data, extract_images: bool = False): +def process_batch(self, batch_data): """ - Process a batch of documents. + Process multiple documents in batch. Args: - batch_data: List of tuples containing (filename, file_content) - extract_images (bool): Whether to extract images from documents - + batch_data (list): List of tuples (filename, content, extract_images, paginate) + Returns: - list: List of processing results for each document + list: List of processing results """ results = [] - total = len(batch_data) + total_files = len(batch_data) - for i, (filename, file_content) in enumerate(batch_data, start=1): + for i, (filename, content, extract_images, paginate) in enumerate(batch_data, 1): try: - result = convert_document_to_markdown(filename, file_content, extract_images) + # Update progress + self.update_state( + state="PROGRESS", + meta={ + "current": i, + "total": total_files, + "status": f"Processing {filename}" + } + ) + + # Process the document + result = process_document_file( + content, + filename=filename, + model_dict=self.model_dict, + extract_images=extract_images, + paginate=paginate + ) results.append(result) + except Exception as e: logger.error(f"Error processing {filename}: {str(e)}") results.append({ "filename": filename, - "markdown": "", + "markdown": "" if not paginate else [], "metadata": { "custom_metadata": {"error": str(e)}, }, "images": {}, "status": "error" }) - - # Update progress - self.update_state(state="PROGRESS", meta={"current": i, "total": total}) - - return results - -# Keep the old function name for backward compatibility -convert_pdf_to_markdown = convert_document_to_markdown \ No newline at end of file + + return results \ No newline at end of file diff --git a/marker_api/model/schema.py b/marker_api/model/schema.py index 52bd7a8..6f059e5 100644 --- a/marker_api/model/schema.py +++ b/marker_api/model/schema.py @@ -45,10 +45,10 @@ class GeneralMetadata(BaseModel): class PDFConversionResult(BaseModel): filename: str - markdown: str + markdown: Union[str, List[str]] # Can be either a single string or a list of strings for paginated output metadata: GeneralMetadata - images: Dict[str, str] - status: str + images: Dict[str, str] = Field(default_factory=dict) + status: str = "success" class ConversionResponse(BaseModel): diff --git a/marker_api/routes.py b/marker_api/routes.py index 2f9d7f6..4de1bf5 100644 --- a/marker_api/routes.py +++ b/marker_api/routes.py @@ -17,7 +17,7 @@ SUPPORTED_EXTENSIONS = ['.pdf', '.docx'] # , '.pptx', '.xlsx', '.html', '.epub'] -def parse_document_and_return_markdown(file_bytes: bytes, filename: str, extract_images: bool, model_dict): +def parse_document_and_return_markdown(file_bytes: bytes, filename: str, extract_images: bool, model_dict, paginate: bool = True): """ Function to parse a document (PDF, DOCX, etc.) and extract text and images. @@ -26,6 +26,7 @@ def parse_document_and_return_markdown(file_bytes: bytes, filename: str, extract filename (str): Original filename to determine file type extract_images (bool): Whether to extract images or not. model_dict: The model dictionary from create_model_dict() + paginate (bool): Whether to paginate the output based on original document pages. Returns: tuple: A tuple containing the full text, metadata, and image data. @@ -41,9 +42,10 @@ def parse_document_and_return_markdown(file_bytes: bytes, filename: str, extract temp_file_path = temp_file.name try: - # Create converter with model dictionary + # Create converter with model dictionary and pagination config converter = PdfConverter( artifact_dict=model_dict, + config={"paginate_output": paginate} if paginate else None ) # Convert document using the new API - pass the file path directly @@ -52,11 +54,27 @@ def parse_document_and_return_markdown(file_bytes: bytes, filename: str, extract # Extract text and metadata from rendered output full_text, _, images = text_from_rendered(rendered) + # Split text into pages if pagination is enabled + if paginate and isinstance(full_text, str): + import re + # Split on any numbered page marker pattern ({n}------...) + pages = re.split(r'\{(\d+)\}[-]+', full_text) + # The split will create: ['before first marker', '0', 'page1 content', '1', 'page2 content', ...] + # We want only the content parts (every other item starting from index 2) + pages = [page.strip() for page in pages[2::2]] + # Remove empty pages + pages = [page for page in pages if page] + full_text = pages + end_time = time.time() processing_time = end_time - start_time logger.info(f"Document processing completed in {processing_time:.2f} seconds") - logger.info(f"Extracted text length: {len(full_text)} characters") + if isinstance(full_text, list): + logger.info(f"Extracted {len(full_text)} pages") + logger.info(f"Total text length: {sum(len(page) for page in full_text)} characters") + else: + logger.info(f"Extracted text length: {len(full_text)} characters") logger.info(f"Images extracted: {len(images) if images else 0}") # Process images if extraction is enabled @@ -89,7 +107,7 @@ def parse_document_and_return_markdown(file_bytes: bytes, filename: str, extract raise -def process_document_file(file_bytes: bytes, filename: str = None, model_dict=None, extract_images: bool = True): +def process_document_file(file_bytes: bytes, filename: str = None, model_dict=None, extract_images: bool = True, paginate: bool = True): """ Process a document file (PDF, DOCX, etc.) and return markdown text with metadata. @@ -98,6 +116,7 @@ def process_document_file(file_bytes: bytes, filename: str = None, model_dict=No filename (str): The original filename (optional) model_dict: Pre-loaded model dictionary extract_images (bool): Whether to extract images + paginate (bool): Whether to paginate the output based on original document pages Returns: dict: Processing results matching PDFConversionResult schema @@ -112,7 +131,7 @@ def process_document_file(file_bytes: bytes, filename: str = None, model_dict=No logger.info(f"Processing {file_extension} file: {filename}") full_text, images, processing_time = parse_document_and_return_markdown( - file_bytes, filename, extract_images, model_dict + file_bytes, filename, extract_images, model_dict, paginate ) # Convert metadata to match GeneralMetadata schema @@ -120,13 +139,14 @@ def process_document_file(file_bytes: bytes, filename: str = None, model_dict=No "custom_metadata": { "processing_time": processing_time, "file_type": file_extension, - "text_length": len(full_text), + "text_length": len(full_text) if isinstance(full_text, str) else sum(len(page) for page in full_text), + "pages": len(full_text) if isinstance(full_text, list) else None }, } result = { "filename": filename or f"document{file_extension}", - "markdown": full_text, + "markdown": full_text, # This will be either a string or a list of strings depending on pagination "metadata": general_metadata, "images": images if extract_images else {}, "status": "success" @@ -139,7 +159,7 @@ def process_document_file(file_bytes: bytes, filename: str = None, model_dict=No logger.error(f"Error in process_document_file: {str(e)}") return { "filename": filename or "document.pdf", - "markdown": "", + "markdown": "" if not paginate else [], "metadata": { "custom_metadata": {"error": str(e), "file_type": file_extension if filename else ".pdf"}, }, diff --git a/server.py b/server.py index 23fd7ed..36e93e7 100644 --- a/server.py +++ b/server.py @@ -105,10 +105,16 @@ async def root(): @app.post("/convert", response_model=PDFConversionResult) async def convert_document_to_markdown( file: UploadFile = File(...), - extract_images: bool = Form(False) + extract_images: bool = Form(False), + paginate: bool = Form(True) ): """ Endpoint to convert a document (PDF, DOCX, PPTX, XLSX, HTML, EPUB) to markdown. + + Parameters: + - file: The document file to convert + - extract_images: Whether to extract and include images in the output + - paginate: Whether to split the output into pages based on the original document structure """ global model_dict @@ -137,7 +143,8 @@ async def convert_document_to_markdown( file_content, filename=file.filename, model_dict=model_dict, - extract_images=extract_images + extract_images=extract_images, + paginate=paginate ) # Clean any PIL Image objects from the response cleaned_response = response From 888d8976790158dde87ec74be345347b7b1287e0 Mon Sep 17 00:00:00 2001 From: kaan-playbook Date: Mon, 30 Jun 2025 16:11:50 +0200 Subject: [PATCH 07/16] fixed docker files --- docker/Dockerfile.cpu.distributed-server | 2 +- docker/Dockerfile.gpu.base | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/docker/Dockerfile.cpu.distributed-server b/docker/Dockerfile.cpu.distributed-server index 06c4443..9b872d1 100644 --- a/docker/Dockerfile.cpu.distributed-server +++ b/docker/Dockerfile.cpu.distributed-server @@ -1,5 +1,5 @@ # Use the base image that has models pre-loaded -FROM marker-api-base:latest +FROM marker-api-cpu-base:latest # Copy the rest of the project files COPY . . diff --git a/docker/Dockerfile.gpu.base b/docker/Dockerfile.gpu.base index cd6898b..8a92015 100644 --- a/docker/Dockerfile.gpu.base +++ b/docker/Dockerfile.gpu.base @@ -2,8 +2,6 @@ ARG CUDA_VERSION="11.8.0" ARG CUDNN_VERSION="8" ARG UBUNTU_VERSION="22.04" ARG MAX_JOBS=4 -# Use NVIDIA CUDA base image -FROM nvidia/cuda:12.3.1-cudnn8-devel-ubuntu22.04 FROM nvidia/cuda:$CUDA_VERSION-cudnn$CUDNN_VERSION-devel-ubuntu$UBUNTU_VERSION From a279298903ea047dfcdd702750b4bb26b0dd195f Mon Sep 17 00:00:00 2001 From: kaan-playbook Date: Tue, 1 Jul 2025 11:04:50 +0200 Subject: [PATCH 08/16] change the return of pagination for marker from list to str --- marker_api/routes.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/marker_api/routes.py b/marker_api/routes.py index 4de1bf5..0ed1d89 100644 --- a/marker_api/routes.py +++ b/marker_api/routes.py @@ -55,16 +55,16 @@ def parse_document_and_return_markdown(file_bytes: bytes, filename: str, extract full_text, _, images = text_from_rendered(rendered) # Split text into pages if pagination is enabled - if paginate and isinstance(full_text, str): - import re - # Split on any numbered page marker pattern ({n}------...) - pages = re.split(r'\{(\d+)\}[-]+', full_text) - # The split will create: ['before first marker', '0', 'page1 content', '1', 'page2 content', ...] - # We want only the content parts (every other item starting from index 2) - pages = [page.strip() for page in pages[2::2]] - # Remove empty pages - pages = [page for page in pages if page] - full_text = pages + # if paginate and isinstance(full_text, str): + # import re + # # Split on any numbered page marker pattern ({n}------...) + # pages = re.split(r'\{(\d+)\}[-]+', full_text) + # # The split will create: ['before first marker', '0', 'page1 content', '1', 'page2 content', ...] + # # We want only the content parts (every other item starting from index 2) + # pages = [page.strip() for page in pages[2::2]] + # # Remove empty pages + # pages = [page for page in pages if page] + # full_text = pages end_time = time.time() processing_time = end_time - start_time From 94f2d26997caa045222411fe1b27d8e4658c26d3 Mon Sep 17 00:00:00 2001 From: kaan-playbook Date: Tue, 1 Jul 2025 11:23:37 +0200 Subject: [PATCH 09/16] removed the page split --- marker_api/model/schema.py | 2 +- marker_api/routes.py | 12 ------------ 2 files changed, 1 insertion(+), 13 deletions(-) diff --git a/marker_api/model/schema.py b/marker_api/model/schema.py index 6f059e5..b3d07ef 100644 --- a/marker_api/model/schema.py +++ b/marker_api/model/schema.py @@ -45,7 +45,7 @@ class GeneralMetadata(BaseModel): class PDFConversionResult(BaseModel): filename: str - markdown: Union[str, List[str]] # Can be either a single string or a list of strings for paginated output + markdown: str # Can be either a single string or a list of strings for paginated output metadata: GeneralMetadata images: Dict[str, str] = Field(default_factory=dict) status: str = "success" diff --git a/marker_api/routes.py b/marker_api/routes.py index 0ed1d89..7aa93bb 100644 --- a/marker_api/routes.py +++ b/marker_api/routes.py @@ -54,18 +54,6 @@ def parse_document_and_return_markdown(file_bytes: bytes, filename: str, extract # Extract text and metadata from rendered output full_text, _, images = text_from_rendered(rendered) - # Split text into pages if pagination is enabled - # if paginate and isinstance(full_text, str): - # import re - # # Split on any numbered page marker pattern ({n}------...) - # pages = re.split(r'\{(\d+)\}[-]+', full_text) - # # The split will create: ['before first marker', '0', 'page1 content', '1', 'page2 content', ...] - # # We want only the content parts (every other item starting from index 2) - # pages = [page.strip() for page in pages[2::2]] - # # Remove empty pages - # pages = [page for page in pages if page] - # full_text = pages - end_time = time.time() processing_time = end_time - start_time From 03fd2a0261001afd9e2bbe623446a0303960422e Mon Sep 17 00:00:00 2001 From: kaan-playbook Date: Mon, 7 Jul 2025 16:46:08 +0300 Subject: [PATCH 10/16] added docx to pdf conversion for docx files --- marker_api/model/schema.py | 1 + marker_api/routes.py | 116 ++++++++++++++++++++++++++++++------- 2 files changed, 96 insertions(+), 21 deletions(-) diff --git a/marker_api/model/schema.py b/marker_api/model/schema.py index b3d07ef..8a158d4 100644 --- a/marker_api/model/schema.py +++ b/marker_api/model/schema.py @@ -41,6 +41,7 @@ class GeneralMetadata(BaseModel): toc: Optional[List[Dict[str, Any]]] = None pages: Optional[int] = None custom_metadata: Dict[str, Any] = Field(default_factory=dict) + converted_pdf_base64: Optional[str] = None # Base64 encoded PDF content class PDFConversionResult(BaseModel): diff --git a/marker_api/routes.py b/marker_api/routes.py index 7aa93bb..044dc24 100644 --- a/marker_api/routes.py +++ b/marker_api/routes.py @@ -3,6 +3,7 @@ import base64 import tempfile import io +import subprocess from PIL import Image from marker.converters.pdf import PdfConverter from marker.models import create_model_dict @@ -10,6 +11,7 @@ from marker.logger import configure_logging import logging from marker_api.utils import process_image_to_base64 +from marker.config.parser import ConfigParser # Initialize logging configure_logging() @@ -17,6 +19,54 @@ SUPPORTED_EXTENSIONS = ['.pdf', '.docx'] # , '.pptx', '.xlsx', '.html', '.epub'] +def convert_docx_to_pdf(docx_path: str, pdf_path: str) -> None: + """ + Convert DOCX to PDF using LibreOffice. + + Args: + docx_path (str): Path to input DOCX file + pdf_path (str): Path where to save the PDF file + """ + try: + # Convert to absolute paths + docx_path = os.path.abspath(docx_path) + pdf_path = os.path.abspath(pdf_path) + + # Create output directory if it doesn't exist + os.makedirs(os.path.dirname(pdf_path), exist_ok=True) + + # Use LibreOffice to convert DOCX to PDF + # --headless: run without UI + # --convert-to pdf: convert to PDF format + # --outdir: specify output directory + cmd = [ + 'soffice', + '--headless', + '--convert-to', 'pdf', # Use the Writer PDF export filter + '--outdir', os.path.dirname(pdf_path), + docx_path + ] + + logger.info(f"Converting DOCX to PDF: {docx_path} -> {pdf_path}") + process = subprocess.run(cmd, capture_output=True, text=True) + + if process.returncode != 0: + raise Exception(f"LibreOffice conversion failed: {process.stderr}") + + # LibreOffice creates the PDF with the same name as input but .pdf extension + # Move it to the desired location if different + generated_pdf = os.path.join( + os.path.dirname(pdf_path), + os.path.splitext(os.path.basename(docx_path))[0] + '.pdf' + ) + if generated_pdf != pdf_path and os.path.exists(generated_pdf): + os.rename(generated_pdf, pdf_path) + + except subprocess.CalledProcessError as e: + raise Exception(f"Failed to convert DOCX to PDF: {str(e)}") + except Exception as e: + raise Exception(f"Error during DOCX to PDF conversion: {str(e)}") + def parse_document_and_return_markdown(file_bytes: bytes, filename: str, extract_images: bool, model_dict, paginate: bool = True): """ Function to parse a document (PDF, DOCX, etc.) and extract text and images. @@ -29,7 +79,7 @@ def parse_document_and_return_markdown(file_bytes: bytes, filename: str, extract paginate (bool): Whether to paginate the output based on original document pages. Returns: - tuple: A tuple containing the full text, metadata, and image data. + tuple: A tuple containing the full text, metadata, image data, and converted PDF content. """ try: start_time = time.time() @@ -42,10 +92,31 @@ def parse_document_and_return_markdown(file_bytes: bytes, filename: str, extract temp_file_path = temp_file.name try: + # If the file is a DOCX, convert it to PDF first + if file_extension == '.docx': + pdf_temp_file = tempfile.NamedTemporaryFile(suffix='.pdf', delete=False) + pdf_temp_path = pdf_temp_file.name + pdf_temp_file.close() + + try: + convert_docx_to_pdf(temp_file_path, pdf_temp_path) + temp_file_path = pdf_temp_path # Use the converted PDF + except Exception as e: + raise Exception(f"Failed to convert DOCX to PDF: {str(e)}") + # Create converter with model dictionary and pagination config + config = { + "paginate_output": paginate, # Add page breaks + # "format_lines": True, # Better formatting + # "force_ocr": False, # Don't force OCR for DOCX + # "strip_existing_ocr": False, # Keep existing text + # "disable_image_extraction": True, # Keep images + } + config_parser = ConfigParser(config) + # Initialize the converter with config converter = PdfConverter( artifact_dict=model_dict, - config={"paginate_output": paginate} if paginate else None + config=config_parser.generate_config_dict() ) # Convert document using the new API - pass the file path directly @@ -68,28 +139,29 @@ def parse_document_and_return_markdown(file_bytes: bytes, filename: str, extract # Process images if extraction is enabled image_data = {} if extract_images and images: - # Handle different types of images return value - if isinstance(images, dict): - for image_name, image_obj in images.items(): - try: - if isinstance(image_obj, Image.Image): - # Handle PIL Image objects - image_data[image_name] = process_image_to_base64(image_obj, image_name) - else: - logger.warning(f"Unexpected image data type for {image_name}: {type(image_obj)}") - except Exception as img_error: - logger.error(f"Error processing image {image_name}: {str(img_error)}") - continue - else: - logger.info(f"Images returned as {type(images)}: {images}") + for idx, img in enumerate(images): + image_data[f"image_{idx}"] = process_image_to_base64(img) - return full_text, image_data, processing_time + # Get the converted PDF content + converted_pdf_content = None + if hasattr(rendered, 'pdf_bytes') and rendered.pdf_bytes: + converted_pdf_content = base64.b64encode(rendered.pdf_bytes).decode('utf-8') + elif os.path.exists(temp_file_path): + # If pdf_bytes not available, read from the temporary file + with open(temp_file_path, 'rb') as f: + converted_pdf_content = base64.b64encode(f.read()).decode('utf-8') + + return full_text, image_data, processing_time, converted_pdf_content finally: - # Clean up temporary file - if os.path.exists(temp_file_path): + # Clean up the temporary files + try: os.unlink(temp_file_path) - + if file_extension == '.docx' and 'pdf_temp_path' in locals(): + os.unlink(pdf_temp_path) + except Exception as e: + logger.warning(f"Failed to delete temporary file(s): {e}") + except Exception as e: logger.error(f"Error processing document: {str(e)}") raise @@ -118,7 +190,7 @@ def process_document_file(file_bytes: bytes, filename: str = None, model_dict=No logger.info(f"Processing {file_extension} file: {filename}") - full_text, images, processing_time = parse_document_and_return_markdown( + full_text, images, processing_time, converted_pdf = parse_document_and_return_markdown( file_bytes, filename, extract_images, model_dict, paginate ) @@ -130,6 +202,7 @@ def process_document_file(file_bytes: bytes, filename: str = None, model_dict=No "text_length": len(full_text) if isinstance(full_text, str) else sum(len(page) for page in full_text), "pages": len(full_text) if isinstance(full_text, list) else None }, + "converted_pdf_base64": converted_pdf } result = { @@ -150,6 +223,7 @@ def process_document_file(file_bytes: bytes, filename: str = None, model_dict=No "markdown": "" if not paginate else [], "metadata": { "custom_metadata": {"error": str(e), "file_type": file_extension if filename else ".pdf"}, + "converted_pdf_base64": None }, "images": {}, "status": "error" From e34ec30099b64a51cf9028cc838de40d41df0e74 Mon Sep 17 00:00:00 2001 From: kaan-playbook Date: Thu, 10 Jul 2025 11:27:56 +0300 Subject: [PATCH 11/16] update conversion command fo docx to pdf --- marker_api/routes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/marker_api/routes.py b/marker_api/routes.py index 044dc24..f250760 100644 --- a/marker_api/routes.py +++ b/marker_api/routes.py @@ -40,7 +40,7 @@ def convert_docx_to_pdf(docx_path: str, pdf_path: str) -> None: # --convert-to pdf: convert to PDF format # --outdir: specify output directory cmd = [ - 'soffice', + 'libreoffice', '--headless', '--convert-to', 'pdf', # Use the Writer PDF export filter '--outdir', os.path.dirname(pdf_path), From 01b2baa48a8d2ed81efd4ab383ef8258962dde9a Mon Sep 17 00:00:00 2001 From: kaan-playbook Date: Tue, 15 Jul 2025 10:05:37 +0200 Subject: [PATCH 12/16] Change temporary file removal for docx and pdf --- marker_api/routes.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/marker_api/routes.py b/marker_api/routes.py index f250760..a0c4937 100644 --- a/marker_api/routes.py +++ b/marker_api/routes.py @@ -91,6 +91,7 @@ def parse_document_and_return_markdown(file_bytes: bytes, filename: str, extract temp_file.write(file_bytes) temp_file_path = temp_file.name + original_temp_path = temp_file_path # Keep track of original file try: # If the file is a DOCX, convert it to PDF first if file_extension == '.docx': @@ -155,12 +156,12 @@ def parse_document_and_return_markdown(file_bytes: bytes, filename: str, extract finally: # Clean up the temporary files - try: - os.unlink(temp_file_path) - if file_extension == '.docx' and 'pdf_temp_path' in locals(): - os.unlink(pdf_temp_path) - except Exception as e: - logger.warning(f"Failed to delete temporary file(s): {e}") + for path in [original_temp_path, temp_file_path]: + try: + if path and os.path.exists(path): + os.unlink(path) + except OSError: + pass # File already deleted or doesn't exist except Exception as e: logger.error(f"Error processing document: {str(e)}") From 7dffd6d7a750daf63374c96be6741310a97693c0 Mon Sep 17 00:00:00 2001 From: kaan-playbook Date: Thu, 7 Aug 2025 11:23:20 +0300 Subject: [PATCH 13/16] add doc support --- marker_api/routes.py | 6 +++--- server.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/marker_api/routes.py b/marker_api/routes.py index a0c4937..c9a788a 100644 --- a/marker_api/routes.py +++ b/marker_api/routes.py @@ -17,7 +17,7 @@ configure_logging() logger = logging.getLogger(__name__) -SUPPORTED_EXTENSIONS = ['.pdf', '.docx'] # , '.pptx', '.xlsx', '.html', '.epub'] +SUPPORTED_EXTENSIONS = ['.pdf', '.docx', '.doc'] # , '.pptx', '.xlsx', '.html', '.epub'] def convert_docx_to_pdf(docx_path: str, pdf_path: str) -> None: """ @@ -93,8 +93,8 @@ def parse_document_and_return_markdown(file_bytes: bytes, filename: str, extract original_temp_path = temp_file_path # Keep track of original file try: - # If the file is a DOCX, convert it to PDF first - if file_extension == '.docx': + # If the file is a DOCX, convert it to DF first + if file_extension == '.docx' or file_extension == '.doc': pdf_temp_file = tempfile.NamedTemporaryFile(suffix='.pdf', delete=False) pdf_temp_path = pdf_temp_file.name pdf_temp_file.close() diff --git a/server.py b/server.py index 36e93e7..5fa0c98 100644 --- a/server.py +++ b/server.py @@ -128,7 +128,7 @@ async def convert_document_to_markdown( raise HTTPException(status_code=413, detail="File too large. Maximum size is 50MB.") # Check file type - allowed_extensions = {'.pdf', '.docx', '.pptx', '.xlsx', '.html', '.epub'} + allowed_extensions = {'.pdf', '.docx', '.pptx', '.xlsx', '.html', '.epub', '.doc'} file_extension = os.path.splitext(file.filename)[1].lower() if file.filename else '' if file_extension not in allowed_extensions: From 04e7f62447c75017d13674d021d311d411c0e33e Mon Sep 17 00:00:00 2001 From: kaan-playbook Date: Wed, 13 Aug 2025 16:43:01 +0200 Subject: [PATCH 14/16] enable logging --- server.py | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/server.py b/server.py index 5fa0c98..3bee692 100644 --- a/server.py +++ b/server.py @@ -28,6 +28,36 @@ # Initialize logging configure_logging() + +def apply_marker_formatter_and_route_to_root( + fmt="[%(asctime)s]-[%(levelname)s]-[%(name)s]: %(message)s", + datefmt="%Y-%m-%d %H:%M:%S", + level=logging.INFO, +) -> None: + formatter = logging.Formatter(fmt=fmt, datefmt=datefmt) + marker_logger = logging.getLogger("marker") + root = logging.getLogger() + + # If marker has no handlers, nothing to do + if not marker_logger.handlers: + return + + # Move marker handlers to root with new formatter (prevents duplicates) + handlers_to_move = list(marker_logger.handlers) + for h in handlers_to_move: + try: + h.setFormatter(formatter) + except Exception: + pass + marker_logger.removeHandler(h) + if h not in root.handlers: + root.addHandler(h) + + root.setLevel(level) + marker_logger.propagate = True # marker logs now flow to root + +apply_marker_formatter_and_route_to_root() + logger = logging.getLogger(__name__) # Global variable to hold model dict From 76e4fb00cb14f3f2112cacd37016fe81835dd420 Mon Sep 17 00:00:00 2001 From: kaan-playbook Date: Thu, 21 Aug 2025 11:26:59 +0200 Subject: [PATCH 15/16] displaying the original file name in logs --- marker_api/routes.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/marker_api/routes.py b/marker_api/routes.py index c9a788a..c829572 100644 --- a/marker_api/routes.py +++ b/marker_api/routes.py @@ -19,13 +19,14 @@ SUPPORTED_EXTENSIONS = ['.pdf', '.docx', '.doc'] # , '.pptx', '.xlsx', '.html', '.epub'] -def convert_docx_to_pdf(docx_path: str, pdf_path: str) -> None: +def convert_docx_to_pdf(docx_path: str, pdf_path: str, original_filename: str = None) -> None: """ Convert DOCX to PDF using LibreOffice. Args: docx_path (str): Path to input DOCX file pdf_path (str): Path where to save the PDF file + original_filename (str): Original filename for logging purposes """ try: # Convert to absolute paths @@ -47,7 +48,8 @@ def convert_docx_to_pdf(docx_path: str, pdf_path: str) -> None: docx_path ] - logger.info(f"Converting DOCX to PDF: {docx_path} -> {pdf_path}") + display_name = original_filename or os.path.basename(docx_path) + logger.info(f"Converting DOCX to PDF: {display_name}") process = subprocess.run(cmd, capture_output=True, text=True) if process.returncode != 0: @@ -100,7 +102,7 @@ def parse_document_and_return_markdown(file_bytes: bytes, filename: str, extract pdf_temp_file.close() try: - convert_docx_to_pdf(temp_file_path, pdf_temp_path) + convert_docx_to_pdf(temp_file_path, pdf_temp_path, filename) temp_file_path = pdf_temp_path # Use the converted PDF except Exception as e: raise Exception(f"Failed to convert DOCX to PDF: {str(e)}") @@ -129,13 +131,14 @@ def parse_document_and_return_markdown(file_bytes: bytes, filename: str, extract end_time = time.time() processing_time = end_time - start_time - logger.info(f"Document processing completed in {processing_time:.2f} seconds") + display_name = filename or "document" + logger.info(f"Document processing completed for '{display_name}' in {processing_time:.2f} seconds") if isinstance(full_text, list): - logger.info(f"Extracted {len(full_text)} pages") + logger.info(f"Extracted {len(full_text)} pages from '{display_name}'") logger.info(f"Total text length: {sum(len(page) for page in full_text)} characters") else: - logger.info(f"Extracted text length: {len(full_text)} characters") - logger.info(f"Images extracted: {len(images) if images else 0}") + logger.info(f"Extracted text length: {len(full_text)} characters from '{display_name}'") + logger.info(f"Images extracted from '{display_name}': {len(images) if images else 0}") # Process images if extraction is enabled image_data = {} From bd58b23c1f87a3e2b8f53638a902ad37607b0b23 Mon Sep 17 00:00:00 2001 From: kaan-playbook Date: Fri, 7 Nov 2025 18:09:29 +0100 Subject: [PATCH 16/16] Change /health endpoint to /healthz for Kubernetes --- client/marker_api_client/__init__.py | 4 ++-- distributed_server.py | 2 +- server.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/client/marker_api_client/__init__.py b/client/marker_api_client/__init__.py index 6cb2e5c..4f194ac 100644 --- a/client/marker_api_client/__init__.py +++ b/client/marker_api_client/__init__.py @@ -65,7 +65,7 @@ def __exit__(self, exc_type, exc_val, exc_tb): def check_health(self): logger.info("Checking server health...") - response = self.session.get(f"{self.base_url}/health") + response = self.session.get(f"{self.base_url}/healthz") response.raise_for_status() health_data = HealthResponse(**response.json()) self.server_type = health_data.type @@ -74,7 +74,7 @@ def check_health(self): async def acheck_health(self): logger.info("Checking server health asynchronously...") - async with self.async_session.get(f"{self.base_url}/health") as response: + async with self.async_session.get(f"{self.base_url}/healthz") as response: response.raise_for_status() health_data = HealthResponse(**(await response.json())) self.server_type = health_data.type diff --git a/distributed_server.py b/distributed_server.py index 34a76e9..90bd63d 100644 --- a/distributed_server.py +++ b/distributed_server.py @@ -50,7 +50,7 @@ ) -@app.get("/health", response_model=HealthResponse) +@app.get("/healthz", response_model=HealthResponse) def server(): """ Root endpoint to check server status. diff --git a/server.py b/server.py index 3bee692..dfdf42a 100644 --- a/server.py +++ b/server.py @@ -122,7 +122,7 @@ async def lifespan(app: FastAPI): # app = gr.mount_gradio_app(app, demo_ui, path="") -@app.get("/health") +@app.get("/healthz") async def health_check(): return {"message": "Marker API is running", "status": "ok", "version": "1.7.5"}