AI Document Agent

# document_processor/ingestion.py
from langchain.document_loaders import UnstructuredPDFLoader
from langchain.text_splitter import RecursiveCharacterTextSplitter
from langchain_openai import OpenAIEmbeddings
from pinecone import Pinecone
import hashlib
 
class LegalDocumentProcessor:
    def __init__(self):
        self.embeddings = OpenAIEmbeddings(model="text-embedding-3-large")
        self.pc = Pinecone(api_key=os.environ["PINECONE_API_KEY"])
        self.index = self.pc.Index("legal-docs")
 
    def process_document(self, file_path: str, metadata: dict) -> str:
        loader = UnstructuredPDFLoader(
            file_path,
            mode="elements",
            strategy="hi_res",
            infer_table_structure=True,
        )
        elements = loader.load()
 
        # Reconstruct logical sections from document elements
        sections = self._reconstruct_sections(elements)
 
        splitter = RecursiveCharacterTextSplitter(
            chunk_size=1500,
            chunk_overlap=300,
            separators=["\n\nArticle ", "\n\n", "\n", ". "]
        )
 
        doc_id = hashlib.sha256(file_path.encode()).hexdigest()[:16]
        vectors = []
 
        for i, chunk in enumerate(splitter.split_documents(sections)):
            embedding = self.embeddings.embed_query(chunk.page_content)
            vectors.append({
                "id": f"{doc_id}-{i}",
                "values": embedding,
                "metadata": {
                    **metadata,
                    "doc_id": doc_id,
                    "chunk_index": i,
                    "text": chunk.page_content,
                }
            })
 
        self.index.upsert(vectors=vectors, namespace=metadata["client_id"])
        return doc_id
 
    def _reconstruct_sections(self, elements):
        # Groups elements by logical section
        # using headings as section markers
        sections = []
        current_section = []
        for el in elements:
            if el.metadata.get("category") == "Title":
                if current_section:
                    sections.append("\n".join(current_section))
                current_section = [el.page_content]
            else:
                current_section.append(el.page_content)
        if current_section:
            sections.append("\n".join(current_section))
        return sections

# agent/legal_agent.py
from langchain.agents import AgentExecutor, create_openai_tools_agent
from langchain_openai import ChatOpenAI
from langchain.tools import tool
 
@tool
def extract_parties(doc_id: str) -> dict:
    """Extracts the parties (name, role, legal representative) from a contract."""
    chunks = retrieve_relevant_chunks(doc_id, "contracting parties signatories representatives")
    response = llm.invoke(EXTRACTION_PROMPT.format(
        task="parties",
        schema='{"parties": [{"name": str, "role": str, "legal_rep": str}]}',
        context="\n".join(chunks)
    ))
    return json.loads(response.content)
 
@tool
def detect_unusual_clauses(doc_id: str, contract_type: str) -> list[dict]:
    """Detects unusual or potentially risky clauses."""
    reference_clauses = get_standard_clauses(contract_type)
    doc_chunks = retrieve_all_chunks(doc_id)
 
    unusual = []
    for chunk in doc_chunks:
        similarity = compute_clause_similarity(chunk, reference_clauses)
        if similarity < 0.65:
            unusual.append({
                "text": chunk,
                "risk_level": classify_risk(chunk),
                "explanation": generate_risk_explanation(chunk)
            })
    return unusual
 
@tool
def generate_summary_memo(doc_id: str, language: str = "en") -> str:
    """Generates a structured legal summary memo."""
    parties = extract_parties(doc_id)
    key_terms = extract_key_terms(doc_id)
    unusual_clauses = detect_unusual_clauses(doc_id, "generic")
 
    return llm.invoke(MEMO_PROMPT.format(
        parties=parties,
        key_terms=key_terms,
        unusual_clauses=unusual_clauses,
        language=language
    )).content
 
legal_agent = AgentExecutor(
    agent=create_openai_tools_agent(
        llm=ChatOpenAI(model="gpt-4o", temperature=0),
        tools=[extract_parties, detect_unusual_clauses, generate_summary_memo],
        prompt=LEGAL_AGENT_PROMPT,
    ),
    tools=[extract_parties, detect_unusual_clauses, generate_summary_memo],
    verbose=True,
    max_iterations=10,
)

# api/main.py
from fastapi import FastAPI, BackgroundTasks, UploadFile
from redis import Redis
import rq
 
app = FastAPI()
redis_conn = Redis(host="redis", port=6379)
queue = rq.Queue("document_processing", connection=redis_conn)
 
@app.post("/documents/analyze")
async def analyze_document(
    file: UploadFile,
    client_id: str,
    contract_type: str,
    background_tasks: BackgroundTasks,
):
    file_path = await save_temp_file(file)
 
    job = queue.enqueue(
        process_and_analyze,
        file_path,
        client_id,
        contract_type,
        job_timeout=300,
    )
 
    return {"job_id": job.id, "status": "queued"}
 
@app.get("/documents/{job_id}/status")
async def get_analysis_status(job_id: str):
    job = rq.job.Job.fetch(job_id, connection=redis_conn)
    return {
        "status": job.get_status(),
        "result": job.result if job.is_finished else None,
    }