Build a RAG agent that collects Korean DART filings and US SEC documents, stores them in ChromaDB, and answers natural-language questions grounded in source documents. It handles questions such as "What was the 2024 operating profit of Samsung Electronics?" and "What are the recent earnings risks for AAPL?"
[Document ingestion pipeline]
DART API โโโ
SEC API โโโผโโ text chunking โ embedding โ ChromaDB
News crawl โโ
[Query pipeline]
User question โ embed the question โ similar document search (Top-K)
โ inject into LLM context โ grounded answer| Component | Choice | Reason |
|---|---|---|
| Embedding model | jhgan/ko-sroberta-multitask | Tuned for Korean (DART filings are in Korean), runs locally |
| Vector DB | ChromaDB (local) | Simple install, persistence supported |
| LLM | Llama 3.1 8B (Ollama) | CPU inference, free |
| Chunk size | 512 tokens, 64 overlap | Best fit for filing paragraphs |
import requests
import os
import zipfile
import io
from pathlib import Path
from dotenv import load_dotenv
load_dotenv()
DART_KEY = os.getenv("DART_API_KEY")
class DartCollector:
BASE = "https://opendart.fss.or.kr/api"
def get_corp_code(self, company_name: str) -> str | None:
"""Company name โ DART corp code"""
res = requests.get(
f"{self.BASE}/company.json",
params={"crtfc_key": DART_KEY, "corp_name": company_name},
)
items = res.json().get("list", [])
return items[0]["corp_code"] if items else None
def get_financial_report(self, corp_code: str, year: int, report_type: str = "11011") -> list[dict]:
"""
Fetch the key financial items of an annual report
report_type: 11011=annual report, 11012=half-year report, 11013=quarterly report
"""
res = requests.get(
f"{self.BASE}/fnlttSinglAcnt.json",
params={
"crtfc_key": DART_KEY,
"corp_code": corp_code,
"bsns_year": str(year),
"reprt_code": report_type,
},
)
data = res.json()
if data.get("status") != "000":
return []
return data.get("list", [])
def get_disclosures(self, corp_code: str, bgn_date: str, end_date: str) -> list[dict]:
"""List filings"""
res = requests.get(
f"{self.BASE}/list.json",
params={
"crtfc_key": DART_KEY,
"corp_code": corp_code,
"bgn_de": bgn_date,
"end_de": end_date,
"sort": "date",
"sort_mth": "desc",
"count": 20,
},
)
return res.json().get("list", [])
def get_disclosure_text(self, rcept_no: str) -> str:
"""Download the filing source and extract the text"""
res = requests.get(
f"{self.BASE}/document.xml",
params={"crtfc_key": DART_KEY, "rcept_no": rcept_no},
)
# Unzip, then extract the text
try:
z = zipfile.ZipFile(io.BytesIO(res.content))
texts = []
for name in z.namelist():
if name.endswith(".xml") or name.endswith(".html"):
content = z.read(name).decode("utf-8", errors="ignore")
# Strip tags
import re
texts.append(re.sub(r"<[^>]+>", " ", content))
return " ".join(texts)[:10000] # limit to 10,000 characters
except Exception:
return ""
def build_dart_documents(company_name: str, years: list[int]) -> list[dict]:
"""
Convert a company's annual report data into a list of documents
Returns: [{"content": str, "metadata": dict}]
"""
collector = DartCollector()
corp_code = collector.get_corp_code(company_name)
if not corp_code:
return []
documents = []
for year in years:
items = collector.get_financial_report(corp_code, year)
for item in items:
content = (
f"[{company_name} {year} financial data]\n"
f"Account: {item.get('account_nm')}\n"
f"Current period: KRW {item.get('thstrm_amount')}\n"
f"Prior period: KRW {item.get('frmtrm_amount')}\n"
)
documents.append({
"content": content,
"metadata": {
"company": company_name,
"year": year,
"account": item.get("account_nm"),
"source": "DART",
},
})
return documentsimport requests
from langchain_core.tools import tool
SEC_HEADERS = {"User-Agent": "aidevops contact@aidevops.us"}
class SecCollector:
BASE = "https://data.sec.gov"
def get_cik(self, ticker: str) -> str | None:
"""Ticker โ CIK number"""
res = requests.get(
"https://www.sec.gov/files/company_tickers.json",
headers=SEC_HEADERS,
)
data = res.json()
ticker_upper = ticker.upper()
for item in data.values():
if item["ticker"] == ticker_upper:
return str(item["cik_str"]).zfill(10)
return None
def get_recent_10k(self, cik: str, count: int = 3) -> list[dict]:
"""List of recent 10-K (annual report) filings"""
res = requests.get(
f"{self.BASE}/submissions/CIK{cik}.json",
headers=SEC_HEADERS,
)
data = res.json()
filings = data.get("filings", {}).get("recent", {})
results = []
forms = filings.get("form", [])
dates = filings.get("filingDate", [])
accs = filings.get("accessionNumber", [])
for i, form in enumerate(forms):
if form == "10-K" and len(results) < count:
results.append({
"form": form,
"date": dates[i],
"accession": accs[i],
"url": f"https://www.sec.gov/Archives/edgar/full-index/{dates[i][:4]}/",
})
return results
def get_filing_text(self, cik: str, accession: str) -> str:
"""Extract 10-K text (first 8,000 characters)"""
acc_clean = accession.replace("-", "")
url = f"{self.BASE}/Archives/edgar/data/{int(cik)}/{acc_clean}/{accession}.txt"
try:
res = requests.get(url, headers=SEC_HEADERS, timeout=10)
import re
text = re.sub(r"<[^>]+>", " ", res.text)
text = re.sub(r"\s+", " ", text)
return text[:8000]
except Exception:
return ""
def build_sec_documents(ticker: str) -> list[dict]:
"""Convert a US company's 10-K data into a list of documents"""
collector = SecCollector()
cik = collector.get_cik(ticker)
if not cik:
return []
documents = []
for filing in collector.get_recent_10k(cik, count=2):
text = collector.get_filing_text(cik, filing["accession"])
if text:
documents.append({
"content": f"[{ticker} 10-K {filing['date']}]\n{text}",
"metadata": {
"ticker": ticker,
"form": "10-K",
"date": filing["date"],
"source": "SEC",
},
})
return documentsimport chromadb
from chromadb.config import Settings
from langchain_community.embeddings import HuggingFaceEmbeddings
from langchain.text_splitter import RecursiveCharacterTextSplitter
from langchain_community.vectorstores import Chroma
from pathlib import Path
# Embedding model tuned for Korean (local, free)
EMBED_MODEL = "jhgan/ko-sroberta-multitask"
DB_PATH = "D:/ai-agent-finance/chroma_db"
def get_embeddings():
return HuggingFaceEmbeddings(
model_name=EMBED_MODEL,
model_kwargs={"device": "cpu"},
encode_kwargs={"normalize_embeddings": True},
)
def get_vectorstore(collection: str = "finance") -> Chroma:
return Chroma(
collection_name=collection,
embedding_function=get_embeddings(),
persist_directory=DB_PATH,
)
def ingest_documents(documents: list[dict], collection: str = "finance") -> int:
"""
Chunk the documents and store them in the vector DB.
documents: [{"content": str, "metadata": dict}]
Returns: number of chunks stored
"""
splitter = RecursiveCharacterTextSplitter(
chunk_size=512,
chunk_overlap=64,
separators=["\n\n", "\n", "ใ", ". ", " "],
)
chunks, metadatas = [], []
for doc in documents:
splits = splitter.split_text(doc["content"])
chunks.extend(splits)
metadatas.extend([doc["metadata"]] * len(splits))
vs = get_vectorstore(collection)
vs.add_texts(texts=chunks, metadatas=metadatas)
vs.persist()
print(f"โ Stored {len(chunks)} chunks in the '{collection}' collection.")
return len(chunks)
# Usage example
if __name__ == "__main__":
from collectors.dart_collector import build_dart_documents
from collectors.sec_collector import build_sec_documents
# Ingest a Korean company
docs = build_dart_documents("์ผ์ฑ์ ์", years=[2023, 2024]) # DART looks companies up by their Korean name
ingest_documents(docs, collection="kr_finance")
# Ingest a US company
docs = build_sec_documents("AAPL")
ingest_documents(docs, collection="us_finance")from langchain_core.tools import tool
from rag.vectorstore import get_vectorstore
def _search(query: str, collection: str, k: int = 4) -> str:
vs = get_vectorstore(collection)
docs = vs.similarity_search(query, k=k)
if not docs:
return "No relevant documents found."
return "\n\n---\n\n".join(
f"[Source: {d.metadata.get('source','?')} {d.metadata.get('date','')}]\n{d.page_content}"
for d in docs
)
@tool
def search_kr_disclosure(query: str) -> str:
"""
Search the Korean company DART filings database for relevant content.
query: what to search for (e.g. "Samsung Electronics 2024 operating profit", "SK hynix semiconductor inventory")
"""
return _search(query, "kr_finance")
@tool
def search_us_filing(query: str) -> str:
"""
Search the US company SEC 10-K database for relevant content.
query: what to search for (e.g. "Apple revenue 2024", "NVDA data center risk")
"""
return _search(query, "us_finance")from langchain_ollama import ChatOllama
from langchain_anthropic import ChatAnthropic
from langchain.agents import create_react_agent, AgentExecutor
from langchain_core.prompts import PromptTemplate
from rag.retriever import search_kr_disclosure, search_us_filing
from tools.us_stock import get_us_stock_price
from tools.kr_stock import get_kr_stock_price
import os
RESEARCH_TOOLS = [
search_kr_disclosure,
search_us_filing,
get_kr_stock_price,
get_us_stock_price,
]
SYSTEM = """You are a financial investment research expert.
You provide accurate analysis based on filings and financial data.
Key principles:
- Answer only on the basis of the retrieved documents
- Cite the source (DART/SEC)
- Quote figures exactly as they appear in the source
- Do not give investment advice"""
def build_research_agent(use_claude: bool = False) -> AgentExecutor:
if use_claude:
llm = ChatAnthropic(
model="claude-sonnet-4-6",
api_key=os.getenv("ANTHROPIC_API_KEY"),
temperature=0,
)
else:
llm = ChatOllama(model="llama3.1:8b", temperature=0)
prompt = PromptTemplate.from_template(
SYSTEM + """
Available tools:
{tools}
Tool names: {tool_names}
Question: {input}
{agent_scratchpad}"""
)
agent = create_react_agent(llm, RESEARCH_TOOLS, prompt)
return AgentExecutor(
agent=agent,
tools=RESEARCH_TOOLS,
verbose=True,
max_iterations=6,
handle_parsing_errors=True,
)Choosing Claude vs Ollama: DART/SEC documents are long and complex. Start with local Llama 3.1 8B, and switch to use_claude=True if the analysis quality is poor. Claude is strong at long-context processing and multilingual comprehension.
from agents.research_agent import build_research_agent
agent = build_research_agent(use_claude=False) # local Ollama
questions = [
"What was the 2024 operating profit of Samsung Electronics?",
"What are the main financial risks of SK hynix?",
"What key business risks does Apple mention in its 2024 10-K?",
"Explain the earnings trend of the Samsung Electronics semiconductor division",
]
for q in questions:
print(f"\nQ: {q}")
result = agent.invoke({"input": q})
print(f"A: {result['output']}")# Run document ingestion first
python rag/vectorstore.py
# Test the agent
python -c "
from agents.research_agent import build_research_agent
agent = build_research_agent()
r = agent.invoke({'input': 'What was the latest operating profit of Samsung Electronics?'})
print(r['output'])
"