agentgit/indexer.py
6144 bytes
1 lines
1
"""agentgit — project indexer: clone a repo, split it into chunks, store the text.
3
The rule is simple: agents learn from real files, cited as path:line. Nothing is
4
paraphrased; what comes back is the repository's own text.
6
from __future__ import annotations
11
from pathlib import Path
14
".git", "node_modules", "dist", "build", "out", ".next", ".nuxt", "vendor",
15
"__pycache__", ".venv", "venv", "target", ".turbo", ".cache", "coverage",
16
".pytest_cache", ".mypy_cache", ".idea", ".vscode", "site-packages",
19
"package-lock.json", "pnpm-lock.yaml", "yarn.lock", "poetry.lock", "Cargo.lock",
20
"go.sum", "composer.lock", "Gemfile.lock",
23
".py": "python", ".pyi": "python", ".c": "c", ".h": "c", ".cc": "cpp", ".cpp": "cpp",
24
".hpp": "cpp", ".rs": "rust", ".go": "go", ".js": "javascript", ".mjs": "javascript",
25
".cjs": "javascript", ".jsx": "javascript", ".ts": "typescript", ".tsx": "typescript",
26
".java": "java", ".kt": "kotlin", ".rb": "ruby", ".php": "php", ".cs": "csharp",
27
".swift": "swift", ".sh": "shell", ".bash": "shell", ".sql": "sql", ".css": "css",
28
".html": "html", ".md": "markdown", ".mdx": "markdown", ".yml": "yaml", ".yaml": "yaml",
29
".toml": "toml", ".json": "json", ".txt": "text", ".sn": "sinema", ".vp": "veloce",
30
".proto": "protobuf", ".lua": "lua", ".zig": "zig", ".ex": "elixir", ".exs": "elixir",
32
TEXT_EXT = set(EXT_LANG)
33
MAX_FILE_BYTES = 512 * 1024
35
MAX_TOTAL_BYTES = 40 * 1024 * 1024
40
def run(cmd: list[str], cwd: str | None = None, timeout: int = 600) -> str:
41
r = subprocess.run(cmd, cwd=cwd, capture_output=True, text=True, timeout=timeout)
43
raise RuntimeError(f"{cmd[0]} failed: {r.stderr.strip()[:300]}")
47
def clone(url: str, dest: Path, ref: str = "") -> str:
48
dest.parent.mkdir(parents=True, exist_ok=True)
49
if (dest / ".git").exists():
50
run(["git", "fetch", "--depth", "1", "origin", ref or "HEAD"], cwd=str(dest))
51
run(["git", "reset", "--hard", "FETCH_HEAD"], cwd=str(dest))
53
cmd = ["git", "clone", "--depth", "1", "--quiet"]
55
cmd += ["--branch", ref]
56
cmd += [url, str(dest)]
58
return run(["git", "rev-parse", "--short", "HEAD"], cwd=str(dest)).strip()
61
def is_text(path: Path) -> bool:
63
with open(path, "rb") as f:
72
def chunk_lines(lines: list[str], path: str, lang: str) -> list[dict]:
73
"""Chunks of ~1400 chars, cut at a blank line when possible."""
79
while i < n and size < MAX_CHUNK:
82
# perpanjang ke baris kosong terdekat (maks 30 baris) supaya fungsi utuh
84
while look < n and look - i < 30 and lines[look].strip():
86
if look < n and look - start < 200:
88
body = "".join(lines[start:i]).rstrip()
93
"start_line": start + 1,
99
i = max(i - OVERLAP_LINES, start + 1)
103
def collect(root: Path) -> tuple[list[dict], int, int]:
104
chunks: list[dict] = []
107
root = root.resolve()
108
for dirpath, dirnames, filenames in os.walk(root):
109
dirnames[:] = sorted(d for d in dirnames if d not in SKIP_DIRS and not d.startswith(".git"))
110
for fn in sorted(filenames):
111
if fn in SKIP_FILES or fn.endswith(".min.js") or fn.endswith(".map"):
113
p = Path(dirpath) / fn
114
ext = p.suffix.lower()
115
if ext not in TEXT_EXT:
117
size = min(p.stat().st_size, MAX_FILE_BYTES) if p.exists() else 0
118
if not size or size > MAX_FILE_BYTES or not is_text(p):
120
if files >= MAX_FILES or total + size > MAX_TOTAL_BYTES:
121
return chunks, files, total
123
text = p.read_text(encoding="utf-8", errors="replace")
126
rel = str(p.relative_to(root))
127
lang = EXT_LANG.get(ext, "")
128
chunks.extend(chunk_lines(text.splitlines(keepends=True), rel, lang))
131
return chunks, files, total
134
def summary(root: Path) -> dict:
135
"""Repo shape (without reading every file) for `project_summary`."""
136
root = root.resolve()
137
langs: dict[str, int] = {}
138
dirs: dict[str, int] = {}
140
for dirpath, dirnames, filenames in os.walk(root):
141
dirnames[:] = [d for d in dirnames if d not in SKIP_DIRS]
143
ext = Path(fn).suffix.lower()
144
if ext not in TEXT_EXT:
147
langs[EXT_LANG.get(ext, ext)] = langs.get(EXT_LANG.get(ext, ext), 0) + 1
148
rel = os.path.relpath(dirpath, root)
149
top = rel.split(os.sep)[0] if rel != "." else "."
150
dirs[top] = dirs.get(top, 0) + 1
153
if (root / ".git").exists():
155
commit = run(["git", "rev-parse", "--short", "HEAD"], cwd=str(root)).strip()
156
subject = run(["git", "log", "-1", "--pretty=%s"], cwd=str(root)).strip()
161
"languages": dict(sorted(langs.items(), key=lambda kv: -kv[1])[:8]),
162
"top_dirs": dict(sorted(dirs.items(), key=lambda kv: -kv[1])[:12]),
164
"last_commit_subject": subject,