aimb-x/agentgit
repositories / aimb-x / agentgit

agentgit public

git clone https://git.169.58.159.131.nip.io/aimb-x/agentgit.git
agentgit/indexer.py 6144 bytes 1 lines
1
"""agentgit — project indexer: clone a repo, split it into chunks, store the text.
2
3
The rule is simple: agents learn from real files, cited as path:line. Nothing is
4
paraphrased; what comes back is the repository's own text.
5
"""
6
from __future__ import annotations
7
8
import os
9
import re
10
import subprocess
11
from pathlib import Path
12
13
SKIP_DIRS = {
14
".git", "node_modules", "dist", "build", "out", ".next", ".nuxt", "vendor",
15
"__pycache__", ".venv", "venv", "target", ".turbo", ".cache", "coverage",
16
".pytest_cache", ".mypy_cache", ".idea", ".vscode", "site-packages",
17
}
18
SKIP_FILES = {
19
"package-lock.json", "pnpm-lock.yaml", "yarn.lock", "poetry.lock", "Cargo.lock",
20
"go.sum", "composer.lock", "Gemfile.lock",
21
}
22
EXT_LANG = {
23
".py": "python", ".pyi": "python", ".c": "c", ".h": "c", ".cc": "cpp", ".cpp": "cpp",
24
".hpp": "cpp", ".rs": "rust", ".go": "go", ".js": "javascript", ".mjs": "javascript",
25
".cjs": "javascript", ".jsx": "javascript", ".ts": "typescript", ".tsx": "typescript",
26
".java": "java", ".kt": "kotlin", ".rb": "ruby", ".php": "php", ".cs": "csharp",
27
".swift": "swift", ".sh": "shell", ".bash": "shell", ".sql": "sql", ".css": "css",
28
".html": "html", ".md": "markdown", ".mdx": "markdown", ".yml": "yaml", ".yaml": "yaml",
29
".toml": "toml", ".json": "json", ".txt": "text", ".sn": "sinema", ".vp": "veloce",
30
".proto": "protobuf", ".lua": "lua", ".zig": "zig", ".ex": "elixir", ".exs": "elixir",
31
}
32
TEXT_EXT = set(EXT_LANG)
33
MAX_FILE_BYTES = 512 * 1024
34
MAX_FILES = 4000
35
MAX_TOTAL_BYTES = 40 * 1024 * 1024
36
MAX_CHUNK = 1400
37
OVERLAP_LINES = 6
38
39
40
def run(cmd: list[str], cwd: str | None = None, timeout: int = 600) -> str:
41
r = subprocess.run(cmd, cwd=cwd, capture_output=True, text=True, timeout=timeout)
42
if r.returncode != 0:
43
raise RuntimeError(f"{cmd[0]} failed: {r.stderr.strip()[:300]}")
44
return r.stdout
45
46
47
def clone(url: str, dest: Path, ref: str = "") -> str:
48
dest.parent.mkdir(parents=True, exist_ok=True)
49
if (dest / ".git").exists():
50
run(["git", "fetch", "--depth", "1", "origin", ref or "HEAD"], cwd=str(dest))
51
run(["git", "reset", "--hard", "FETCH_HEAD"], cwd=str(dest))
52
else:
53
cmd = ["git", "clone", "--depth", "1", "--quiet"]
54
if ref:
55
cmd += ["--branch", ref]
56
cmd += [url, str(dest)]
57
run(cmd, timeout=900)
58
return run(["git", "rev-parse", "--short", "HEAD"], cwd=str(dest)).strip()
59
60
61
def is_text(path: Path) -> bool:
62
try:
63
with open(path, "rb") as f:
64
head = f.read(2048)
65
except OSError:
66
return False
67
if b"\x00" in head:
68
return False
69
return True
70
71
72
def chunk_lines(lines: list[str], path: str, lang: str) -> list[dict]:
73
"""Chunks of ~1400 chars, cut at a blank line when possible."""
74
out: list[dict] = []
75
i, n = 0, len(lines)
76
while i < n:
77
start = i
78
size = 0
79
while i < n and size < MAX_CHUNK:
80
size += len(lines[i])
81
i += 1
82
# perpanjang ke baris kosong terdekat (maks 30 baris) supaya fungsi utuh
83
look = i
84
while look < n and look - i < 30 and lines[look].strip():
85
look += 1
86
if look < n and look - start < 200:
87
i = look + 1
88
body = "".join(lines[start:i]).rstrip()
89
if body.strip():
90
out.append({
91
"path": path,
92
"lang": lang,
93
"start_line": start + 1,
94
"end_line": i,
95
"body": body,
96
})
97
if i >= n:
98
break
99
i = max(i - OVERLAP_LINES, start + 1)
100
return out
101
102
103
def collect(root: Path) -> tuple[list[dict], int, int]:
104
chunks: list[dict] = []
105
files = 0
106
total = 0
107
root = root.resolve()
108
for dirpath, dirnames, filenames in os.walk(root):
109
dirnames[:] = sorted(d for d in dirnames if d not in SKIP_DIRS and not d.startswith(".git"))
110
for fn in sorted(filenames):
111
if fn in SKIP_FILES or fn.endswith(".min.js") or fn.endswith(".map"):
112
continue
113
p = Path(dirpath) / fn
114
ext = p.suffix.lower()
115
if ext not in TEXT_EXT:
116
continue
117
size = min(p.stat().st_size, MAX_FILE_BYTES) if p.exists() else 0
118
if not size or size > MAX_FILE_BYTES or not is_text(p):
119
continue
120
if files >= MAX_FILES or total + size > MAX_TOTAL_BYTES:
121
return chunks, files, total
122
try:
123
text = p.read_text(encoding="utf-8", errors="replace")
124
except OSError:
125
continue
126
rel = str(p.relative_to(root))
127
lang = EXT_LANG.get(ext, "")
128
chunks.extend(chunk_lines(text.splitlines(keepends=True), rel, lang))
129
files += 1
130
total += size
131
return chunks, files, total
132
133
134
def summary(root: Path) -> dict:
135
"""Repo shape (without reading every file) for `project_summary`."""
136
root = root.resolve()
137
langs: dict[str, int] = {}
138
dirs: dict[str, int] = {}
139
files = 0
140
for dirpath, dirnames, filenames in os.walk(root):
141
dirnames[:] = [d for d in dirnames if d not in SKIP_DIRS]
142
for fn in filenames:
143
ext = Path(fn).suffix.lower()
144
if ext not in TEXT_EXT:
145
continue
146
files += 1
147
langs[EXT_LANG.get(ext, ext)] = langs.get(EXT_LANG.get(ext, ext), 0) + 1
148
rel = os.path.relpath(dirpath, root)
149
top = rel.split(os.sep)[0] if rel != "." else "."
150
dirs[top] = dirs.get(top, 0) + 1
151
commit = ""
152
subject = ""
153
if (root / ".git").exists():
154
try:
155
commit = run(["git", "rev-parse", "--short", "HEAD"], cwd=str(root)).strip()
156
subject = run(["git", "log", "-1", "--pretty=%s"], cwd=str(root)).strip()
157
except Exception:
158
pass
159
return {
160
"files": files,
161
"languages": dict(sorted(langs.items(), key=lambda kv: -kv[1])[:8]),
162
"top_dirs": dict(sorted(dirs.items(), key=lambda kv: -kv[1])[:12]),
163
"commit": commit,
164
"last_commit_subject": subject,
165
}