1
"""agentgit — penyimpanan di atas **kilat** (mesin buatan sendiri), bukan SQLite.
3
Koleksi (nilai JSON, kunci string):
4
users/agents/projects/jobs → kunci = id
5
docmeta → kunci = "{project_id}/{path}#{start_line}"
6
notemeta → key = note id
7
events → kunci = nomor urut 10 digit (dipangkas ke 2000)
10
Indexed text lives in `docs` (file content) and `notes` (content + tags).
11
Search uses kilat's inverted index and ranking.
13
from __future__ import annotations
20
from pathlib import Path
22
from .kilat import Kilat
25
"tidak", "ada", "ini", "itu", "yang", "dan", "atau", "dengan", "untuk", "dari", "ke",
26
"di", "pada", "adalah", "bisa", "akan", "saya", "kamu", "kita", "kami", "mereka", "apa",
27
"apakah", "bagaimana", "kenapa", "cara", "juga", "sudah", "belum", "masih", "hanya",
28
"kalau", "jika", "saat", "ketika", "seperti", "tentang", "dalam", "oleh", "agar",
29
"the", "a", "an", "is", "are", "was", "were", "be", "of", "to", "in", "on", "for",
30
"and", "or", "with", "how", "what", "why", "does", "do", "did", "this", "that",
31
"these", "those", "it", "its", "as", "at", "by", "from", "into",
37
return time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())
40
def hash_token(token: str) -> str:
41
return hashlib.sha256(token.encode()).hexdigest()
44
def new_token(kind: str) -> str:
45
return f"ag_{kind}_{secrets.token_urlsafe(24)}"
49
return json.dumps(obj, ensure_ascii=False, default=str)
52
def query_terms(q: str) -> list[str]:
53
words = [w for w in re.findall(r"[0-9A-Za-z_]{3,}", q) if w.lower() not in STOP]
58
def __init__(self, path: str | Path):
60
self.k = Kilat(self.path)
61
indexed = set(self.k.stats().get("indexed") or [])
62
for coll in ("docs", "notes"):
63
if coll not in indexed:
64
self.k.index_coll(coll)
65
self._users: dict[str, dict] = {}
66
self._agents: dict[str, dict] = {}
67
self._by_utoken: dict[str, dict] = {}
68
self._by_atoken: dict[str, dict] = {}
71
# ---------- util ----------
72
def _load_tokens(self) -> None:
73
for _, v in self.k.scan("users"):
75
self._users[str(row["id"])] = row
76
self._by_utoken[row["token_hash"]] = row
77
for _, v in self.k.scan("agents"):
79
self._agents[str(row["id"])] = row
80
self._by_atoken[row["token_hash"]] = row
82
def _put(self, coll: str, key: str, obj: dict) -> None:
83
self.k.put(coll, key, dumps(obj))
85
def _get(self, coll: str, key: str) -> dict | None:
86
raw = self.k.get(coll, key)
87
return json.loads(raw) if raw else None
89
def _next_id(self, coll: str) -> int:
90
cur = int(self.k.get_str("meta", f"seq.{coll}") or "0") + 1
91
self.k.put("meta", f"seq.{coll}", str(cur))
94
# ---------- users & agents ----------
95
def create_user(self, name: str, email: str = "") -> tuple[int, str]:
96
if self.user_by_name(name):
97
raise ValueError(f"user {name!r} already exists")
98
uid = self._next_id("users")
99
token = new_token("usr")
100
row = {"id": uid, "name": name, "email": email, "token_hash": hash_token(token),
102
self._put("users", str(uid), row)
103
self._users[str(uid)] = row
104
self._by_utoken[row["token_hash"]] = row
107
def user_by_name(self, name: str) -> dict | None:
108
for row in self._users.values():
109
if row["name"] == name:
113
def user(self, uid: int) -> dict | None:
114
return self._users.get(str(uid))
116
def user_by_token(self, token: str) -> dict | None:
117
return self._by_utoken.get(hash_token(token))
119
def create_agent(self, user_id: int, name: str, description: str = "", model: str = "") -> tuple[int, str]:
120
for row in self._agents.values():
121
if row["user_id"] == user_id and row["name"] == name:
122
raise ValueError(f"agent {name!r} already exists for this user")
123
aid = self._next_id("agents")
124
token = new_token("agt")
125
row = {"id": aid, "user_id": user_id, "name": name, "description": description,
126
"model": model, "token_hash": hash_token(token), "created_at": now(),
127
"last_seen": None, "calls": 0}
128
self._put("agents", str(aid), row)
129
self._agents[str(aid)] = row
130
self._by_atoken[row["token_hash"]] = row
133
def agent_by_token(self, token: str) -> dict | None:
134
row = self._by_atoken.get(hash_token(token))
138
row["calls"] = row.get("calls", 0) + 1
139
row["last_seen"] = now()
140
row["owner"] = self._users.get(str(row["user_id"]), {}).get("name", "")
141
stored = {k: v for k, v in row.items() if k != "owner"}
142
self._agents[str(row["id"])] = stored
143
self._put("agents", str(row["id"]), stored)
146
# ---------- projects ----------
147
def add_project(self, user_id: int, name: str, url: str, ref: str, path: str,
148
visibility: str = "owner") -> int:
149
if visibility not in ("owner", "public"):
150
raise ValueError("visibility must be 'owner' or 'public'")
151
pid = self._next_id("projects")
152
row = {"id": pid, "user_id": user_id, "name": name, "url": url, "ref": ref, "path": path,
153
"visibility": visibility, "status": "pending", "error": "", "commit_sha": "",
154
"files": 0, "bytes": 0, "chunks": 0, "indexed_at": None, "created_at": now()}
155
self._put("projects", str(pid), row)
158
def project(self, project_id: int) -> dict | None:
159
return self._get("projects", str(project_id))
161
def _with_owner(self, p: dict) -> dict:
163
p["owner"] = self._users.get(str(p["user_id"]), {}).get("name", "")
166
def projects(self) -> list[dict]:
167
return sorted((self._with_owner(json.loads(v)) for _, v in self.k.scan("projects")),
168
key=lambda p: p["id"])
170
def public_projects(self) -> list[dict]:
171
return sorted((p for p in self.projects() if p["visibility"] == "public"),
172
key=lambda p: (-p["chunks"], p["name"]))
174
def all_projects(self) -> list[dict]:
175
return sorted((self._with_owner(json.loads(v)) for _, v in self.k.scan("projects")),
176
key=lambda p: (p["owner"], p["name"]))
178
def project_by_name(self, name: str, only_public: bool = True) -> dict | None:
179
for p in self.projects():
180
if p["name"] == name and (not only_public or p["visibility"] == "public"):
184
def project_ids(self) -> list[int]:
185
return [p["id"] for p in self.projects()]
187
def visible_projects(self, agent: dict) -> list[dict]:
188
return [p for p in self.projects()
189
if p["user_id"] == agent["user_id"] or p["visibility"] == "public"]
191
def can_read(self, agent: dict, project: dict) -> bool:
192
return project["user_id"] == agent["user_id"] or project["visibility"] == "public"
194
def update_project(self, project_id: int, **fields) -> None:
195
row = self.project(project_id)
199
self._put("projects", str(project_id), row)
201
def project_langs(self, project_id: int) -> dict:
202
counts: dict[str, set[str]] = {}
203
for _, raw in self.k.scan_prefix("docmeta", f"{project_id}/"):
205
counts.setdefault(d.get("lang") or "text", set()).add(d["path"])
206
return dict(sorted(((k, len(v)) for k, v in counts.items()), key=lambda kv: -kv[1]))
208
# ---------- jobs ----------
209
def enqueue_job(self, project_id: int) -> int:
210
jid = self._next_id("jobs")
211
self._put("jobs", str(jid), {"id": jid, "project_id": project_id, "status": "queued",
212
"message": "", "created_at": now(), "started_at": None,
213
"finished_at": None})
216
def next_job(self) -> dict | None:
217
jobs = sorted((json.loads(v) for _, v in self.k.scan("jobs")), key=lambda j: j["id"])
219
if j["status"] == "queued":
220
j["status"] = "running"
221
j["started_at"] = now()
222
self._put("jobs", str(j["id"]), j)
226
def finish_job(self, job_id: int, ok: bool, message: str) -> None:
227
j = self._get("jobs", str(job_id))
230
j.update(status="done" if ok else "failed", message=message, finished_at=now())
231
self._put("jobs", str(job_id), j)
233
def latest_job(self, project_id: int) -> dict | None:
234
jobs = [json.loads(v) for _, v in self.k.scan("jobs")]
235
jobs = [j for j in jobs if j["project_id"] == project_id]
236
return max(jobs, key=lambda j: j["id"]) if jobs else None
238
# ---------- documents ----------
239
def replace_docs(self, project_id: int, chunks: list[dict], files: int, nbytes: int,
241
for key, _ in self.k.scan_prefix("docs", f"{project_id}/"):
242
self.k.delete("docs", key)
243
for key, _ in self.k.scan_prefix("docmeta", f"{project_id}/"):
244
self.k.delete("docmeta", key)
245
# impor massal: fsync sekali di akhir, bukan per rekaman
246
self.k.set_sync(False)
249
key = f"{project_id}/{c['path']}#{c['start_line']}"
250
self.k.put_indexed("docs", key, c["body"])
251
self.k.put("docmeta", key, dumps({
252
"project_id": project_id, "path": c["path"], "lang": c["lang"],
253
"start_line": c["start_line"], "end_line": c["end_line"], "body": c["body"],
257
self.k.set_sync(True)
258
self.update_project(project_id, status="ready", error="", commit_sha=commit, files=files,
259
bytes=nbytes, chunks=len(chunks), indexed_at=now())
262
def mark_project(self, project_id: int, status: str, error: str = "") -> None:
263
self.update_project(project_id, status=status, error=error)
265
def _doc_meta(self, key: str) -> dict | None:
266
raw = self.k.get("docmeta", key)
267
return json.loads(raw) if raw else None
269
def search(self, query: str, project_ids: list[int], limit: int = 8) -> list[dict]:
272
ids = set(project_ids)
273
raw = self.k.search("docs", query, max(limit * 6, 24))
275
for h in raw.get("hits", []):
276
meta = self._doc_meta(h["key"])
277
if not meta or meta["project_id"] not in ids:
280
"project_id": meta["project_id"],
281
"path": meta["path"],
282
"lang": meta["lang"],
283
"start_line": meta["start_line"],
284
"end_line": meta["end_line"],
286
"score": -float(h["score"]), # makin kecil makin relevan (gaya bm25)
288
if len(out) >= limit:
292
def doc_text(self, project_id: int, path: str, start: int = 1) -> list[dict]:
294
for _, raw in self.k.scan_prefix("docmeta", f"{project_id}/{path}#"):
296
if d["end_line"] >= start:
298
return sorted(rows, key=lambda d: d["start_line"])
300
def file_map(self, project_id: int) -> list[dict]:
301
agg: dict[str, dict] = {}
302
for _, raw in self.k.scan_prefix("docmeta", f"{project_id}/"):
304
a = agg.setdefault(d["path"], {"path": d["path"], "lang": d["lang"], "chunks": 0,
305
"lines": 0, "bytes": 0})
307
a["lines"] = max(a["lines"], d["end_line"])
308
a["bytes"] += len(d["body"])
309
return sorted(agg.values(), key=lambda a: -a["lines"])
311
def total_lines(self, project_id: int, path: str) -> int:
312
return max((d["end_line"] for d in self.doc_text(project_id, path)), default=0)
314
# ---------- notes ----------
315
def add_note(self, agent_id: int, body: str, tags: str = "", project_id: int | None = None) -> int:
316
nid = self._next_id("notes")
317
self.k.put_indexed("notes", str(nid), f"{body} {tags}")
318
self.k.put("notemeta", str(nid), dumps({"id": nid, "agent_id": agent_id,
319
"project_id": project_id, "body": body,
320
"tags": tags, "created_at": now()}))
323
def recall(self, query: str, agent_id: int, limit: int = 8) -> list[dict]:
324
raw = self.k.search("notes", query, max(limit * 4, 16))
326
for h in raw.get("hits", []):
327
m = self.k.get("notemeta", h["key"])
331
if n["agent_id"] != agent_id:
333
n["snip"] = h["snip"]
334
n["score"] = -float(h["score"])
336
if len(out) >= limit:
340
def count_notes(self, agent_id: int) -> int:
341
return sum(1 for _, raw in self.k.scan("notemeta") if json.loads(raw)["agent_id"] == agent_id)
343
# ---------- audit & stats ----------
344
def log(self, tool: str, ok: bool, ms: int, agent_id: int | None = None, detail: str = "") -> None:
345
seq = int(self.k.get_str("meta", "seq.events") or "0") + 1
346
self.k.put("meta", "seq.events", str(seq))
347
self.k.put("events", f"{seq:010d}", dumps({"ts": now(), "agent_id": agent_id, "tool": tool,
348
"ok": 1 if ok else 0, "ms": ms,
349
"detail": (detail or "")[:400]}))
353
def _trim_events(self) -> None:
354
keys = [k for k, _ in self.k.scan("events")]
355
if len(keys) > EVENTS_KEEP:
356
for k in keys[: len(keys) - EVENTS_KEEP]:
357
self.k.delete("events", k)
359
def recent_events(self, limit: int = 40) -> list[dict]:
360
rows = [json.loads(v) for _, v in self.k.scan("events")]
361
rows.sort(key=lambda r: r["ts"], reverse=True)
364
def agents_list(self) -> list[dict]:
365
counts: dict[int, int] = {}
366
for _, raw in self.k.scan("notemeta"):
368
counts[n["agent_id"]] = counts.get(n["agent_id"], 0) + 1
370
for row in self._agents.values():
372
r["owner"] = self._users.get(str(r["user_id"]), {}).get("name", "")
373
r["notes"] = counts.get(r["id"], 0)
375
return sorted(out, key=lambda r: r["id"])
377
def stats(self) -> dict:
378
projects = [json.loads(v) for _, v in self.k.scan("projects")]
380
"users": self.k.count("users"),
381
"agents": self.k.count("agents"),
382
"projects": len(projects),
383
"projects_ready": sum(1 for p in projects if p["status"] == "ready"),
384
"docs": self.k.count("docmeta"),
385
"notes": self.k.count("notemeta"),
386
"calls": self.k.count("events"),
389
def engine(self) -> dict:
390
return self.k.stats()
392
def compact(self) -> None:
395
def close(self) -> None: