aimb-x/agentgit
repositories / aimb-x / agentgit

agentgit public

git clone https://git.169.58.159.131.nip.io/aimb-x/agentgit.git
agentgit/db.py 16272 bytes 1 lines
1
"""agentgit — penyimpanan di atas **kilat** (mesin buatan sendiri), bukan SQLite.
2
3
Koleksi (nilai JSON, kunci string):
4
users/agents/projects/jobs → kunci = id
5
docmeta → kunci = "{project_id}/{path}#{start_line}"
6
notemeta → key = note id
7
events → kunci = nomor urut 10 digit (dipangkas ke 2000)
8
meta → penghitung id
9
10
Indexed text lives in `docs` (file content) and `notes` (content + tags).
11
Search uses kilat's inverted index and ranking.
12
"""
13
from __future__ import annotations
14
15
import hashlib
16
import json
17
import re
18
import secrets
19
import time
20
from pathlib import Path
21
22
from .kilat import Kilat
23
24
STOP = {
25
"tidak", "ada", "ini", "itu", "yang", "dan", "atau", "dengan", "untuk", "dari", "ke",
26
"di", "pada", "adalah", "bisa", "akan", "saya", "kamu", "kita", "kami", "mereka", "apa",
27
"apakah", "bagaimana", "kenapa", "cara", "juga", "sudah", "belum", "masih", "hanya",
28
"kalau", "jika", "saat", "ketika", "seperti", "tentang", "dalam", "oleh", "agar",
29
"the", "a", "an", "is", "are", "was", "were", "be", "of", "to", "in", "on", "for",
30
"and", "or", "with", "how", "what", "why", "does", "do", "did", "this", "that",
31
"these", "those", "it", "its", "as", "at", "by", "from", "into",
32
}
33
EVENTS_KEEP = 2000
34
35
36
def now() -> str:
37
return time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())
38
39
40
def hash_token(token: str) -> str:
41
return hashlib.sha256(token.encode()).hexdigest()
42
43
44
def new_token(kind: str) -> str:
45
return f"ag_{kind}_{secrets.token_urlsafe(24)}"
46
47
48
def dumps(obj) -> str:
49
return json.dumps(obj, ensure_ascii=False, default=str)
50
51
52
def query_terms(q: str) -> list[str]:
53
words = [w for w in re.findall(r"[0-9A-Za-z_]{3,}", q) if w.lower() not in STOP]
54
return words[:12]
55
56
57
class Store:
58
def __init__(self, path: str | Path):
59
self.path = str(path)
60
self.k = Kilat(self.path)
61
indexed = set(self.k.stats().get("indexed") or [])
62
for coll in ("docs", "notes"):
63
if coll not in indexed:
64
self.k.index_coll(coll)
65
self._users: dict[str, dict] = {}
66
self._agents: dict[str, dict] = {}
67
self._by_utoken: dict[str, dict] = {}
68
self._by_atoken: dict[str, dict] = {}
69
self._load_tokens()
70
71
# ---------- util ----------
72
def _load_tokens(self) -> None:
73
for _, v in self.k.scan("users"):
74
row = json.loads(v)
75
self._users[str(row["id"])] = row
76
self._by_utoken[row["token_hash"]] = row
77
for _, v in self.k.scan("agents"):
78
row = json.loads(v)
79
self._agents[str(row["id"])] = row
80
self._by_atoken[row["token_hash"]] = row
81
82
def _put(self, coll: str, key: str, obj: dict) -> None:
83
self.k.put(coll, key, dumps(obj))
84
85
def _get(self, coll: str, key: str) -> dict | None:
86
raw = self.k.get(coll, key)
87
return json.loads(raw) if raw else None
88
89
def _next_id(self, coll: str) -> int:
90
cur = int(self.k.get_str("meta", f"seq.{coll}") or "0") + 1
91
self.k.put("meta", f"seq.{coll}", str(cur))
92
return cur
93
94
# ---------- users & agents ----------
95
def create_user(self, name: str, email: str = "") -> tuple[int, str]:
96
if self.user_by_name(name):
97
raise ValueError(f"user {name!r} already exists")
98
uid = self._next_id("users")
99
token = new_token("usr")
100
row = {"id": uid, "name": name, "email": email, "token_hash": hash_token(token),
101
"created_at": now()}
102
self._put("users", str(uid), row)
103
self._users[str(uid)] = row
104
self._by_utoken[row["token_hash"]] = row
105
return uid, token
106
107
def user_by_name(self, name: str) -> dict | None:
108
for row in self._users.values():
109
if row["name"] == name:
110
return row
111
return None
112
113
def user(self, uid: int) -> dict | None:
114
return self._users.get(str(uid))
115
116
def user_by_token(self, token: str) -> dict | None:
117
return self._by_utoken.get(hash_token(token))
118
119
def create_agent(self, user_id: int, name: str, description: str = "", model: str = "") -> tuple[int, str]:
120
for row in self._agents.values():
121
if row["user_id"] == user_id and row["name"] == name:
122
raise ValueError(f"agent {name!r} already exists for this user")
123
aid = self._next_id("agents")
124
token = new_token("agt")
125
row = {"id": aid, "user_id": user_id, "name": name, "description": description,
126
"model": model, "token_hash": hash_token(token), "created_at": now(),
127
"last_seen": None, "calls": 0}
128
self._put("agents", str(aid), row)
129
self._agents[str(aid)] = row
130
self._by_atoken[row["token_hash"]] = row
131
return aid, token
132
133
def agent_by_token(self, token: str) -> dict | None:
134
row = self._by_atoken.get(hash_token(token))
135
if not row:
136
return None
137
row = dict(row)
138
row["calls"] = row.get("calls", 0) + 1
139
row["last_seen"] = now()
140
row["owner"] = self._users.get(str(row["user_id"]), {}).get("name", "")
141
stored = {k: v for k, v in row.items() if k != "owner"}
142
self._agents[str(row["id"])] = stored
143
self._put("agents", str(row["id"]), stored)
144
return row
145
146
# ---------- projects ----------
147
def add_project(self, user_id: int, name: str, url: str, ref: str, path: str,
148
visibility: str = "owner") -> int:
149
if visibility not in ("owner", "public"):
150
raise ValueError("visibility must be 'owner' or 'public'")
151
pid = self._next_id("projects")
152
row = {"id": pid, "user_id": user_id, "name": name, "url": url, "ref": ref, "path": path,
153
"visibility": visibility, "status": "pending", "error": "", "commit_sha": "",
154
"files": 0, "bytes": 0, "chunks": 0, "indexed_at": None, "created_at": now()}
155
self._put("projects", str(pid), row)
156
return pid
157
158
def project(self, project_id: int) -> dict | None:
159
return self._get("projects", str(project_id))
160
161
def _with_owner(self, p: dict) -> dict:
162
p = dict(p)
163
p["owner"] = self._users.get(str(p["user_id"]), {}).get("name", "")
164
return p
165
166
def projects(self) -> list[dict]:
167
return sorted((self._with_owner(json.loads(v)) for _, v in self.k.scan("projects")),
168
key=lambda p: p["id"])
169
170
def public_projects(self) -> list[dict]:
171
return sorted((p for p in self.projects() if p["visibility"] == "public"),
172
key=lambda p: (-p["chunks"], p["name"]))
173
174
def all_projects(self) -> list[dict]:
175
return sorted((self._with_owner(json.loads(v)) for _, v in self.k.scan("projects")),
176
key=lambda p: (p["owner"], p["name"]))
177
178
def project_by_name(self, name: str, only_public: bool = True) -> dict | None:
179
for p in self.projects():
180
if p["name"] == name and (not only_public or p["visibility"] == "public"):
181
return p
182
return None
183
184
def project_ids(self) -> list[int]:
185
return [p["id"] for p in self.projects()]
186
187
def visible_projects(self, agent: dict) -> list[dict]:
188
return [p for p in self.projects()
189
if p["user_id"] == agent["user_id"] or p["visibility"] == "public"]
190
191
def can_read(self, agent: dict, project: dict) -> bool:
192
return project["user_id"] == agent["user_id"] or project["visibility"] == "public"
193
194
def update_project(self, project_id: int, **fields) -> None:
195
row = self.project(project_id)
196
if not row:
197
return
198
row.update(fields)
199
self._put("projects", str(project_id), row)
200
201
def project_langs(self, project_id: int) -> dict:
202
counts: dict[str, set[str]] = {}
203
for _, raw in self.k.scan_prefix("docmeta", f"{project_id}/"):
204
d = json.loads(raw)
205
counts.setdefault(d.get("lang") or "text", set()).add(d["path"])
206
return dict(sorted(((k, len(v)) for k, v in counts.items()), key=lambda kv: -kv[1]))
207
208
# ---------- jobs ----------
209
def enqueue_job(self, project_id: int) -> int:
210
jid = self._next_id("jobs")
211
self._put("jobs", str(jid), {"id": jid, "project_id": project_id, "status": "queued",
212
"message": "", "created_at": now(), "started_at": None,
213
"finished_at": None})
214
return jid
215
216
def next_job(self) -> dict | None:
217
jobs = sorted((json.loads(v) for _, v in self.k.scan("jobs")), key=lambda j: j["id"])
218
for j in jobs:
219
if j["status"] == "queued":
220
j["status"] = "running"
221
j["started_at"] = now()
222
self._put("jobs", str(j["id"]), j)
223
return j
224
return None
225
226
def finish_job(self, job_id: int, ok: bool, message: str) -> None:
227
j = self._get("jobs", str(job_id))
228
if not j:
229
return
230
j.update(status="done" if ok else "failed", message=message, finished_at=now())
231
self._put("jobs", str(job_id), j)
232
233
def latest_job(self, project_id: int) -> dict | None:
234
jobs = [json.loads(v) for _, v in self.k.scan("jobs")]
235
jobs = [j for j in jobs if j["project_id"] == project_id]
236
return max(jobs, key=lambda j: j["id"]) if jobs else None
237
238
# ---------- documents ----------
239
def replace_docs(self, project_id: int, chunks: list[dict], files: int, nbytes: int,
240
commit: str) -> int:
241
for key, _ in self.k.scan_prefix("docs", f"{project_id}/"):
242
self.k.delete("docs", key)
243
for key, _ in self.k.scan_prefix("docmeta", f"{project_id}/"):
244
self.k.delete("docmeta", key)
245
# impor massal: fsync sekali di akhir, bukan per rekaman
246
self.k.set_sync(False)
247
try:
248
for c in chunks:
249
key = f"{project_id}/{c['path']}#{c['start_line']}"
250
self.k.put_indexed("docs", key, c["body"])
251
self.k.put("docmeta", key, dumps({
252
"project_id": project_id, "path": c["path"], "lang": c["lang"],
253
"start_line": c["start_line"], "end_line": c["end_line"], "body": c["body"],
254
}))
255
finally:
256
self.k.sync()
257
self.k.set_sync(True)
258
self.update_project(project_id, status="ready", error="", commit_sha=commit, files=files,
259
bytes=nbytes, chunks=len(chunks), indexed_at=now())
260
return len(chunks)
261
262
def mark_project(self, project_id: int, status: str, error: str = "") -> None:
263
self.update_project(project_id, status=status, error=error)
264
265
def _doc_meta(self, key: str) -> dict | None:
266
raw = self.k.get("docmeta", key)
267
return json.loads(raw) if raw else None
268
269
def search(self, query: str, project_ids: list[int], limit: int = 8) -> list[dict]:
270
if not project_ids:
271
return []
272
ids = set(project_ids)
273
raw = self.k.search("docs", query, max(limit * 6, 24))
274
out = []
275
for h in raw.get("hits", []):
276
meta = self._doc_meta(h["key"])
277
if not meta or meta["project_id"] not in ids:
278
continue
279
out.append({
280
"project_id": meta["project_id"],
281
"path": meta["path"],
282
"lang": meta["lang"],
283
"start_line": meta["start_line"],
284
"end_line": meta["end_line"],
285
"snip": h["snip"],
286
"score": -float(h["score"]), # makin kecil makin relevan (gaya bm25)
287
})
288
if len(out) >= limit:
289
break
290
return out
291
292
def doc_text(self, project_id: int, path: str, start: int = 1) -> list[dict]:
293
rows = []
294
for _, raw in self.k.scan_prefix("docmeta", f"{project_id}/{path}#"):
295
d = json.loads(raw)
296
if d["end_line"] >= start:
297
rows.append(d)
298
return sorted(rows, key=lambda d: d["start_line"])
299
300
def file_map(self, project_id: int) -> list[dict]:
301
agg: dict[str, dict] = {}
302
for _, raw in self.k.scan_prefix("docmeta", f"{project_id}/"):
303
d = json.loads(raw)
304
a = agg.setdefault(d["path"], {"path": d["path"], "lang": d["lang"], "chunks": 0,
305
"lines": 0, "bytes": 0})
306
a["chunks"] += 1
307
a["lines"] = max(a["lines"], d["end_line"])
308
a["bytes"] += len(d["body"])
309
return sorted(agg.values(), key=lambda a: -a["lines"])
310
311
def total_lines(self, project_id: int, path: str) -> int:
312
return max((d["end_line"] for d in self.doc_text(project_id, path)), default=0)
313
314
# ---------- notes ----------
315
def add_note(self, agent_id: int, body: str, tags: str = "", project_id: int | None = None) -> int:
316
nid = self._next_id("notes")
317
self.k.put_indexed("notes", str(nid), f"{body} {tags}")
318
self.k.put("notemeta", str(nid), dumps({"id": nid, "agent_id": agent_id,
319
"project_id": project_id, "body": body,
320
"tags": tags, "created_at": now()}))
321
return nid
322
323
def recall(self, query: str, agent_id: int, limit: int = 8) -> list[dict]:
324
raw = self.k.search("notes", query, max(limit * 4, 16))
325
out = []
326
for h in raw.get("hits", []):
327
m = self.k.get("notemeta", h["key"])
328
if not m:
329
continue
330
n = json.loads(m)
331
if n["agent_id"] != agent_id:
332
continue
333
n["snip"] = h["snip"]
334
n["score"] = -float(h["score"])
335
out.append(n)
336
if len(out) >= limit:
337
break
338
return out
339
340
def count_notes(self, agent_id: int) -> int:
341
return sum(1 for _, raw in self.k.scan("notemeta") if json.loads(raw)["agent_id"] == agent_id)
342
343
# ---------- audit & stats ----------
344
def log(self, tool: str, ok: bool, ms: int, agent_id: int | None = None, detail: str = "") -> None:
345
seq = int(self.k.get_str("meta", "seq.events") or "0") + 1
346
self.k.put("meta", "seq.events", str(seq))
347
self.k.put("events", f"{seq:010d}", dumps({"ts": now(), "agent_id": agent_id, "tool": tool,
348
"ok": 1 if ok else 0, "ms": ms,
349
"detail": (detail or "")[:400]}))
350
if seq % 200 == 0:
351
self._trim_events()
352
353
def _trim_events(self) -> None:
354
keys = [k for k, _ in self.k.scan("events")]
355
if len(keys) > EVENTS_KEEP:
356
for k in keys[: len(keys) - EVENTS_KEEP]:
357
self.k.delete("events", k)
358
359
def recent_events(self, limit: int = 40) -> list[dict]:
360
rows = [json.loads(v) for _, v in self.k.scan("events")]
361
rows.sort(key=lambda r: r["ts"], reverse=True)
362
return rows[:limit]
363
364
def agents_list(self) -> list[dict]:
365
counts: dict[int, int] = {}
366
for _, raw in self.k.scan("notemeta"):
367
n = json.loads(raw)
368
counts[n["agent_id"]] = counts.get(n["agent_id"], 0) + 1
369
out = []
370
for row in self._agents.values():
371
r = dict(row)
372
r["owner"] = self._users.get(str(r["user_id"]), {}).get("name", "")
373
r["notes"] = counts.get(r["id"], 0)
374
out.append(r)
375
return sorted(out, key=lambda r: r["id"])
376
377
def stats(self) -> dict:
378
projects = [json.loads(v) for _, v in self.k.scan("projects")]
379
return {
380
"users": self.k.count("users"),
381
"agents": self.k.count("agents"),
382
"projects": len(projects),
383
"projects_ready": sum(1 for p in projects if p["status"] == "ready"),
384
"docs": self.k.count("docmeta"),
385
"notes": self.k.count("notemeta"),
386
"calls": self.k.count("events"),
387
}
388
389
def engine(self) -> dict:
390
return self.k.stats()
391
392
def compact(self) -> None:
393
self.k.compact()
394
395
def close(self) -> None:
396
self.k.close()