Initial commit: Phase 1+2 prototype
- Go backend with SQLite, JWT auth, file CRUD - Vue 3 frontend with split/raw/WYSIWYG editor modes - Markdown preview (marked, GFM) - Formatting toolbar + keyboard shortcuts - File tree with search, create, delete - Light/dark theme toggle - Admin panel (user management) - Preferences (timezone, theme, default mode) - Shared documents section (placeholder) - Export: PDF, HTML, MD - Build daemon (Python, stdlib only) - Build job queue API - Docker deployment
This commit is contained in:
@@ -0,0 +1,296 @@
|
||||
#!/usr/bin/env python3
|
||||
"""MarkdownHub Build Daemon.
|
||||
|
||||
Polls MarkdownHub for pending build jobs, runs Pi coding agent
|
||||
in a sandboxed workspace, and pushes results to Gitea.
|
||||
|
||||
No external dependencies — stdlib only.
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import urllib.request
|
||||
import urllib.error
|
||||
from pathlib import Path
|
||||
|
||||
# ─── Configuration ───────────────────────────────────────────────────────────
|
||||
MARKDOWNHUB_URL = os.environ.get("MH_URL", "http://localhost:8090")
|
||||
DAEMON_TOKEN = os.environ.get("MH_DAEMON_TOKEN", "")
|
||||
WORKSPACE_ROOT = Path(os.environ.get("MH_WORKSPACE", os.path.expanduser("~/builds")))
|
||||
POLL_INTERVAL = int(os.environ.get("MH_POLL_INTERVAL", "10")) # seconds
|
||||
MAX_CONCURRENT = int(os.environ.get("MH_MAX_CONCURRENT", "1"))
|
||||
DEFAULT_MODEL = os.environ.get("MH_DEFAULT_MODEL", "")
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def log(msg):
|
||||
print(f"[{time.strftime('%H:%M:%S')}] {msg}", flush=True)
|
||||
|
||||
|
||||
def api_post(path, data=None):
|
||||
"""POST to MarkdownHub API."""
|
||||
url = f"{MARKDOWNHUB_URL}/api/{path}"
|
||||
body = json.dumps(data or {}).encode()
|
||||
headers = {
|
||||
"Content-Type": "application/json",
|
||||
"Authorization": f"Bearer {DAEMON_TOKEN}",
|
||||
}
|
||||
req = urllib.request.Request(url, data=body, headers=headers)
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=30) as resp:
|
||||
raw = resp.read()
|
||||
return json.loads(raw) if raw else {}
|
||||
except urllib.error.HTTPError as e:
|
||||
log(f" API error {e.code}: {e.read().decode()}")
|
||||
return None
|
||||
except Exception as e:
|
||||
log(f" Connection error: {e}")
|
||||
return None
|
||||
|
||||
|
||||
def heartbeat():
|
||||
"""Send heartbeat to MarkdownHub."""
|
||||
api_post("daemon/heartbeat", {"status": "online"})
|
||||
|
||||
|
||||
def poll_job():
|
||||
"""Poll for a pending build job."""
|
||||
return api_post("daemon/poll")
|
||||
|
||||
|
||||
def report_status(job_id, status, log_text=""):
|
||||
"""Report job status back to MarkdownHub."""
|
||||
api_post("daemon/report", {
|
||||
"job_id": job_id,
|
||||
"status": status,
|
||||
"log": log_text,
|
||||
})
|
||||
|
||||
|
||||
def setup_workspace(project_name, gitea_url, gitea_token, gitea_org, repo_name):
|
||||
"""Create project folder and initialize git repo."""
|
||||
project_dir = WORKSPACE_ROOT / project_name
|
||||
if project_dir.exists():
|
||||
# Clean existing
|
||||
subprocess.run(["rm", "-rf", str(project_dir)], check=True)
|
||||
|
||||
project_dir.mkdir(parents=True)
|
||||
|
||||
# Init git and set remote
|
||||
remote_url = f"{gitea_url}/{gitea_org}/{repo_name}.git"
|
||||
# Embed token in URL for push
|
||||
if gitea_token and "://" in remote_url:
|
||||
scheme, rest = remote_url.split("://", 1)
|
||||
remote_url = f"{scheme}://oauth2:{gitea_token}@{rest}"
|
||||
|
||||
subprocess.run(["git", "init"], cwd=project_dir, check=True, capture_output=True)
|
||||
subprocess.run(["git", "remote", "add", "origin", remote_url],
|
||||
cwd=project_dir, check=True, capture_output=True)
|
||||
|
||||
return project_dir
|
||||
|
||||
|
||||
def create_gitea_repo(gitea_url, gitea_token, org, repo_name, private=True):
|
||||
"""Create a repo on Gitea via API."""
|
||||
url = f"{gitea_url}/api/v1/orgs/{org}/repos"
|
||||
data = json.dumps({"name": repo_name, "private": private}).encode()
|
||||
headers = {
|
||||
"Content-Type": "application/json",
|
||||
"Authorization": f"token {gitea_token}",
|
||||
}
|
||||
req = urllib.request.Request(url, data=data, headers=headers)
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=15) as resp:
|
||||
return json.loads(resp.read())
|
||||
except urllib.error.HTTPError as e:
|
||||
if e.code == 409:
|
||||
log(f" Repo {org}/{repo_name} already exists")
|
||||
return {"exists": True}
|
||||
log(f" Gitea error {e.code}: {e.read().decode()}")
|
||||
return None
|
||||
|
||||
|
||||
def run_pi(project_dir, spec_content, model):
|
||||
"""Run Pi coding agent in the project directory."""
|
||||
# Write spec to project dir
|
||||
spec_file = project_dir / "spec.md"
|
||||
spec_file.write_text(spec_content, encoding="utf-8")
|
||||
|
||||
# Build Pi command
|
||||
cmd = ["pi", "-p"]
|
||||
prompt = (
|
||||
"Read spec.md in the current directory. "
|
||||
"Implement the complete project as described. "
|
||||
"Create all necessary files, folders, configs, and documentation. "
|
||||
"When done, do NOT commit or push — just write the files."
|
||||
)
|
||||
cmd.append(prompt)
|
||||
|
||||
if model:
|
||||
cmd.extend(["--model", model])
|
||||
|
||||
log(f" Running: {' '.join(cmd[:4])}...")
|
||||
|
||||
# Run Pi
|
||||
proc = subprocess.Popen(
|
||||
cmd,
|
||||
cwd=project_dir,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.STDOUT,
|
||||
text=True,
|
||||
)
|
||||
|
||||
output_lines = []
|
||||
for line in proc.stdout:
|
||||
output_lines.append(line)
|
||||
if len(output_lines) % 20 == 0:
|
||||
log(f" ... {len(output_lines)} lines of output")
|
||||
|
||||
proc.wait()
|
||||
output = "".join(output_lines)
|
||||
|
||||
if proc.returncode != 0:
|
||||
log(f" Pi exited with code {proc.returncode}")
|
||||
return False, output
|
||||
|
||||
return True, output
|
||||
|
||||
|
||||
def git_commit_and_push(project_dir, message="AI-generated project"):
|
||||
"""Commit all files and push to remote."""
|
||||
try:
|
||||
subprocess.run(["git", "add", "."], cwd=project_dir, check=True, capture_output=True)
|
||||
|
||||
# Check if there's anything to commit
|
||||
result = subprocess.run(["git", "status", "--porcelain"],
|
||||
cwd=project_dir, capture_output=True, text=True)
|
||||
if not result.stdout.strip():
|
||||
log(" Nothing to commit")
|
||||
return True
|
||||
|
||||
subprocess.run(
|
||||
["git", "commit", "-m", message],
|
||||
cwd=project_dir, check=True, capture_output=True,
|
||||
env={**os.environ, "GIT_AUTHOR_NAME": "MarkdownHub", "GIT_AUTHOR_EMAIL": "build@markdownhub",
|
||||
"GIT_COMMITTER_NAME": "MarkdownHub", "GIT_COMMITTER_EMAIL": "build@markdownhub"},
|
||||
)
|
||||
subprocess.run(
|
||||
["git", "push", "-u", "origin", "main"],
|
||||
cwd=project_dir, check=True, capture_output=True,
|
||||
)
|
||||
log(" Pushed to remote")
|
||||
return True
|
||||
except subprocess.CalledProcessError as e:
|
||||
log(f" Git error: {e.stderr.decode() if e.stderr else e}")
|
||||
# Try with master branch
|
||||
try:
|
||||
subprocess.run(
|
||||
["git", "push", "-u", "origin", "master"],
|
||||
cwd=project_dir, check=True, capture_output=True,
|
||||
)
|
||||
return True
|
||||
except subprocess.CalledProcessError:
|
||||
return False
|
||||
|
||||
|
||||
def process_job(job):
|
||||
"""Process a single build job."""
|
||||
job_id = job["job_id"]
|
||||
project_name = job.get("repo_name", f"project-{job_id[:8]}")
|
||||
spec_content = job.get("spec_content", "")
|
||||
gitea_url = job.get("gitea_url", "")
|
||||
gitea_token = job.get("gitea_token", "")
|
||||
gitea_org = job.get("gitea_org", "")
|
||||
model = job.get("model", DEFAULT_MODEL)
|
||||
|
||||
log(f"Processing job {job_id}: {gitea_org}/{project_name}")
|
||||
report_status(job_id, "running", "Setting up workspace...")
|
||||
|
||||
# Create Gitea repo
|
||||
if gitea_url and gitea_token and gitea_org:
|
||||
result = create_gitea_repo(gitea_url, gitea_token, gitea_org, project_name)
|
||||
if result is None:
|
||||
report_status(job_id, "failed", "Failed to create Gitea repo")
|
||||
return
|
||||
|
||||
# Setup workspace
|
||||
try:
|
||||
project_dir = setup_workspace(project_name, gitea_url, gitea_token, gitea_org, project_name)
|
||||
except Exception as e:
|
||||
report_status(job_id, "failed", f"Workspace setup failed: {e}")
|
||||
return
|
||||
|
||||
report_status(job_id, "running", "Running Pi agent...")
|
||||
|
||||
# Run Pi
|
||||
success, output = run_pi(project_dir, spec_content, model)
|
||||
if not success:
|
||||
report_status(job_id, "failed", f"Pi failed:\n{output[-2000:]}")
|
||||
return
|
||||
|
||||
report_status(job_id, "running", "Committing and pushing...")
|
||||
|
||||
# Commit and push
|
||||
if git_commit_and_push(project_dir, f"AI-generated: {project_name}"):
|
||||
repo_url = f"{gitea_url}/{gitea_org}/{project_name}"
|
||||
report_status(job_id, "done", f"Success! Repo: {repo_url}")
|
||||
log(f" Done: {repo_url}")
|
||||
else:
|
||||
report_status(job_id, "failed", f"Git push failed\n{output[-1000:]}")
|
||||
|
||||
|
||||
def validate_config():
|
||||
"""Check required configuration."""
|
||||
if not DAEMON_TOKEN:
|
||||
print("Error: MH_DAEMON_TOKEN environment variable required")
|
||||
print("\nConfiguration (environment variables):")
|
||||
print(" MH_URL MarkdownHub URL (default: http://localhost:8090)")
|
||||
print(" MH_DAEMON_TOKEN API token for daemon auth (required)")
|
||||
print(" MH_WORKSPACE Build workspace root (default: ~/builds)")
|
||||
print(" MH_POLL_INTERVAL Poll interval in seconds (default: 10)")
|
||||
print(" MH_MAX_CONCURRENT Max concurrent jobs (default: 1)")
|
||||
print(" MH_DEFAULT_MODEL Default LLM model for Pi")
|
||||
sys.exit(1)
|
||||
|
||||
WORKSPACE_ROOT.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
|
||||
def main():
|
||||
validate_config()
|
||||
|
||||
log(f"MarkdownHub Build Daemon starting")
|
||||
log(f" Server: {MARKDOWNHUB_URL}")
|
||||
log(f" Workspace: {WORKSPACE_ROOT}")
|
||||
log(f" Poll interval: {POLL_INTERVAL}s")
|
||||
log(f" Max concurrent: {MAX_CONCURRENT}")
|
||||
|
||||
last_heartbeat = 0
|
||||
|
||||
while True:
|
||||
try:
|
||||
# Heartbeat every 30s
|
||||
now = time.time()
|
||||
if now - last_heartbeat > 30:
|
||||
heartbeat()
|
||||
last_heartbeat = now
|
||||
|
||||
# Poll for job
|
||||
job = poll_job()
|
||||
if job and job.get("job_id"):
|
||||
process_job(job)
|
||||
else:
|
||||
time.sleep(POLL_INTERVAL)
|
||||
|
||||
except KeyboardInterrupt:
|
||||
log("Shutting down...")
|
||||
break
|
||||
except Exception as e:
|
||||
log(f"Error: {e}")
|
||||
time.sleep(POLL_INTERVAL)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user