devsite-builder
gitweb.py
#!/usr/bin/env python3
"""
A tiny static git repository browser, in the spirit of
stagit/cgit-static: scan a directory of bare repositories and
generate plain HTML pages for each one (summary, log, commits,
file tree, files, refs).
"""
import os
import re
import html
import shutil
import argparse
import subprocess
from dataclasses import dataclass, field
from datetime import datetime
from siteutils import load_template, render, markdown
REPO_DIR = "repos"
OUTPUT_DIR = "www/git"
CLONE_BASE = "https://milanmajor.dev/git"
RECORD_SEP = "\x1e"
FIELD_SEP = "\x1f"
LOG_FORMAT = FIELD_SEP.join([
"%H", "%P", "%an", "%ae", "%ad", "%s", "%b"
]) + RECORD_SEP
# ------------------------------------------------------------
# Git plumbing
# ------------------------------------------------------------
def decode(raw):
"""
Decode git output as UTF-8, falling back to Latin-1 (which
can represent any byte sequence) for old repositories that
predate UTF-8 conventions.
"""
try:
return raw.decode("utf-8")
except UnicodeDecodeError:
return raw.decode("latin-1")
def git_bytes(git_dir, *args):
"""
Run a git plumbing command against a specific repository
and return its raw stdout bytes.
"""
result = subprocess.run(
["git", "--git-dir", git_dir, *args],
capture_output=True
)
if result.returncode != 0:
raise RuntimeError(
f"git {' '.join(args)} failed in {git_dir}: "
f"{decode(result.stderr).strip()}"
)
return result.stdout
def git(git_dir, *args):
"""
Run a git plumbing command against a specific repository
and return its stdout as text.
"""
return decode(git_bytes(git_dir, *args))
def default_branch(git_dir):
try:
output = git(git_dir, "symbolic-ref", "HEAD", "--short")
return output.strip()
except RuntimeError:
return None
# ------------------------------------------------------------
# Commit
# ------------------------------------------------------------
@dataclass
class Commit:
repo: "Repo"
hash: str
parent: str
author_name: str
author_email: str
date: datetime
subject: str
body: str
@property
def short_hash(self):
return self.hash[:7]
@property
def url(self):
return f"{self.repo.url}commit/{self.hash}.html"
@property
def date_human(self):
return self.date.strftime("%Y-%m-%d %H:%M")
def parse_commits(repo, output):
commits = []
for record in output.split(RECORD_SEP):
record = record.strip("\n")
if not record.strip():
continue
fields = record.split(FIELD_SEP)
commit_hash, parents, author_name, author_email, \
date, subject, body = fields
parent = parents.split(" ")[0] if parents else ""
commits.append(Commit(
repo=repo,
hash=commit_hash,
parent=parent,
author_name=author_name,
author_email=author_email,
date=datetime.strptime(date, "%Y-%m-%d %H:%M:%S"),
subject=subject,
body=body.strip()
))
return commits
def parse_shortstat(text):
"""
Parse the summary line produced by `git show --shortstat`,
e.g. " 2 files changed, 13 insertions(+), 4 deletions(-)",
into a (files, insertions, deletions) tuple of ints. Any
component absent from the line (e.g. a commit with no
insertions) is reported as 0.
"""
def extract(pattern):
match = re.search(pattern, text) if text else None
return int(match.group(1)) if match else 0
files = extract(r"(\d+) files? changed")
insertions = extract(r"(\d+) insertions?\(\+\)")
deletions = extract(r"(\d+) deletions?\(-\)")
return files, insertions, deletions
# ------------------------------------------------------------
# Repo
# ------------------------------------------------------------
@dataclass
class Repo:
name: str
path: str
description: str = ""
@property
def url(self):
return f"/git/{self.name}/"
@property
def clone_url(self):
return f"git clone {CLONE_BASE}/{self.name}.git"
def branch(self):
"""
The repository's "default" branch for building root-level
pages. Prefers whatever HEAD points at, but falls back to
the first branch if HEAD is unborn/detached or points at a
ref that doesn't actually exist (common on repos mirrored
from elsewhere, or converted from another VCS).
"""
branches = self.branches()
if not branches:
return None
head = default_branch(self.path)
return head if head in branches else branches[0]
def log(self, ref=None, limit=None, path=None):
args = ["log", "--pretty=format:" + LOG_FORMAT,
"--date=format:%Y-%m-%d %H:%M:%S"]
if limit:
args.insert(1, f"-n{limit}")
if ref:
args.append(ref)
if path:
args += ["--", path]
output = git(self.path, *args)
return parse_commits(self, output)
def last_commit(self):
try:
commits = self.log(limit=1)
except RuntimeError:
return None
return commits[0] if commits else None
def last_commit_for(self, ref, path):
"""
Return the most recent Commit touching `path` reachable
from `ref`, or None if there isn't one (e.g. broken ref).
"""
try:
commits = self.log(ref=ref, limit=1, path=path)
except RuntimeError:
return None
return commits[0] if commits else None
def branches(self):
output = git(
self.path,
"for-each-ref",
"--format=%(refname:short)",
"refs/heads/"
)
return [
line.strip()
for line in output.splitlines()
if line.strip()
]
def tree(self, ref, path=""):
"""
Return a non-recursive directory listing at `path` for
`ref` as a list of (mode, type, size, name) tuples.
"""
target = f"{ref}:{path}" if path else ref
try:
output = git(self.path, "ls-tree", "-l", target)
except RuntimeError:
return []
entries = []
for line in output.splitlines():
if not line.strip():
continue
meta, name = line.split("\t", 1)
mode, kind, blob_hash, size = meta.split()
entries.append((mode, kind, size, name))
return entries
def read_file(self, ref, path):
try:
raw = git_bytes(self.path, "show", f"{ref}:{path}")
except RuntimeError:
return None
# A NUL byte is a reliable sign of genuinely binary
# content; anything else can be decoded (falling back
# to Latin-1 for pre-UTF-8 era files).
if b"\x00" in raw:
return None
return decode(raw)
def readme(self, ref):
"""
Return the contents of a top-level README file, if any
(tried in order of preference), for display on the
repo's summary page.
"""
for name in ("README.md", "README", "README.txt"):
content = self.read_file(ref, name)
if content is not None:
return content
return None
def last_change(self, ref, path=None):
"""
Return the date of the most recent commit touching
`path` (or the whole tree if `path` is None) reachable
from `ref`.
"""
args = [
"log", "-1",
"--pretty=format:%ad",
"--date=format:%b %d %H:%M",
ref,
]
if path:
args += ["--", path]
output = git(self.path, *args).strip()
return output or "-"
def shortstat(self, commit):
"""
Return (files, insertions, deletions) for a single
commit without generating the (potentially large) full
patch text.
"""
output = git(
self.path,
"show", "--shortstat", "--format=", commit.hash
).strip()
return parse_shortstat(output)
def diff(self, commit):
stat = git(
self.path,
"show", "--stat", "--format=", commit.hash
).strip()
patch = git(
self.path,
"show", "--format=", commit.hash
).strip()
return stat, patch, self.shortstat(commit)
def resolve_git_dir(path):
"""
Determine the actual git-dir for a discovered `repos/*.git`
entry. It might be a proper bare repository, or (as is easy
to end up with by accident) a regular working copy whose
directory just happens to be named "something.git" - in
that case its real git-dir is the nested `.git`.
Returns the resolved git-dir, or None if `path` isn't a git
repository at all.
"""
if (
os.path.isfile(os.path.join(path, "HEAD"))
and os.path.isdir(os.path.join(path, "objects"))
):
return path
nested = os.path.join(path, ".git")
if os.path.isdir(nested) or os.path.isfile(nested):
return nested
return None
def discover_repos():
repos = []
if not os.path.isdir(REPO_DIR):
return repos
for name in sorted(os.listdir(REPO_DIR)):
if not name.endswith(".git"):
continue
candidate = os.path.join(REPO_DIR, name)
if not os.path.isdir(candidate):
continue
git_dir = resolve_git_dir(candidate)
if git_dir is None:
print(f" skip: {name} (not a git repository)")
continue
description = ""
description_path = os.path.join(git_dir, "description")
if os.path.isfile(description_path):
with open(description_path, "r", encoding="utf-8") as f:
text = f.read().strip()
# Git ships a placeholder description in every
# freshly initialized repository; treat it as empty.
if not text.startswith("Unnamed repository"):
description = text
repos.append(Repo(
name=name[:-4],
path=git_dir,
description=description
))
return repos
# ------------------------------------------------------------
# Rendering
# ------------------------------------------------------------
def format_stat_summary(files, insertions, deletions):
if not files:
return "—"
file_label = "file" if files == 1 else "files"
return f'{files} {file_label}, +{insertions} -{deletions}'
def render_commit_row(repo, commit):
files, insertions, deletions = repo.shortstat(commit)
return (
"<tr>"
f'<td>{commit.date_human}</td>'
f'<td><a href="{commit.url}">'
f'{html.escape(commit.subject)}</a></td>'
f'<td>{html.escape(commit.author_name)}</td>'
f'<td class="num" align="right">{files}</td>'
f'<td class="num" align="right">+{insertions}</td>'
f'<td class="num" align="right">-{deletions}</td>'
"</tr>"
)
def render_log(repo, commits):
if not commits:
return '<tr><td colspan="6">No commits.</td></tr>'
return "\n".join(
render_commit_row(repo, commit)
for commit in commits
)
def mode_to_perm(mode, kind):
"""Render a git file mode as an `ls -l`-style permission string."""
if kind == "tree":
return "drwxr-xr-x"
if mode == "120000":
return "lrwxrwxrwx"
if mode == "160000":
return "m---------"
def octal_to_rwx(digit):
digit = int(digit)
return (
("r" if digit & 4 else "-")
+ ("w" if digit & 2 else "-")
+ ("x" if digit & 1 else "-")
)
bits = mode[-3:].rjust(3, "0")
return "-" + "".join(octal_to_rwx(d) for d in bits)
def format_size(size):
"""Render a byte count the way `ls -lh` does: plain below 1024,
otherwise one decimal place with a K/M/G suffix."""
try:
n = float(size)
except ValueError:
return size
if n < 1024:
return str(int(n))
for unit in ("K", "M"):
n /= 1024
if n < 1024:
return f"{n:.1f}{unit}"
return f"{n / 1024:.1f}G"
FOLDER_ICON = "📁"
FILE_ICON = "📄"
def entry_icon(kind):
return FOLDER_ICON if kind == "tree" else FILE_ICON
def render_tree_row(perm, link_target, size, date, numeric=True):
size_cell = (
f'<td class="num" align="right">{size}</td>'
if numeric else "<td></td>"
)
return (
"<tr>"
f"<td>{perm}</td>"
f"<td>{link_target}</td>"
f"{size_cell}"
f"<td>{date}</td>"
"</tr>"
)
def render_tree_entries(repo, ref, path, entries, url_base):
"""
Render a directory listing as a table of rows (mode, name,
size, last-changed date), in the same tabular style as the
commit log page.
"""
rows = []
if path:
parent = "/".join(path.split("/")[:-1])
parent_url = (
f"{url_base}files/{parent}/index.html"
if parent else f"{url_base}files/index.html"
)
rows.append(render_tree_row(
"drwxr-xr-x",
f'<a href="{parent_url}">{FOLDER_ICON} ..</a>', "", "",
numeric=False
))
for mode, kind, size, name in entries:
full_path = f"{path}/{name}" if path else name
date = repo.last_change(ref, full_path)
if kind == "tree":
url = f"{url_base}files/{full_path}/index.html"
label = f"{name}/"
size_label = "-"
else:
url = f"{url_base}file/{full_path}.html"
label = name
size_label = format_size(size)
perm = mode_to_perm(mode, kind)
icon = entry_icon(kind)
link = f'<a href="{url}">{icon} {html.escape(label)}</a>'
rows.append(render_tree_row(perm, link, size_label, date))
if not rows:
return '<tr><td colspan="4">Empty directory.</td></tr>'
return "\n".join(rows)
def render_repo_files(repo, ref, url_base):
"""
Render the repo's root directory listing for the summary
page, GitHub-style: name, last commit touching it (linked
to the commit page), and its date.
"""
entries = repo.tree(ref, "")
if not entries:
return '<tr><td colspan="3">Empty repository.</td></tr>'
rows = []
for mode, kind, size, name in entries:
icon = entry_icon(kind)
if kind == "tree":
url = f"{url_base}files/{name}/index.html"
label = f"{name}/"
else:
url = f"{url_base}file/{name}.html"
label = name
commit = repo.last_commit_for(ref, name)
if commit:
commit_cell = (
f'<a href="{commit.url}">'
f'{html.escape(commit.subject)}</a>'
)
date_cell = commit.date_human
else:
commit_cell = "—"
date_cell = "—"
rows.append(
"<tr>"
f'<td><a href="{url}">{icon} {html.escape(label)}</a></td>'
f'<td>{commit_cell}</td>'
f'<td>{date_cell}</td>'
"</tr>"
)
return "\n".join(rows)
def branch_paths(repo, branch, default_ref):
"""
Return (url_base, output_base) for a given branch: the
default branch's pages live at the repo root, every other
branch gets its own mirrored subtree under branch/<name>/.
"""
if branch == default_ref:
return repo.url, os.path.join(OUTPUT_DIR, repo.name)
return (
f"{repo.url}branch/{branch}/",
os.path.join(OUTPUT_DIR, repo.name, "branch", branch),
)
PAGE_SUFFIX = {
"summary": "index.html",
"log": "log.html",
"files": "files/index.html",
}
def render_branch_select(repo, current_branch, default_ref, page):
suffix = PAGE_SUFFIX[page]
options = []
for branch in repo.branches():
url_base, _ = branch_paths(repo, branch, default_ref)
selected = " selected" if branch == current_branch else ""
options.append(
f'<option value="{url_base}{suffix}"{selected}>'
f'{html.escape(branch)}</option>'
)
return (
'<select class="branch-select" '
'onchange="if (this.value) location.href = this.value;">'
+ "".join(options) +
"</select>"
)
def render_repo_bar(repo, current_branch, default_ref, page):
"""
The GitHub-style bar shown at the top of every repo page:
a branch switcher plus branch count on the left, and a
disclosure widget (a plain <details>, no JS needed to open
it) with a copyable HTTPS clone URL on the right.
"""
branches = repo.branches()
count = len(branches)
label = "branch" if count == 1 else "branches"
clone_url = html.escape(repo.clone_url)
return (
'<div class="repo-bar">'
'<div class="repo-bar-branches">'
'<span class="branch-icon">⎇</span>'
+ render_branch_select(repo, current_branch, default_ref, page) +
f'<span class="branch-count">{count} {label}</span>'
'</div>'
'<details class="clone-box">'
'<summary>Code</summary>'
'<div class="clone-panel">'
'<label for="clone-url" class="clone-label">'
'Clone with HTTPS</label>'
'<div class="clone-field">'
f'<input id="clone-url" type="text" readonly value="{clone_url}" '
'onclick="this.select()">'
'<button type="button" '
'onclick="navigator.clipboard.writeText('
'this.previousElementSibling.value); '
"this.textContent = 'Copied'; "
"setTimeout(() => this.textContent = 'Copy', 1500);"
'">Copy</button>'
'</div>'
'</div>'
'</details>'
'</div>'
)
def nav_values(repo, branch, default_ref, page):
url_base, _ = branch_paths(repo, branch, default_ref)
return {
"name": html.escape(repo.name),
"repo_url": url_base,
"log_url": f"{url_base}log.html",
"files_url": f"{url_base}files/index.html",
"repo_bar": render_repo_bar(repo, branch, default_ref, page),
}
# ------------------------------------------------------------
# Build
# ------------------------------------------------------------
def clean():
if os.path.isdir(OUTPUT_DIR):
shutil.rmtree(OUTPUT_DIR)
os.makedirs(OUTPUT_DIR, exist_ok=True)
def write(path, text):
os.makedirs(os.path.dirname(path), exist_ok=True)
with open(path, "w", encoding="utf-8") as f:
f.write(text)
def build_index(repos):
rows = []
for repo in repos:
last = repo.last_commit()
last_date = last.date_human if last else "—"
rows.append(
"<tr>"
f'<td><a href="{repo.url}">{html.escape(repo.name)}</a></td>'
f'<td>{html.escape(repo.description)}</td>'
f'<td>{last_date}</td>'
"</tr>"
)
template = load_template("git-index.html")
write(
os.path.join(OUTPUT_DIR, "index.html"),
render(template, {"repos": "\n".join(rows)})
)
def build_repo_summary(repo, branch, default_ref):
readme = repo.readme(branch)
url_base, output_base = branch_paths(repo, branch, default_ref)
values = {
**nav_values(repo, branch, default_ref, page="summary"),
"description": html.escape(repo.description),
"files": render_repo_files(repo, branch, url_base),
"readme": (
f'<div class="readme">'
f'{markdown(readme, heading_offset=1)}</div>'
if readme else ""
),
}
template = load_template("git-repo.html")
write(
os.path.join(output_base, "index.html"),
render(template, values)
)
def build_repo_log(repo, branch, default_ref):
commits = repo.log(ref=branch)
_, output_base = branch_paths(repo, branch, default_ref)
values = {
**nav_values(repo, branch, default_ref, page="log"),
"log": render_log(repo, commits),
}
template = load_template("git-log.html")
write(
os.path.join(output_base, "log.html"),
render(template, values)
)
def build_repo_commits(repo, commits, default_ref):
template = load_template("git-commit.html")
for commit in commits:
stat, patch, (files, insertions, deletions) = repo.diff(commit)
parent_html = (
f'<a href="{repo.url}commit/{commit.parent}.html">'
f'{commit.parent[:7]}</a>'
if commit.parent else "(root commit)"
)
body_html = ""
if commit.body:
body_html = (
"<pre class=\"commit-body\">"
f"{html.escape(commit.body)}</pre>"
)
values = {
**nav_values(repo, default_ref, default_ref, page="log"),
"subject": html.escape(commit.subject),
"author": html.escape(
f"{commit.author_name} <{commit.author_email}>"
),
"date": commit.date_human,
"hash": commit.hash,
"parent": parent_html,
"body": body_html,
"summary": format_stat_summary(files, insertions, deletions),
"stat": html.escape(stat) if stat else "(no changes)",
"diff": html.escape(patch) if patch else "(no diff)",
}
write(
os.path.join(
OUTPUT_DIR, repo.name, "commit", f"{commit.hash}.html"
),
render(template, values)
)
def build_repo_tree(repo, branch, default_ref, path=""):
entries = repo.tree(branch, path)
url_base, output_base = branch_paths(repo, branch, default_ref)
values = {
**nav_values(repo, branch, default_ref, page="files"),
"path": html.escape(path) if path else "/",
"path_suffix": f": {html.escape(path)}" if path else "",
"entries": render_tree_entries(
repo, branch, path, entries, url_base
),
}
template = load_template("git-files.html")
output = os.path.join(output_base, "files")
if path:
output = os.path.join(output, path)
write(
os.path.join(output, "index.html"),
render(template, values)
)
for mode, kind, size, name in entries:
full_path = f"{path}/{name}" if path else name
if kind == "tree":
build_repo_tree(repo, branch, default_ref, full_path)
elif kind == "blob":
build_repo_file(repo, branch, default_ref, full_path)
def build_repo_file(repo, branch, default_ref, path):
content = repo.read_file(branch, path)
_, output_base = branch_paths(repo, branch, default_ref)
if content is None:
content = "(binary file, not shown)"
values = {
**nav_values(repo, branch, default_ref, page="files"),
"path": html.escape(path),
"content": html.escape(content),
}
template = load_template("git-file.html")
write(
os.path.join(output_base, "file", f"{path}.html"),
render(template, values)
)
def build():
print("Building git...")
clean()
repos = discover_repos()
if not repos:
print(" no repositories found")
for repo in repos:
ref = repo.branch()
if ref is None:
print(f" skip: {repo.name} (empty repository)")
continue
for branch in repo.branches():
build_repo_summary(repo, branch, ref)
build_repo_log(repo, branch, ref)
build_repo_tree(repo, branch, ref)
# Collect every reachable commit (across all branches) so
# that links from any log page resolve to a real page.
all_commits = {}
for branch in repo.branches():
for commit in repo.log(ref=branch):
all_commits[commit.hash] = commit
build_repo_commits(repo, all_commits.values(), ref)
print(f" repo: {repo.name}")
build_index(repos)
print(" index")
print(f"Built {len(repos)} repositories.")
# ------------------------------------------------------------
# CLI
# ------------------------------------------------------------
def main():
parser = argparse.ArgumentParser(
description="Milán Major's tiny static git browser"
)
subparsers = parser.add_subparsers(dest="command")
subparsers.add_parser("build", help="build the git pages")
subparsers.add_parser("clean", help="remove generated files")
args = parser.parse_args()
if args.command == "clean":
clean()
return
if args.command == "build":
build()
return
parser.print_help()
if __name__ == "__main__":
main()