Initial release: self-hostable APT repository server and CLI
urapt is a self-hostable APT repository server with a companion CLI for pushing and managing Debian .deb packages. Server (urapt-server): - REST API + APT endpoint, SQLite storage (pure-Go modernc driver, no CGO) - .deb files stored content-addressed on disk, reference-counted for dedup - Server-managed RSA-4096 OpenPGP signing key (ProtonMail/go-crypto) - APT indices (Release/InRelease/Packages[.gz/.xz]) generated on demand from the DB, cached in memory, signed with the server key - Full APT model: repositories -> distributions -> components -> architectures - Bearer-token auth for REST; HTTP Basic auth for private-repo APT reads - First registrant becomes admin; repo-scoped permissions (read/write/read-write/admin) plus owner and server-admin roles - Multipart package push with control-field extraction, list/show/delete, pool serving, blob ref-count cleanup - Audit log CLI (urapt): - register/login/logout/whoami, token management - repo/distro/component/arch CRUD, member management - push/pull/ls/show/rm for packages - apt-config helper that emits apt setup commands (key, sources.list, auth.conf for private repos) Packaging & docs: - Dockerfile (multi-stage distroless), docker-compose.yml, sample config - README quick start, architecture overview, config reference, security notes - PLAN.md design blueprint, CHANGELOG.md, GPL-3.0 LICENSE - GitHub Actions CI (test, lint, cross-build for linux/darwin amd64/arm64) - Makefile release target producing static binaries + tarballs + checksums Tests cover the data-access layer, auth/permission checks, APT index generation, .deb parsing, GPG signing, the REST API, and the typed API client. Verified end-to-end on a Raspberry Pi (arm64) pushing and installing a real package.
This commit is contained in:
@@ -0,0 +1,267 @@
|
||||
// Package apt generates APT repository indices (Packages, Release, InRelease,
|
||||
// Release.gpg) entirely from in-memory package data. Indices are never written
|
||||
// to disk by this package; the caller caches and serves them.
|
||||
package apt
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"compress/gzip"
|
||||
"crypto/md5"
|
||||
"crypto/sha1"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/ulikunitz/xz"
|
||||
)
|
||||
|
||||
// Signer is implemented by anything able to clearsign and detached-sign the
|
||||
// Release file (e.g. *gpg.Key).
|
||||
type Signer interface {
|
||||
ClearSign(data []byte) ([]byte, error)
|
||||
DetachedSign(data []byte) ([]byte, error)
|
||||
}
|
||||
|
||||
// PackageRow is a single package's data needed to emit its index entry.
|
||||
type PackageRow struct {
|
||||
Component string
|
||||
Name string
|
||||
Version string
|
||||
Architecture string
|
||||
PoolPath string
|
||||
Size int64
|
||||
MD5sum string
|
||||
SHA1 string
|
||||
SHA256 string
|
||||
DescriptionMD5 string
|
||||
RawControl string
|
||||
}
|
||||
|
||||
// Suite describes a (repository, distribution) for which to generate indices.
|
||||
type Suite struct {
|
||||
Origin string
|
||||
Label string
|
||||
Suite string
|
||||
Codename string
|
||||
Description string
|
||||
Components []string
|
||||
Architectures []string
|
||||
Packages []PackageRow
|
||||
}
|
||||
|
||||
// Indices holds all generated index artifacts for a suite, keyed by their path
|
||||
// relative to the suite directory.
|
||||
type Indices struct {
|
||||
Packages map[string][]byte
|
||||
PackagesGz map[string][]byte
|
||||
PackagesXz map[string][]byte
|
||||
Release []byte
|
||||
InRelease []byte
|
||||
ReleaseGpg []byte
|
||||
}
|
||||
|
||||
// Generate builds all indices for the suite and signs the Release file using
|
||||
// signer. If signer is nil, InRelease/ReleaseGpg are left empty.
|
||||
func Generate(s *Suite, signer Signer) (*Indices, error) {
|
||||
components := dedupSorted(s.Components)
|
||||
arches := dedupSorted(s.Architectures)
|
||||
|
||||
idx := &Indices{
|
||||
Packages: map[string][]byte{},
|
||||
PackagesGz: map[string][]byte{},
|
||||
PackagesXz: map[string][]byte{},
|
||||
}
|
||||
|
||||
// Group packages by (component, arch), including arch="all" in every arch.
|
||||
type key struct{ comp, arch string }
|
||||
groups := map[key][]PackageRow{}
|
||||
for _, p := range s.Packages {
|
||||
for _, arch := range arches {
|
||||
if p.Architecture == arch || p.Architecture == "all" {
|
||||
k := key{p.Component, arch}
|
||||
groups[k] = append(groups[k], p)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for _, comp := range components {
|
||||
for _, arch := range arches {
|
||||
rows := groups[key{comp, arch}]
|
||||
sort.SliceStable(rows, func(i, j int) bool {
|
||||
if rows[i].Name != rows[j].Name {
|
||||
return rows[i].Name < rows[j].Name
|
||||
}
|
||||
return rows[i].Version < rows[j].Version
|
||||
})
|
||||
pkgBytes := generatePackagesIndex(rows)
|
||||
relPath := fmt.Sprintf("%s/binary-%s/Packages", comp, arch)
|
||||
idx.Packages[relPath] = pkgBytes
|
||||
idx.PackagesGz[relPath+".gz"] = gzipBytes(pkgBytes)
|
||||
xzBytes, err := xzBytes(pkgBytes)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("xz compress %s: %w", relPath, err)
|
||||
}
|
||||
idx.PackagesXz[relPath+".xz"] = xzBytes
|
||||
}
|
||||
}
|
||||
|
||||
release, err := generateRelease(s, components, arches, idx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
idx.Release = release
|
||||
|
||||
if signer != nil {
|
||||
clear, err := signer.ClearSign(release)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("clearsign: %w", err)
|
||||
}
|
||||
idx.InRelease = clear
|
||||
det, err := signer.DetachedSign(release)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("detach sign: %w", err)
|
||||
}
|
||||
idx.ReleaseGpg = det
|
||||
}
|
||||
return idx, nil
|
||||
}
|
||||
|
||||
// generatePackagesIndex emits the Packages file body for one (component, arch).
|
||||
// The result is always non-nil (an empty byte slice if there are no rows).
|
||||
func generatePackagesIndex(rows []PackageRow) []byte {
|
||||
var buf bytes.Buffer
|
||||
for _, r := range rows {
|
||||
stanza := strings.TrimRight(r.RawControl, "\n")
|
||||
buf.WriteString(stanza)
|
||||
buf.WriteByte('\n')
|
||||
writeField(&buf, "Filename", r.PoolPath)
|
||||
writeFieldInt(&buf, "Size", r.Size)
|
||||
writeField(&buf, "MD5sum", r.MD5sum)
|
||||
writeField(&buf, "SHA1", r.SHA1)
|
||||
writeField(&buf, "SHA256", r.SHA256)
|
||||
if r.DescriptionMD5 != "" {
|
||||
writeField(&buf, "Description-md5", r.DescriptionMD5)
|
||||
}
|
||||
buf.WriteByte('\n')
|
||||
}
|
||||
out := buf.Bytes()
|
||||
if out == nil {
|
||||
out = []byte{}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// fileEntry is one index file's path and bytes, used for Release checksums.
|
||||
type fileEntry struct {
|
||||
path string
|
||||
data []byte
|
||||
}
|
||||
|
||||
func (e fileEntry) md5() string { sum := md5.Sum(e.data); return hex.EncodeToString(sum[:]) }
|
||||
func (e fileEntry) sha1() string { sum := sha1.Sum(e.data); return hex.EncodeToString(sum[:]) }
|
||||
func (e fileEntry) sha256() string {
|
||||
sum := sha256.Sum256(e.data)
|
||||
return hex.EncodeToString(sum[:])
|
||||
}
|
||||
|
||||
func generateRelease(s *Suite, components, arches []string, idx *Indices) ([]byte, error) {
|
||||
var entries []fileEntry
|
||||
for path, data := range idx.Packages {
|
||||
entries = append(entries, fileEntry{path: path, data: data})
|
||||
}
|
||||
for path, data := range idx.PackagesGz {
|
||||
entries = append(entries, fileEntry{path: path, data: data})
|
||||
}
|
||||
for path, data := range idx.PackagesXz {
|
||||
entries = append(entries, fileEntry{path: path, data: data})
|
||||
}
|
||||
sort.Slice(entries, func(i, j int) bool { return entries[i].path < entries[j].path })
|
||||
|
||||
var buf bytes.Buffer
|
||||
if s.Origin != "" {
|
||||
writeField(&buf, "Origin", s.Origin)
|
||||
}
|
||||
if s.Label != "" {
|
||||
writeField(&buf, "Label", s.Label)
|
||||
}
|
||||
writeField(&buf, "Suite", s.Suite)
|
||||
codename := s.Codename
|
||||
if codename == "" {
|
||||
codename = s.Suite
|
||||
}
|
||||
writeField(&buf, "Codename", codename)
|
||||
writeField(&buf, "Date", time.Now().UTC().Format("Mon, 02 Jan 2006 15:04:05 MST"))
|
||||
if s.Description != "" {
|
||||
writeField(&buf, "Description", s.Description)
|
||||
}
|
||||
if len(arches) > 0 {
|
||||
writeField(&buf, "Architectures", strings.Join(arches, " "))
|
||||
}
|
||||
if len(components) > 0 {
|
||||
writeField(&buf, "Components", strings.Join(components, " "))
|
||||
}
|
||||
|
||||
writeChecksumBlock(&buf, "MD5Sum", entries, func(e fileEntry) string { return e.md5() })
|
||||
writeChecksumBlock(&buf, "SHA1", entries, func(e fileEntry) string { return e.sha1() })
|
||||
writeChecksumBlock(&buf, "SHA256", entries, func(e fileEntry) string { return e.sha256() })
|
||||
return buf.Bytes(), nil
|
||||
}
|
||||
|
||||
func writeChecksumBlock(buf *bytes.Buffer, name string, entries []fileEntry, hashFn func(fileEntry) string) {
|
||||
buf.WriteString(name + ":\n")
|
||||
for _, e := range entries {
|
||||
fmt.Fprintf(buf, " %s %16d %s\n", hashFn(e), len(e.data), e.path)
|
||||
}
|
||||
}
|
||||
|
||||
func writeField(buf *bytes.Buffer, key, val string) {
|
||||
if val == "" {
|
||||
return
|
||||
}
|
||||
fmt.Fprintf(buf, "%s: %s\n", key, val)
|
||||
}
|
||||
|
||||
func writeFieldInt(buf *bytes.Buffer, key string, val int64) {
|
||||
fmt.Fprintf(buf, "%s: %d\n", key, val)
|
||||
}
|
||||
|
||||
func gzipBytes(data []byte) []byte {
|
||||
var buf bytes.Buffer
|
||||
gz := gzip.NewWriter(&buf)
|
||||
_, _ = gz.Write(data)
|
||||
_ = gz.Close()
|
||||
return buf.Bytes()
|
||||
}
|
||||
|
||||
func xzBytes(data []byte) ([]byte, error) {
|
||||
var buf bytes.Buffer
|
||||
xw, err := xz.NewWriter(&buf)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if _, err := xw.Write(data); err != nil {
|
||||
_ = xw.Close()
|
||||
return nil, err
|
||||
}
|
||||
if err := xw.Close(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return buf.Bytes(), nil
|
||||
}
|
||||
|
||||
func dedupSorted(in []string) []string {
|
||||
seen := map[string]bool{}
|
||||
var out []string
|
||||
for _, v := range in {
|
||||
if v == "" || seen[v] {
|
||||
continue
|
||||
}
|
||||
seen[v] = true
|
||||
out = append(out, v)
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
Reference in New Issue
Block a user