// Package apt generates APT repository indices (Packages, Release, InRelease, // Release.gpg) entirely from in-memory package data. Indices are never written // to disk by this package; the caller caches and serves them. package apt import ( "bytes" "compress/gzip" "crypto/md5" "crypto/sha1" "crypto/sha256" "encoding/hex" "fmt" "sort" "strings" "time" "github.com/ulikunitz/xz" ) // Signer is implemented by anything able to clearsign and detached-sign the // Release file (e.g. *gpg.Key). type Signer interface { ClearSign(data []byte) ([]byte, error) DetachedSign(data []byte) ([]byte, error) } // PackageRow is a single package's data needed to emit its index entry. type PackageRow struct { Component string Name string Version string Architecture string PoolPath string Size int64 MD5sum string SHA1 string SHA256 string DescriptionMD5 string RawControl string } // Suite describes a (repository, distribution) for which to generate indices. type Suite struct { Origin string Label string Suite string Codename string Description string Components []string Architectures []string Packages []PackageRow } // Indices holds all generated index artifacts for a suite, keyed by their path // relative to the suite directory. type Indices struct { Packages map[string][]byte PackagesGz map[string][]byte PackagesXz map[string][]byte Release []byte InRelease []byte ReleaseGpg []byte } // Generate builds all indices for the suite and signs the Release file using // signer. If signer is nil, InRelease/ReleaseGpg are left empty. func Generate(s *Suite, signer Signer) (*Indices, error) { components := dedupSorted(s.Components) arches := dedupSorted(s.Architectures) idx := &Indices{ Packages: map[string][]byte{}, PackagesGz: map[string][]byte{}, PackagesXz: map[string][]byte{}, } // Group packages by (component, arch), including arch="all" in every arch. type key struct{ comp, arch string } groups := map[key][]PackageRow{} for _, p := range s.Packages { for _, arch := range arches { if p.Architecture == arch || p.Architecture == "all" { k := key{p.Component, arch} groups[k] = append(groups[k], p) } } } for _, comp := range components { for _, arch := range arches { rows := groups[key{comp, arch}] sort.SliceStable(rows, func(i, j int) bool { if rows[i].Name != rows[j].Name { return rows[i].Name < rows[j].Name } return rows[i].Version < rows[j].Version }) pkgBytes := generatePackagesIndex(rows) relPath := fmt.Sprintf("%s/binary-%s/Packages", comp, arch) idx.Packages[relPath] = pkgBytes idx.PackagesGz[relPath+".gz"] = gzipBytes(pkgBytes) xzBytes, err := xzBytes(pkgBytes) if err != nil { return nil, fmt.Errorf("xz compress %s: %w", relPath, err) } idx.PackagesXz[relPath+".xz"] = xzBytes } } release, err := generateRelease(s, components, arches, idx) if err != nil { return nil, err } idx.Release = release if signer != nil { clearsigned, err := signer.ClearSign(release) if err != nil { return nil, fmt.Errorf("clearsign: %w", err) } idx.InRelease = clearsigned det, err := signer.DetachedSign(release) if err != nil { return nil, fmt.Errorf("detach sign: %w", err) } idx.ReleaseGpg = det } return idx, nil } // generatePackagesIndex emits the Packages file body for one (component, arch). // The result is always non-nil (an empty byte slice if there are no rows). func generatePackagesIndex(rows []PackageRow) []byte { var buf bytes.Buffer for _, r := range rows { stanza := strings.TrimRight(r.RawControl, "\n") buf.WriteString(stanza) buf.WriteByte('\n') writeField(&buf, "Filename", r.PoolPath) writeFieldInt(&buf, "Size", r.Size) writeField(&buf, "MD5sum", r.MD5sum) writeField(&buf, "SHA1", r.SHA1) writeField(&buf, "SHA256", r.SHA256) if r.DescriptionMD5 != "" { writeField(&buf, "Description-md5", r.DescriptionMD5) } buf.WriteByte('\n') } out := buf.Bytes() if out == nil { out = []byte{} } return out } // fileEntry is one index file's path and bytes, used for Release checksums. type fileEntry struct { path string data []byte } func (e fileEntry) md5() string { sum := md5.Sum(e.data); return hex.EncodeToString(sum[:]) } func (e fileEntry) sha1() string { sum := sha1.Sum(e.data); return hex.EncodeToString(sum[:]) } func (e fileEntry) sha256() string { sum := sha256.Sum256(e.data) return hex.EncodeToString(sum[:]) } func generateRelease(s *Suite, components, arches []string, idx *Indices) ([]byte, error) { var entries []fileEntry for path, data := range idx.Packages { entries = append(entries, fileEntry{path: path, data: data}) } for path, data := range idx.PackagesGz { entries = append(entries, fileEntry{path: path, data: data}) } for path, data := range idx.PackagesXz { entries = append(entries, fileEntry{path: path, data: data}) } sort.Slice(entries, func(i, j int) bool { return entries[i].path < entries[j].path }) var buf bytes.Buffer if s.Origin != "" { writeField(&buf, "Origin", s.Origin) } if s.Label != "" { writeField(&buf, "Label", s.Label) } writeField(&buf, "Suite", s.Suite) codename := s.Codename if codename == "" { codename = s.Suite } writeField(&buf, "Codename", codename) writeField(&buf, "Date", time.Now().UTC().Format("Mon, 02 Jan 2006 15:04:05 MST")) if s.Description != "" { writeField(&buf, "Description", s.Description) } if len(arches) > 0 { writeField(&buf, "Architectures", strings.Join(arches, " ")) } if len(components) > 0 { writeField(&buf, "Components", strings.Join(components, " ")) } writeChecksumBlock(&buf, "MD5Sum", entries, func(e fileEntry) string { return e.md5() }) writeChecksumBlock(&buf, "SHA1", entries, func(e fileEntry) string { return e.sha1() }) writeChecksumBlock(&buf, "SHA256", entries, func(e fileEntry) string { return e.sha256() }) return buf.Bytes(), nil } func writeChecksumBlock(buf *bytes.Buffer, name string, entries []fileEntry, hashFn func(fileEntry) string) { buf.WriteString(name + ":\n") for _, e := range entries { fmt.Fprintf(buf, " %s %16d %s\n", hashFn(e), len(e.data), e.path) } } func writeField(buf *bytes.Buffer, key, val string) { if val == "" { return } fmt.Fprintf(buf, "%s: %s\n", key, val) } func writeFieldInt(buf *bytes.Buffer, key string, val int64) { fmt.Fprintf(buf, "%s: %d\n", key, val) } func gzipBytes(data []byte) []byte { var buf bytes.Buffer gz := gzip.NewWriter(&buf) _, _ = gz.Write(data) _ = gz.Close() return buf.Bytes() } func xzBytes(data []byte) ([]byte, error) { var buf bytes.Buffer xw, err := xz.NewWriter(&buf) if err != nil { return nil, err } if _, err := xw.Write(data); err != nil { _ = xw.Close() return nil, err } if err := xw.Close(); err != nil { return nil, err } return buf.Bytes(), nil } func dedupSorted(in []string) []string { seen := map[string]bool{} var out []string for _, v := range in { if v == "" || seen[v] { continue } seen[v] = true out = append(out, v) } sort.Strings(out) return out }