publicsuffix: embed table data

Use //go:embed to embed the public suffix tables,
rather than generating .go files containing the data.

Creating an empty git repo and generating commits for the
last 20 updates to the public suffix list, the total size
of the repository directory as measured by "du -sh" decreases
from 2.2M to 668K when using embedding.

For golang/go#15518.

Change-Id: Id71759765831a7699e7a182937095b3820bb643b
Reviewed-on: https://go-review.googlesource.com/c/net/+/450935
Run-TryBot: Damien Neil <[email protected]>
TryBot-Result: Gopher Robot <[email protected]>
Reviewed-by: Nigel Tao <[email protected]>
Reviewed-by: Nigel Tao (INACTIVE; USE @golang.org INSTEAD) <[email protected]>

Author: neild

publicsuffix/data/children

@@ -22,6 +22,7 @@ package main
 import (
 	"bufio"
 	"bytes"
+	"encoding/binary"
 	"flag"
 	"fmt"
 	"go/format"
@@ -55,6 +56,7 @@ const (
 )
 
 var (
+	combinedText  string
 	maxChildren   int
 	maxTextOffset int
 	maxTextLength int
@@ -115,11 +117,10 @@ var (
 	shaRE  = regexp.MustCompile(`"sha":"([^"]+)"`)
 	dateRE = regexp.MustCompile(`"committer":{[^{]+"date":"([^"]+)"`)
 
-	comments = flag.Bool("comments", false, "generate table.go comments, for debugging")
-	subset   = flag.Bool("subset", false, "generate only a subset of the full table, for debugging")
-	url      = flag.String("url", defaultURL, "URL of the publicsuffix.org list. If empty, stdin is read instead")
-	v        = flag.Bool("v", false, "verbose output (to stderr)")
-	version  = flag.String("version", "", "the effective_tld_names.dat version")
+	subset  = flag.Bool("subset", false, "generate only a subset of the full table, for debugging")
+	url     = flag.String("url", defaultURL, "URL of the publicsuffix.org list. If empty, stdin is read instead")
+	v       = flag.Bool("v", false, "verbose output (to stderr)")
+	version = flag.String("version", "", "the effective_tld_names.dat version")
 )
 
 func main() {
@@ -254,7 +255,33 @@ func main1() error {
 	}
 	sort.Strings(labelsList)
 
-	if err := generate(printReal, &root, "table.go"); err != nil {
+	combinedText = combineText(labelsList)
+	if combinedText == "" {
+		return fmt.Errorf("internal error: combineText returned no text")
+	}
+	for _, label := range labelsList {
+		offset, length := strings.Index(combinedText, label), len(label)
+		if offset < 0 {
+			return fmt.Errorf("internal error: could not find %q in text %q", label, combinedText)
+		}
+		maxTextOffset, maxTextLength = max(maxTextOffset, offset), max(maxTextLength, length)
+		if offset >= 1<<nodesBitsTextOffset {
+			return fmt.Errorf("text offset %d is too large, or nodeBitsTextOffset is too small", offset)
+		}
+		if length >= 1<<nodesBitsTextLength {
+			return fmt.Errorf("text length %d is too large, or nodeBitsTextLength is too small", length)
+		}
+		labelEncoding[label] = uint64(offset)<<nodesBitsTextLength | uint64(length)
+	}
+
+	if err := root.walk(assignIndexes); err != nil {
+		return err
+	}
+
+	if err := generate(printMetadata, &root, "table.go"); err != nil {
+		return err
+	}
+	if err := generateBinaryData(&root, combinedText); err != nil {
 		return err
 	}
 	if err := generate(printTest, &root, "table_test.go"); err != nil {
@@ -307,18 +334,63 @@ func printTest(w io.Writer, n *node) error {
 		fmt.Fprintf(w, "%q,\n", rule)
 	}
 	fmt.Fprintf(w, "}\n\nvar nodeLabels = [...]string{\n")
-	if err := n.walk(w, printNodeLabel); err != nil {
+	if err := n.walk(func(n *node) error {
+		return printNodeLabel(w, n)
+	}); err != nil {
 		return err
 	}
 	fmt.Fprintf(w, "}\n")
 	return nil
 }
 
-func printReal(w io.Writer, n *node) error {
+func generateBinaryData(root *node, combinedText string) error {
+	if err := os.WriteFile("data/text", []byte(combinedText), 0666); err != nil {
+		return err
+	}
+
+	var nodes []byte
+	if err := root.walk(func(n *node) error {
+		for _, c := range n.children {
+			nodes = appendNodeEncoding(nodes, c)
+		}
+		return nil
+	}); err != nil {
+		return err
+	}
+	if err := os.WriteFile("data/nodes", nodes, 0666); err != nil {
+		return err
+	}
+
+	var children []byte
+	for _, c := range childrenEncoding {
+		children = binary.BigEndian.AppendUint32(children, c)
+	}
+	if err := os.WriteFile("data/children", children, 0666); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func appendNodeEncoding(b []byte, n *node) []byte {
+	encoding := labelEncoding[n.label]
+	if n.icann {
+		encoding |= 1 << (nodesBitsTextLength + nodesBitsTextOffset)
+	}
+	encoding |= uint64(n.childrenIndex) << (nodesBitsTextLength + nodesBitsTextOffset + nodesBitsICANN)
+	for i := nodesBits - 8; i >= 0; i -= 8 {
+		b = append(b, byte((encoding>>i)&0xff))
+	}
+	return b
+}
+
+func printMetadata(w io.Writer, n *node) error {
 	const header = `// generated by go run gen.go; DO NOT EDIT
 
 package publicsuffix
 
+import _ "embed"
+
 const version = %q
 
 const (
@@ -343,74 +415,36 @@ const (
 // numTLD is the number of top level domains.
 const numTLD = %d
 
+// text is the combined text of all labels.
+//
+//go:embed data/text
+var text string
+
 `
 	fmt.Fprintf(w, header, *version,
 		nodesBits,
 		nodesBitsChildren, nodesBitsICANN, nodesBitsTextOffset, nodesBitsTextLength,
 		childrenBitsWildcard, childrenBitsNodeType, childrenBitsHi, childrenBitsLo,
 		nodeTypeNormal, nodeTypeException, nodeTypeParentOnly, len(n.children))
-
-	text := combineText(labelsList)
-	if text == "" {
-		return fmt.Errorf("internal error: makeText returned no text")
-	}
-	for _, label := range labelsList {
-		offset, length := strings.Index(text, label), len(label)
-		if offset < 0 {
-			return fmt.Errorf("internal error: could not find %q in text %q", label, text)
-		}
-		maxTextOffset, maxTextLength = max(maxTextOffset, offset), max(maxTextLength, length)
-		if offset >= 1<<nodesBitsTextOffset {
-			return fmt.Errorf("text offset %d is too large, or nodeBitsTextOffset is too small", offset)
-		}
-		if length >= 1<<nodesBitsTextLength {
-			return fmt.Errorf("text length %d is too large, or nodeBitsTextLength is too small", length)
-		}
-		labelEncoding[label] = uint64(offset)<<nodesBitsTextLength | uint64(length)
-	}
-	fmt.Fprintf(w, "// Text is the combined text of all labels.\nconst text = ")
-	for len(text) > 0 {
-		n, plus := len(text), ""
-		if n > 64 {
-			n, plus = 64, " +"
-		}
-		fmt.Fprintf(w, "%q%s\n", text[:n], plus)
-		text = text[n:]
-	}
-
-	if err := n.walk(w, assignIndexes); err != nil {
-		return err
-	}
-
 	fmt.Fprintf(w, `
-
 // nodes is the list of nodes. Each node is represented as a %v-bit integer,
 // which encodes the node's children, wildcard bit and node type (as an index
 // into the children array), ICANN bit and text.
 //
-// If the table was generated with the -comments flag, there is a //-comment
-// after each node's data. In it is the nodes-array indexes of the children,
-// formatted as (n0x1234-n0x1256), with * denoting the wildcard bit. The
-// nodeType is printed as + for normal, ! for exception, and o for parent-only
-// nodes that have children but don't match a domain label in their own right.
-// An I denotes an ICANN domain.
-//
 // The layout within the node, from MSB to LSB, is:
 //	[%2d bits] unused
 //	[%2d bits] children index
 //	[%2d bits] ICANN bit
 //	[%2d bits] text index
 //	[%2d bits] text length
-var nodes = [...]uint8{
+//
+//go:embed data/nodes
+var nodes uint40String
 `,
 		nodesBits,
 		nodesBits-nodesBitsChildren-nodesBitsICANN-nodesBitsTextOffset-nodesBitsTextLength,
 		nodesBitsChildren, nodesBitsICANN, nodesBitsTextOffset, nodesBitsTextLength)
-	if err := n.walk(w, printNode); err != nil {
-		return err
-	}
-	fmt.Fprintf(w, `}
-
+	fmt.Fprintf(w, `
 // children is the list of nodes' children, the parent's wildcard bit and the
 // parent's node type. If a node has no children then their children index
 // will be in the range [0, 6), depending on the wildcard bit and node type.
@@ -421,27 +455,13 @@ var nodes = [...]uint8{
 //	[%2d bits] node type
 //	[%2d bits] high nodes index (exclusive) of children
 //	[%2d bits] low nodes index (inclusive) of children
-var children=[...]uint32{
+//
+//go:embed data/children
+var children uint32String
 `,
 		32-childrenBitsWildcard-childrenBitsNodeType-childrenBitsHi-childrenBitsLo,
 		childrenBitsWildcard, childrenBitsNodeType, childrenBitsHi, childrenBitsLo)
-	for i, c := range childrenEncoding {
-		s := "---------------"
-		lo := c & (1<<childrenBitsLo - 1)
-		hi := (c >> childrenBitsLo) & (1<<childrenBitsHi - 1)
-		if lo != hi {
-			s = fmt.Sprintf("n0x%04x-n0x%04x", lo, hi)
-		}
-		nodeType := int(c>>(childrenBitsLo+childrenBitsHi)) & (1<<childrenBitsNodeType - 1)
-		wildcard := c>>(childrenBitsLo+childrenBitsHi+childrenBitsNodeType) != 0
-		if *comments {
-			fmt.Fprintf(w, "0x%08x, // c0x%04x (%s)%s %s\n",
-				c, i, s, wildcardStr(wildcard), nodeTypeStr(nodeType))
-		} else {
-			fmt.Fprintf(w, "0x%x,\n", c)
-		}
-	}
-	fmt.Fprintf(w, "}\n\n")
+
 	fmt.Fprintf(w, "// max children %d (capacity %d)\n", maxChildren, 1<<nodesBitsChildren-1)
 	fmt.Fprintf(w, "// max text offset %d (capacity %d)\n", maxTextOffset, 1<<nodesBitsTextOffset-1)
 	fmt.Fprintf(w, "// max text length %d (capacity %d)\n", maxTextLength, 1<<nodesBitsTextLength-1)
@@ -465,12 +485,12 @@ type node struct {
 	children []*node
 }
 
-func (n *node) walk(w io.Writer, f func(w1 io.Writer, n1 *node) error) error {
-	if err := f(w, n); err != nil {
+func (n *node) walk(f func(*node) error) error {
+	if err := f(n); err != nil {
 		return err
 	}
 	for _, c := range n.children {
-		if err := c.walk(w, f); err != nil {
+		if err := c.walk(f); err != nil {
 			return err
 		}
 	}
@@ -516,7 +536,7 @@ var childrenEncoding = []uint32{
 
 var firstCallToAssignIndexes = true
 
-func assignIndexes(w io.Writer, n *node) error {
+func assignIndexes(n *node) error {
 	if len(n.children) != 0 {
 		// Assign nodesIndex.
 		n.firstChild = nextNodesIndex
@@ -561,32 +581,6 @@ func assignIndexes(w io.Writer, n *node) error {
 	return nil
 }
 
-func printNode(w io.Writer, n *node) error {
-	for _, c := range n.children {
-		s := "---------------"
-		if len(c.children) != 0 {
-			s = fmt.Sprintf("n0x%04x-n0x%04x", c.firstChild, c.firstChild+len(c.children))
-		}
-		encoding := labelEncoding[c.label]
-		if c.icann {
-			encoding |= 1 << (nodesBitsTextLength + nodesBitsTextOffset)
-		}
-		encoding |= uint64(c.childrenIndex) << (nodesBitsTextLength + nodesBitsTextOffset + nodesBitsICANN)
-		for i := nodesBits - 8; i >= 0; i -= 8 {
-			fmt.Fprintf(w, "0x%02x, ", (encoding>>i)&0xff)
-		}
-		if *comments {
-			fmt.Fprintf(w, "// n0x%04x c0x%04x (%s)%s %s %s %s\n",
-				c.nodesIndex, c.childrenIndex, s, wildcardStr(c.wildcard),
-				nodeTypeStr(c.nodeType), icannStr(c.icann), c.label,
-			)
-		} else {
-			fmt.Fprintf(w, "\n")
-		}
-	}
-	return nil
-}
-
 func printNodeLabel(w io.Writer, n *node) error {
 	for _, c := range n.children {
 		fmt.Fprintf(w, "%q,\n", c.label)

publicsuffix/data/nodes

@@ -101,10 +101,10 @@ loop:
 			break
 		}
 
-		u := uint32(nodeValue(f) >> (nodesBitsTextOffset + nodesBitsTextLength))
+		u := uint32(nodes.get(f) >> (nodesBitsTextOffset + nodesBitsTextLength))
 		icannNode = u&(1<<nodesBitsICANN-1) != 0
 		u >>= nodesBitsICANN
-		u = children[u&(1<<nodesBitsChildren-1)]
+		u = children.get(u & (1<<nodesBitsChildren - 1))
 		lo = u & (1<<childrenBitsLo - 1)
 		u >>= childrenBitsLo
 		hi = u & (1<<childrenBitsHi - 1)
@@ -154,18 +154,9 @@ func find(label string, lo, hi uint32) uint32 {
 	return notFound
 }
 
-func nodeValue(i uint32) uint64 {
-	off := uint64(i * (nodesBits / 8))
-	return uint64(nodes[off])<<32 |
-		uint64(nodes[off+1])<<24 |
-		uint64(nodes[off+2])<<16 |
-		uint64(nodes[off+3])<<8 |
-		uint64(nodes[off+4])
-}
-
 // nodeLabel returns the label for the i'th node.
 func nodeLabel(i uint32) string {
-	x := nodeValue(i)
+	x := nodes.get(i)
 	length := x & (1<<nodesBitsTextLength - 1)
 	x >>= nodesBitsTextLength
 	offset := x & (1<<nodesBitsTextOffset - 1)
@@ -189,3 +180,24 @@ func EffectiveTLDPlusOne(domain string) (string, error) {
 	}
 	return domain[1+strings.LastIndex(domain[:i], "."):], nil
 }
+
+type uint32String string
+
+func (u uint32String) get(i uint32) uint32 {
+	off := i * 4
+	return (uint32(u[off])<<24 |
+		uint32(u[off+1])<<16 |
+		uint32(u[off+2])<<8 |
+		uint32(u[off+3]))
+}
+
+type uint40String string
+
+func (u uint40String) get(i uint32) uint64 {
+	off := uint64(i * (nodesBits / 8))
+	return uint64(u[off])<<32 |
+		uint64(u[off+1])<<24 |
+		uint64(u[off+2])<<16 |
+		uint64(u[off+3])<<8 |
+		uint64(u[off+4])
+}