Move root-level index code to index package

api.go

@@ -33,7 +33,6 @@ const (
 	sliceHeaderBytes  uint64 = 24
 	stringHeaderBytes uint64 = 16
 	pointerSize       uint64 = 8
-	interfaceBytes    uint64 = 16
 )
 
 // FileMatch contains all the matches within a file.
@@ -136,10 +135,10 @@ func (m *FileMatch) sizeBytes() (sz uint64) {
 	return
 }
 
-// addScore increments the score of the FileMatch by the computed score. If
+// AddScore increments the score of the FileMatch by the computed score. If
 // debugScore is true, it also adds a debug string to the FileMatch. If raw is
 // -1, it is ignored. Otherwise, it is added to the debug string.
-func (m *FileMatch) addScore(what string, computed float64, raw float64, debugScore bool) {
+func (m *FileMatch) AddScore(what string, computed float64, raw float64, debugScore bool) {
 	if computed != 0 && debugScore {
 		var b strings.Builder
 		fmt.Fprintf(&b, "%s", what)
@@ -695,6 +694,10 @@ func (r *Repository) UnmarshalJSON(data []byte) error {
 	return nil
 }
 
+func (r *Repository) GetPriority() float64 {
+	return r.priority
+}
+
 // monthsSince1970 returns the number of months since 1970. It returns values in
 // the range [0, maxUInt16]. The upper bound is reached in the year 7431, the
 // lower bound for all dates before 1970.
@@ -1014,6 +1017,17 @@ type SearchOptions struct {
 	SpanContext map[string]string
 }
 
+func (o *SearchOptions) SetDefaults() {
+	if o.ShardMaxMatchCount == 0 {
+		// We cap the total number of matches, so overly broad
+		// searches don't crash the machine.
+		o.ShardMaxMatchCount = 100000
+	}
+	if o.TotalMaxMatchCount == 0 {
+		o.TotalMaxMatchCount = 10 * o.ShardMaxMatchCount
+	}
+}
+
 // String returns a succinct representation of the options. This is meant for
 // human consumption in logs and traces.
 //

api_test.go

@@ -150,12 +150,6 @@ func TestMatchSize(t *testing.T) {
 	}, {
 		v:    ChunkMatch{},
 		size: 120,
-	}, {
-		v:    candidateMatch{},
-		size: 80,
-	}, {
-		v:    candidateChunk{},
-		size: 40,
 	}}
 	for _, c := range cases {
 		got := reflect.TypeOf(c.v).Size()

cmd/flags.go

@@ -20,23 +20,22 @@ import (
 	"os"
 	"path/filepath"
 
-	"github.com/sourcegraph/zoekt"
-	"github.com/sourcegraph/zoekt/build"
+	"github.com/sourcegraph/zoekt/index"
 )
 
 var (
 	version = flag.Bool("version", false, "Print version number")
-	opts    = &build.Options{}
+	opts    = &index.Options{}
 )
 
 func init() {
 	opts.Flags(flag.CommandLine)
 }
 
-func OptionsFromFlags() *build.Options {
+func OptionsFromFlags() *index.Options {
 	if *version {
 		name := filepath.Base(os.Args[0])
-		fmt.Printf("%s version %q\n", name, zoekt.Version)
+		fmt.Printf("%s version %q\n", name, index.Version)
 		os.Exit(0)
 	}
 

cmd/zoekt-index/main.go

@@ -23,9 +23,8 @@ import (
 	"runtime/pprof"
 	"strings"
 
-	"github.com/sourcegraph/zoekt"
-	"github.com/sourcegraph/zoekt/build"
 	"github.com/sourcegraph/zoekt/cmd"
+	"github.com/sourcegraph/zoekt/index"
 	"go.uber.org/automaxprocs/maxprocs"
 )
 
@@ -103,14 +102,14 @@ func main() {
 	}
 }
 
-func indexArg(arg string, opts build.Options, ignore map[string]struct{}) error {
+func indexArg(arg string, opts index.Options, ignore map[string]struct{}) error {
 	dir, err := filepath.Abs(filepath.Clean(arg))
 	if err != nil {
 		return err
 	}
 
 	opts.RepositoryDescription.Name = filepath.Base(dir)
-	builder, err := build.NewBuilder(opts)
+	builder, err := index.NewBuilder(opts)
 	if err != nil {
 		return err
 	}
@@ -135,7 +134,7 @@ func indexArg(arg string, opts build.Options, ignore map[string]struct{}) error
 	for f := range comm {
 		displayName := strings.TrimPrefix(f.name, dir+"/")
 		if f.size > int64(opts.SizeMax) && !opts.IgnoreSizeMax(displayName) {
-			if err := builder.Add(zoekt.Document{
+			if err := builder.Add(index.Document{
 				Name:       displayName,
 				SkipReason: fmt.Sprintf("document size %d larger than limit %d", f.size, opts.SizeMax),
 			}); err != nil {

cmd/zoekt-indexserver/main.go

@@ -33,7 +33,7 @@ import (
 	"strings"
 	"time"
 
-	"github.com/sourcegraph/zoekt"
+	"github.com/sourcegraph/zoekt/index"
 	"github.com/sourcegraph/zoekt/internal/gitindex"
 )
 
@@ -206,13 +206,13 @@ func deleteIfOrphan(repoDir string, fn string) error {
 	}
 	defer f.Close()
 
-	ifile, err := zoekt.NewIndexFile(f)
+	ifile, err := index.NewIndexFile(f)
 	if err != nil {
 		return nil
 	}
 	defer ifile.Close()
 
-	repos, _, err := zoekt.ReadMetadata(ifile)
+	repos, _, err := index.ReadMetadata(ifile)
 	if err != nil {
 		return nil
 	}

cmd/zoekt-merge-index/main.go

@@ -8,21 +8,21 @@ import (
 	"path/filepath"
 	"strings"
 
-	"github.com/sourcegraph/zoekt"
+	"github.com/sourcegraph/zoekt/index"
 )
 
 // merge merges the input shards into a compound shard in dstDir. It returns the
 // full path to the compound shard. The input shards are removed on success.
 func merge(dstDir string, names []string) (string, error) {
-	var files []zoekt.IndexFile
+	var files []index.IndexFile
 	for _, fn := range names {
 		f, err := os.Open(fn)
 		if err != nil {
 			return "", nil
 		}
 		defer f.Close()
 
-		indexFile, err := zoekt.NewIndexFile(f)
+		indexFile, err := index.NewIndexFile(f)
 		if err != nil {
 			return "", err
 		}
@@ -31,14 +31,14 @@ func merge(dstDir string, names []string) (string, error) {
 		files = append(files, indexFile)
 	}
 
-	tmpName, dstName, err := zoekt.Merge(dstDir, files...)
+	tmpName, dstName, err := index.Merge(dstDir, files...)
 	if err != nil {
 		return "", err
 	}
 
 	// Delete input shards.
 	for _, name := range names {
-		paths, err := zoekt.IndexFilePaths(name)
+		paths, err := index.IndexFilePaths(name)
 		if err != nil {
 			return "", fmt.Errorf("zoekt-merge-index: %w", err)
 		}
@@ -83,13 +83,13 @@ func explode(dstDir string, inputShard string) error {
 	}
 	defer f.Close()
 
-	indexFile, err := zoekt.NewIndexFile(f)
+	indexFile, err := index.NewIndexFile(f)
 	if err != nil {
 		return err
 	}
 	defer indexFile.Close()
 
-	exploded, err := zoekt.Explode(dstDir, indexFile)
+	exploded, err := index.Explode(dstDir, indexFile)
 	defer func() {
 		// best effort removal of tmp files. If os.Remove fails, indexserver will delete
 		// the leftover tmp files during the next cleanup.
@@ -104,7 +104,7 @@ func explode(dstDir string, inputShard string) error {
 	// remove the input shard first to avoid duplicate indexes. In the worst case,
 	// the process is interrupted just after we delete the compound shard, in which
 	// case we have to reindex the lost repos.
-	paths, err := zoekt.IndexFilePaths(inputShard)
+	paths, err := index.IndexFilePaths(inputShard)
 	if err != nil {
 		return err
 	}

cmd/zoekt-repo-index/main.go

@@ -41,8 +41,8 @@ import (
 
 	"github.com/google/slothfs/manifest"
 	"github.com/sourcegraph/zoekt"
-	"github.com/sourcegraph/zoekt/build"
 	"github.com/sourcegraph/zoekt/ignore"
+	"github.com/sourcegraph/zoekt/index"
 	"github.com/sourcegraph/zoekt/internal/gitindex"
 	"go.uber.org/automaxprocs/maxprocs"
 
@@ -127,7 +127,7 @@ func main() {
 	revPrefix := flag.String("rev_prefix", "refs/remotes/origin/", "prefix for references")
 	baseURLStr := flag.String("base_url", "", "base url to interpret repository names")
 	repoCacheDir := flag.String("repo_cache", "", "root for repository cache")
-	indexDir := flag.String("index", build.DefaultDir, "index directory for *.zoekt files")
+	indexDir := flag.String("index", index.DefaultDir, "index directory for *.zoekt files")
 	manifestRepoURL := flag.String("manifest_repo_url", "", "set a URL for a git repository holding manifest XML file. Provide the BRANCH:XML-FILE as further command-line arguments")
 	manifestRevPrefix := flag.String("manifest_rev_prefix", "refs/remotes/origin/", "prefixes for branches in manifest repository")
 	repoName := flag.String("name", "", "set repository name")
@@ -150,7 +150,7 @@ func main() {
 		*repoName = filepath.Join(u.Host, u.Path)
 	}
 
-	opts := build.Options{
+	opts := index.Options{
 		Parallelism: *parallelism,
 		SizeMax:     *sizeMax,
 		ShardMax:    *shardLimit,
@@ -258,7 +258,7 @@ func main() {
 		return
 	}
 
-	builder, err := build.NewBuilder(opts)
+	builder, err := index.NewBuilder(opts)
 	if err != nil {
 		log.Fatal(err)
 	}
@@ -269,7 +269,7 @@ func main() {
 			log.Fatal(err)
 		}
 
-		doc := zoekt.Document{
+		doc := index.Document{
 			Name:              k.FullPath(),
 			Content:           data,
 			SubRepositoryPath: k.SubRepoPath,