Document all the commands + packages (#904)

This PR adds doc comments for all packages/ commands.

cmd/zoekt-archive-index/main.go

@@ -1,6 +1,6 @@
-// Command zoekt-archive-index indexes an archive.
+// Command zoekt-archive-index indexes a git archive.
 //
-// Example via github.com:
+// Examples using github.com:
 //
 //	zoekt-archive-index -incremental -commit b57cb1605fd11ba2ecfa7f68992b4b9cc791934d -name github.com/gorilla/mux -strip_components 1 https://codeload.github.com/gorilla/mux/legacy.tar.gz/b57cb1605fd11ba2ecfa7f68992b4b9cc791934d
 //

cmd/zoekt-dynamic-indexserver/main.go

@@ -12,10 +12,10 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-// This program manages a zoekt dynamic indexing deployment:
+// Command zoekt-dynamic-indexserver starts a server to manage dynamic indexing. In contrast to
+// zoekt-indexserver, it's designed for a "push-based" indexing model. The server
 // * listens to indexing commands
 // * reindexes specified repositories
-
 package main
 
 import (

cmd/zoekt-git-clone/main.go

@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-// This binary fetches all repos of a user or organization and clones
+// Command zoekt-git-clone fetches all repos of a user or organization and clones
 // them.  It is strongly recommended to get a personal API token from
 // https://github.com/settings/tokens, save the token in a file, and
 // point the --token option to it.

cmd/zoekt-git-index/main.go

@@ -12,6 +12,8 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+// Command zoekt-git-index indexes a single git repository. It works directly with git
+// repositories and supports git-specific features like branches and submodules.
 package main
 
 import (

cmd/zoekt-index/main.go

@@ -12,6 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+// Command zoekt-index indexes a directory of files.
 package main
 
 import (

cmd/zoekt-indexserver/main.go

@@ -12,11 +12,8 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-// This program manages a zoekt indexing deployment:
-// * recycling logs
-// * periodically fetching new data.
-// * periodically reindexing all git repos.
-
+// Command zoekt-indexserver starts a service that periodically reindexes repositories. It follows
+// a "pull-based" design, where it reaches out to code hosts to fetch new data.
 package main
 
 import (

cmd/zoekt-merge-index/main.go

@@ -1,3 +1,16 @@
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Command zoekt-merge-index merges a set of index shards into a compound shard.
 package main
 
 import (

cmd/zoekt-mirror-bitbucket-server/main.go

@@ -10,9 +10,8 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-// This binary fetches all repos of a project, and of a specific type, in case
-// these are specified, and clones them. By default it fetches and clones all
-// existing repos.
+// Command zoekt-mirror-bitbucket-server fetches all repos of a bitbucket project,
+// optionally of a specific type, and clones them.
 package main
 
 import (

cmd/zoekt-mirror-gerrit/main.go

@@ -12,8 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-// This binary fetches all repos of a Gerrit host.
-
+// Command zoekt-mirror-gerrit fetches all repos of a Gerrit host.
 package main
 
 import (

cmd/zoekt-mirror-gitea/main.go

@@ -12,10 +12,10 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-// This binary fetches all repos of a user or organization and clones
-// them.  It is strongly recommended to get a personal API token from
-// https://gitea.com/user/settings/applications, save the token in a
-// file, and point the --token option to it.
+// Command zoekt-mirror-gerrit fetches all repos of a gitea user or organization
+// and clones them. It is strongly recommended to get a personal API token from
+// https://gitea.com/user/settings/applications, save the token in a file, and point
+// the --token option to it.
 package main
 
 import (

cmd/zoekt-mirror-github/main.go

@@ -12,10 +12,10 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-// This binary fetches all repos of a user or organization and clones
-// them.  It is strongly recommended to get a personal API token from
-// https://github.com/settings/tokens, save the token in a file, and
-// point the --token option to it.
+// Command zoekt-mirror-github fetches all repos of a github user or organization
+// and clones them. It is strongly recommended to get a personal API token from
+// https://github.com/settings/tokens, save the token in a file, and point the
+// --token option to it.
 package main
 
 import (

cmd/zoekt-mirror-gitiles/main.go

@@ -12,8 +12,8 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-// This binary fetches all repos of a Gitiles host.  It does double
-// duty for other "simple" web hosts
+// Command zoekt-mirror-gitiles fetches all repos of a Gitiles host.
+// It does double duty for other "simple" web hosts.
 package main
 
 import (

cmd/zoekt-mirror-gitlab/main.go

@@ -10,13 +10,13 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-// This binary fetches all repos for a user from gitlab.
+// Command zoekt-mirror-gitlab fetches all repos for a user from gitlab.
 //
 // It is recommended to use a gitlab personal access token:
 // https://docs.gitlab.com/ce/user/profile/personal_access_tokens.html. This
 // token should be stored in a file and the --token option should be used.
 // In addition, the token should be present in the ~/.netrc of the user running
-// the mirror command. For example, the ~/.netrc may look like:
+// Command mirror. For example, the ~/.netrc may look like:
 //
 //	machine gitlab.com
 //	login oauth

cmd/zoekt-repo-index/main.go

@@ -12,9 +12,10 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-// zoekt-repo-index indexes a repo-based repository. The constituent git
+// Command zoekt-repo-index indexes repository that uses the Android 'repo'
+// tool (https://android.googlesource.com/tools/repo). The constituent git
 // repositories should already have been downloaded to the --repo_cache
-// directory, eg.
+// directory, for example:
 //
 // go install github.com/sourcegraph/zoekt/cmd/zoekt-repo-index &&
 //

cmd/zoekt-sourcegraph-indexserver/main.go

@@ -1,5 +1,18 @@
-// Command zoekt-sourcegraph-indexserver periodically reindexes enabled
-// repositories on sourcegraph
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Command zoekt-sourcegraph-indexserver periodically reindexes repositories
+// from a Sourcegraph instance. It uses a "pull-based" design, where it periodically
+// reaches out to the Sourcegraph instance for the list of repositories to reindex.
 package main
 
 import (

cmd/zoekt-test/main.go

@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-// zoekt-test compares the search engine results with raw substring search
+// Command zoekt-test compares the zoekt results with raw substring search.
 package main
 
 import (

cmd/zoekt-webserver/main.go

@@ -12,9 +12,8 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-// Command zoekt-webserver responds to search queries, using an index generated
-// by another program such as zoekt-indexserver.
-
+// Command zoekt-webserver starts a server that responds to search queries, using
+// an index generated by another program such as zoekt-indexserver.
 package main
 
 import (

cmd/zoekt/main.go

@@ -12,6 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+// The 'zoekt' command supports searching over an index directory or shard.
 package main
 
 import (

ignore/ignore.go

@@ -1,3 +1,15 @@
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 // Package ignore provides helpers to support ignore-files similar to .gitignore
 package ignore
 

index/builder.go

@@ -12,8 +12,8 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-// package build implements a more convenient interface for building
-// zoekt indices.
+// Package index contains logic for building Zoekt indexes. NOTE: this package is not considered
+// part of the public API, and it is not recommended to rely on it in external code.
 package index
 
 import (

query/doc.go

@@ -0,0 +1,16 @@
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package query contains the API for creating Zoekt queries. Queries can be
+// constructed directly through query.Q objects, or by parsing a string using
+// query.Parse
+package query

web/doc.go

@@ -0,0 +1,15 @@
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package web contains the logic for spinning up a zoekt webserver. It's exposed separately
+// from zoekt-webserver to allow for customizing the endpoints and format templates.
+package web