Add the Grafana dashboard (#16)

Improve docker-compose Improve CI Improve Dockerfile Add the docstring Update the buf version
arttet · Sep 28, 2021 · 70e44ce · 70e44ce
1 parent 1aed921
commit 70e44ce
Show file tree

Hide file tree

Showing 23 changed files with 2,058 additions and 70 deletions.
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -1,11 +1,35 @@
 name: build
 
-on: [push, pull_request]
+on:
+  push:
+  pull_request:
+  schedule:
+    # https://crontab.guru/
+    # At 12:00 AM, only on Monday
+    - cron: 0 0 * * 1
 
 jobs:
   build:
+    strategy:
+      matrix:
+        go-version: [1.17.x]
+        os: [ubuntu-latest, macos-latest, windows-latest]
+    runs-on: ${{ matrix.os }}
+    steps:
+      - name: Install Go
+        uses: actions/setup-go@v2
+        with:
+          go-version: ${{ matrix.go-version }}
+      - name: Checkout code
+        uses: actions/checkout@v2
+      - name: Build
+        shell: bash
+        run: |
+          make all
+
+  docker:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v2
-    - name: Build a Docker image
-      run: docker build . --file Dockerfile --tag image:$(date +%s)
+      - uses: actions/checkout@v2
+      - name: Build a Docker image
+        run: docker build . --file Dockerfile --tag image:$(date +%s)
diff --git a/.github/workflows/image.yml b/.github/workflows/image.yml
@@ -2,22 +2,21 @@ name: image
 
 on:
   push:
-    branches:
-      - main
+    tags:
+      - '*'
 
 jobs:
   deploy:
     name: Build
     runs-on: ubuntu-latest
     env:
       REGISTRY: ghcr.io
-      IMAGE_NAME: docker.pkg.github.com/${{ github.repository }}
       COMMIT_HASH: ${{ github.sha }}
     environment: default
 
     steps:
       - name: Set env version
-        run: echo "IMAGE_TAG=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV
+        run: echo "GIT_TAG=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV
 
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v1
@@ -44,4 +43,7 @@ jobs:
       - uses: docker/build-push-action@v2
         with:
           push: true
-          tags: ghcr.io/${{ github.repository_owner }}/reddit-feed-api/server:${{ env.IMAGE_TAG }}
+          build-args: |
+            VERSION=${{ env.GIT_TAG }}
+            COMMIT_HASH=${{ env.COMMIT_HASH }}
+          tags: ghcr.io/${{ github.repository_owner }}/reddit-feed-api/server:${{ env.GIT_TAG }}
diff --git a/Dockerfile b/Dockerfile
@@ -3,6 +3,8 @@ ARG GITHUB_PATH=github.com/arttet/reddit-feed-api
 
 FROM golang:1.17-alpine AS builder
 
+ARG VERSION COMMIT_HASH
+
 RUN apk add --update make git protoc protobuf protobuf-dev curl
 COPY . /home/${GITHUB_PATH}
 WORKDIR /home/${GITHUB_PATH}
@@ -17,7 +19,7 @@ WORKDIR /root/
 
 COPY --from=builder /home/${GITHUB_PATH}/bin/reddit-feed-api .
 COPY --from=builder /home/${GITHUB_PATH}/config.yml .
-COPY --from=builder /home/${GITHUB_PATH}/migrations/ ./migrations/
+COPY --from=builder /home/${GITHUB_PATH}/migrations/ ./migrations/001_init_db.sql
 
 RUN chown root:root reddit-feed-api
 CMD ["./reddit-feed-api"]
diff --git a/Makefile b/Makefile
@@ -12,22 +12,18 @@ SERVICE_EXE=./bin/$(SERVICE_NAME)$(shell go env GOEXE)
 
 ###############################################################################
 
-# https://github.com/bufbuild/buf/releases
-BUF_VERSION=v1.0.0-rc1
-
-OS_NAME=$(shell uname -s)
-OS_ARCH=$(shell uname -m)
-GOBIN?=$(GOPATH)/bin
-
-ifeq ("NT", "$(findstring NT,$(OS_NAME))")
-OS_NAME=Windows
-endif
+.PHONY: all
+all: deps build
 
-###############################################################################
+.PHONY: deps
+deps: .deps
 
 .PHONY: build
 build: generate .build
 
+.PHONY: generate
+generate: .generate
+
 .PHONY: run
 run:
 	go run \
@@ -74,9 +70,6 @@ grpcui:
 
 ###############################################################################
 
-.PHONY: deps
-deps: .deps
-
 .deps:
 	go env -w GO111MODULE=on
 
@@ -96,17 +89,25 @@ deps: .deps
 
 ###############################################################################
 
-.PHONY: generate
-generate: .generate
+# https://github.com/bufbuild/buf/releases
+BUF_VERSION=v1.0.0-rc2
 
-.generate:
-	@command -v buf 2>&1 > /dev/null || (echo "Install buf" && \
-		mkdir -p "$(GOBIN)" && \
-		curl -k -sSL0 https://github.com/bufbuild/buf/releases/download/$(BUF_VERSION)/buf-$(OS_NAME)-$(OS_ARCH)$(shell go env GOEXE) -o "$(GOBIN)/buf" && \
-		chmod +x "$(GOBIN)/buf")
-	@PATH="$(GOBIN):$(PATH)" buf generate
+OS_NAME=$(shell uname -s)
+OS_ARCH=$(shell uname -m)
+GO_BIN=$(shell go env GOPATH)/bin
+BUF_EXE=$(GO_BIN)/buf$(shell go env GOEXE)
 
-	mv pkg/$(SERVICE_NAME)/$(SERVICE_PATH)/pkg/$(SERVICE_NAME)/* pkg/$(SERVICE_NAME)
+ifeq ("NT", "$(findstring NT,$(OS_NAME))")
+OS_NAME=Windows
+endif
+
+.generate:
+	@ command -v buf 2>&1 > /dev/null || (echo "Install buf" && \
+		mkdir -p "$(GO_BIN)" && \
+		curl -k -sSL0 https://github.com/bufbuild/buf/releases/download/$(BUF_VERSION)/buf-$(OS_NAME)-$(OS_ARCH)$(shell go env GOEXE) -o "$(BUF_EXE)" && \
+		chmod +x "$(BUF_EXE)")
+	@ $(BUF_EXE) generate
+	cp -R pkg/$(SERVICE_NAME)/$(SERVICE_PATH)/pkg/$(SERVICE_NAME)/* pkg/$(SERVICE_NAME)/
 	rm -rf pkg/$(SERVICE_NAME)/github.com/
 	cd pkg/$(SERVICE_NAME) && ls go.mod || (go mod init $(SERVICE_PATH)/pkg/$(SERVICE_NAME) && go mod tidy)
 

diff --git a/README.md b/README.md
@@ -20,7 +20,7 @@ docker-compose build
 
 ### For local development
 ```sh
-./reddit-feed-api --cfg config-dev.yml --migration up
+make run
 ```
 
 ### Docker
@@ -96,6 +96,13 @@ The Swagger UI is an open source project to visually render documentation for an
 
 - http://localhost:8080
 
+### Grafana:
+
+- http://localhost:3000
+    - login `admin`
+    - password `ADMIN`
+    - [Dashboard: Reddit Feed API](http://localhost:3000/d/QXuFMwN7z/reddit-feed-api?orgId=1&refresh=5s)
+
 ### Jaeger UI
 
 Monitor and troubleshoot transactions in complex distributed systems.
@@ -108,12 +115,6 @@ Prometheus is an open-source systems monitoring and alerting toolkit
 
 - http://localhost:9090
 
-### Grafana:
-
-- http://localhost:3000
-    - login `admin`
-    - password `MYPASSWORT`
-
 ### Kafka UI
 
 UI for Apache Kafka is a simple tool that makes your data flows observable, helps find and troubleshoot issues faster and deliver optimal performance. Its lightweight dashboard makes it easy to track key metrics of your Kafka clusters - Brokers, Topics, Partitions, Production, and Consumption.

diff --git a/doc/README.md b/doc/README.md
@@ -0,0 +1,33 @@
+## Requirements
+
+Your task will be to build a simplified version of the Reddit feed API that powers
+http://old.reddit.com. **Please use Go as the programming language**. This will be a REST API
+that allows users to do the following:
+
+**Create new posts.** Posts should be validated for correctness. They should have the following
+fields:
+
+1. Title
+2. Author: This should be a random 8 character string prefixed with t2_. The 8 character
+string should only contain lowercase letters and numbers. For example, my user ID is
+t2_11qnzrqv.
+3. Link: This should be a valid URL. It's ok if your validation is not perfect.
+4. Subreddit: The subreddit associated with this post.
+5. Content: In the case of a text-only post. **A post cannot have both a link and content
+populated.**
+6. Score: The total score associated with the upvotes and downvotes of a post.
+7. Promoted: A boolean field indicating whether or not the post is an ad or not.
+8. NSFW: Not safe for work. A boolean that indicates whether or not the post is safe for
+work
+
+**Generate a feed of posts.** This feed should have the following characteristics:
+
+1. It should be ranked by score, and the post with the highest score should show up first.
+2. It should be paginated, and each page should have at most 27 posts. Your API should
+support fetching a specific page in the feed.
+3. If a page has 3 posts or greater, the second post should always be a promoted post if a
+promoted post is available, regardless of the score.
+4. If a page has greater than 16 posts, the 16th post should always be a promoted post if a
+promoted post is available, regardless of the score.
+5. As an exception to rules 3 and 4, **a promoted post should never be shown adjacent
+to an NSFW post**. You can ignore rules 3 and 4 in this case.
diff --git a/docker-compose.yaml b/docker-compose.yaml
@@ -59,7 +59,7 @@ services:
     networks:
       - reddit-network
     volumes:
-      # - pgdata:/var/lib/postgresql/data
+      - pg_data:/var/lib/postgresql/data
       - ./external/scripts/init-database.sh:/docker-entrypoint-initdb.d/init-database.sh
 
   jaeger:
@@ -89,7 +89,7 @@ services:
     networks:
       - reddit-network
     volumes:
-      - "./external/prometheus.yml:/etc/prometheus/prometheus.yml"
+      - ./external/prometheus.yml:/etc/prometheus/prometheus.yml
 
   zookeeper:
     image: confluentinc/cp-zookeeper
@@ -151,7 +151,7 @@ services:
       - 8080:8080
     restart: always
     volumes:
-      - ./external/swagger:/docs
+      - swagger_data:/docs
     environment:
       SWAGGER_JSON: /docs/api.swagger.json
     depends_on:
@@ -167,12 +167,13 @@ services:
     links:
       - prometheus
     environment:
-      - GF_SECURITY_ADMIN_PASSWORD=MYPASSWORT
+      - GF_SECURITY_ADMIN_PASSWORD=ADMIN
       - GF_USERS_ALLOW_SIGN_UP=false
     networks:
       - reddit-network
     volumes:
-      - "./data/grafana:/var/lib/grafana"
+      - grafana_data:/var/lib/grafana
+      - grafana_init:/etc/grafana/provisioning
 
   # Graylog: https://hub.docker.com/r/graylog/graylog/
   graylog:
@@ -212,7 +213,7 @@ services:
     networks:
       - reddit-network
     volumes:
-      - mongodata:/data/db
+      - mongo_data:/data/db
 
   # Elasticsearch: https://www.elastic.co/guide/en/elasticsearch/reference/6.x/docker.html
   elasticsearch:
@@ -233,14 +234,32 @@ services:
     networks:
       - reddit-network
     volumes:
-      - esdata:/usr/share/elasticsearch/data
+      - es_data:/usr/share/elasticsearch/data
 
 volumes:
-  pgdata:
+  pg_data:
     driver: local
-  esdata:
+  swagger_data:
     driver: local
-  mongodata:
+    driver_opts:
+      type: none
+      o: bind
+      device: ${PWD}/external/swagger
+  grafana_data:
+    driver: local
+    driver_opts:
+      type: none
+      o: bind
+      device: ${PWD}/data/grafana
+  grafana_init:
+    driver: local
+    driver_opts:
+      type: none
+      o: bind
+      device: ${PWD}/external/grafana
+  mongo_data:
+    driver: local
+  es_data:
     driver: local
 
 networks: