Fix charts for SQLite

This fixes a regression introduces in 6bc0162; that changed so that stat
rows were no longer stored if there are no visits to store. That works
fine, except for hit_stats in SQLite since it will do a "select",
"delete", merges with new pageviews, and then an insert. But if there
are no new visits (only pageviews) it will never re-insert the rows.

So check the full array instead, and write a migration to correct the
wrong data.

.gitignore

@@ -21,3 +21,6 @@
 # Coverage reports
 /coverage
 /coverage.*
+
+# Dev log
+/.dev.log

CHANGELOG.md

@@ -4,6 +4,10 @@ This list is not comprehensive, and only lists new features and major changes,
 but not every minor bugfix. The goatcounter.com service generally runs the
 latest master.
 
+2022-11-15 v2.4.1
+-----------------
+- Fix regression that caused the charts for SQLite to be off.
+
 2022-11-08 v2.4.0
 -----------------
 - Add a more fully-featured API that can also retrieve the dashboard statistics.

cron/hit_stat.go

@@ -8,17 +8,19 @@ import (
 	"context"
 	"strconv"
 
+	"golang.org/x/exp/slices"
 	"zgo.at/errors"
 	"zgo.at/goatcounter/v2"
 	"zgo.at/zdb"
 	"zgo.at/zstd/zjson"
 )
 
+var empty = []int{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}
+
 func updateHitStats(ctx context.Context, hits []goatcounter.Hit) error {
 	return errors.Wrap(zdb.TX(ctx, func(ctx context.Context) error {
 		type gt struct {
 			count  []int
-			total  int
 			day    string
 			hour   string
 			pathID int64
@@ -51,7 +53,6 @@ func updateHitStats(ctx context.Context, hits []goatcounter.Hit) error {
 			hour, _ := strconv.ParseInt(h.CreatedAt.Format("15"), 10, 8)
 			if h.FirstVisit {
 				v.count[hour] += 1
-				v.total += 1
 			}
 			grouped[k] = v
 		}
@@ -79,9 +80,10 @@ func updateHitStats(ctx context.Context, hits []goatcounter.Hit) error {
 		// }
 
 		for _, v := range grouped {
-			if v.total > 0 {
-				ins.Values(siteID, v.day, v.pathID, zjson.MustMarshal(v.count))
+			if slices.Equal(v.count, empty) {
+				continue
 			}
+			ins.Values(siteID, v.day, v.pathID, zjson.MustMarshal(v.count))
 		}
 		return errors.Wrap(ins.Finish(), "updateHitStats hit_stats")
 	}), "cron.updateHitStats")

db/migrate/gomig/2022-11-15-1-correct-hit-stats.go

@@ -0,0 +1,162 @@
+// Copyright © Martin Tournoij – This file is part of GoatCounter and published
+// under the terms of a slightly modified EUPL v1.2 license, which can be found
+// in the LICENSE file or at https://license.goatcounter.com
+
+package gomig
+
+import (
+	"context"
+	"strconv"
+
+	"golang.org/x/exp/slices"
+	"zgo.at/errors"
+	"zgo.at/goatcounter/v2"
+	"zgo.at/zdb"
+	"zgo.at/zstd/zjson"
+)
+
+func CorrectHitStats(ctx context.Context) error {
+	// Only for SQLite
+	if zdb.SQLDialect(ctx) == zdb.DialectPostgreSQL {
+		return nil
+	}
+
+	err := zdb.TX(goatcounter.NewCache(goatcounter.NewConfig(ctx)), func(ctx context.Context) error {
+		err := zdb.Exec(ctx, `delete from hit_stats where day >= '2022-11-08'`)
+		if err != nil {
+			return err
+		}
+
+		var sites goatcounter.Sites
+		err = sites.UnscopedList(ctx)
+		if err != nil {
+			return err
+		}
+
+		for _, s := range sites {
+			var hits goatcounter.Hits
+			err = zdb.Select(ctx, &hits, `select * from hits where
+				created_at >= '2022-11-08 00:00:00' and first_visit=1 and bot in (0, 1)`)
+			if err != nil {
+				return err
+			}
+
+			err = updateHitStats(goatcounter.WithSite(ctx, &s), hits)
+			if err != nil {
+				return err
+			}
+		}
+		return nil
+	})
+
+	if err == nil {
+		err = zdb.Exec(ctx, `insert into version values ('2022-11-15-1-correct-hit-stats')`)
+	}
+	return err
+}
+
+// below is a copy of cron/hit_stat.go
+
+var empty = []int{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}
+
+func updateHitStats(ctx context.Context, hits []goatcounter.Hit) error {
+	return errors.Wrap(zdb.TX(ctx, func(ctx context.Context) error {
+		type gt struct {
+			count  []int
+			day    string
+			hour   string
+			pathID int64
+		}
+		grouped := map[string]gt{}
+		for _, h := range hits {
+			if h.Bot > 0 {
+				continue
+			}
+
+			day := h.CreatedAt.Format("2006-01-02")
+			dayHour := h.CreatedAt.Format("2006-01-02 15:00:00")
+			k := day + strconv.FormatInt(h.PathID, 10)
+			v := grouped[k]
+			if len(v.count) == 0 {
+				v.day = day
+				v.hour = dayHour
+				v.pathID = h.PathID
+				v.count = make([]int, 24)
+
+				if zdb.SQLDialect(ctx) == zdb.DialectSQLite {
+					var err error
+					v.count, err = existingHitStats(ctx, h.Site, day, v.pathID)
+					if err != nil {
+						return err
+					}
+				}
+			}
+
+			hour, _ := strconv.ParseInt(h.CreatedAt.Format("15"), 10, 8)
+			if h.FirstVisit {
+				v.count[hour] += 1
+			}
+			grouped[k] = v
+		}
+
+		siteID := goatcounter.MustGetSite(ctx).ID
+		ins := zdb.NewBulkInsert(ctx, "hit_stats", []string{"site_id", "day", "path_id", "stats"})
+		if zdb.SQLDialect(ctx) == zdb.DialectPostgreSQL {
+			ins.OnConflict(`on conflict on constraint "hit_stats#site_id#path_id#day" do update set
+				stats = (
+					with x as (
+						select
+							unnest(string_to_array(trim(hit_stats.stats, '[]'), ',')::int[]) as orig,
+							unnest(string_to_array(trim(excluded.stats,  '[]'), ',')::int[]) as new
+					)
+					select '[' || array_to_string(array_agg(orig + new), ',') || ']' from x
+				) `)
+		}
+		// } else {
+		// TODO: merge the arrays here and get rid of existingHitStats();
+		// it's kinda tricky with SQLite :-/
+		//
+		// ins.OnConflict(`on conflict(site_id, path_id, day) do update set
+		// 	stats = excluded.stats
+		// `)
+		// }
+
+		for _, v := range grouped {
+			if slices.Equal(v.count, empty) {
+				continue
+			}
+			ins.Values(siteID, v.day, v.pathID, zjson.MustMarshal(v.count))
+		}
+		return errors.Wrap(ins.Finish(), "updateHitStats hit_stats")
+	}), "cron.updateHitStats")
+}
+
+func existingHitStats(ctx context.Context, siteID int64, day string, pathID int64) ([]int, error) {
+	var ex []struct {
+		Stats []byte `db:"stats"`
+	}
+	err := zdb.Select(ctx, &ex, `/* existingHitStats */
+		select stats from hit_stats
+		where site_id=$1 and day=$2 and path_id=$3 limit 1`,
+		siteID, day, pathID)
+	if err != nil {
+		return nil, errors.Wrap(err, "existingHitStats")
+	}
+	if len(ex) == 0 {
+		return make([]int, 24), nil
+	}
+
+	err = zdb.Exec(ctx, `delete from hit_stats where
+		site_id=$1 and day=$2 and path_id=$3`,
+		siteID, day, pathID)
+	if err != nil {
+		return nil, errors.Wrap(err, "delete")
+	}
+
+	var ru []int
+	if ex[0].Stats != nil {
+		zjson.MustUnmarshal(ex[0].Stats, &ru)
+	}
+
+	return ru, nil
+}

db/migrate/gomig/gomig.go

@@ -7,5 +7,6 @@ package gomig
 import "context"
 
 var Migrations = map[string]func(context.Context) error{
-	"2021-12-08-1-set-chart-text": KeepAsText,
+	"2021-12-08-1-set-chart-text":    KeepAsText,
+	"2022-11-15-1-correct-hit-stats": CorrectHitStats,
 }