Skip to content

Commit

Permalink
✨ Support gc trigger webhook (#323)
Browse files Browse the repository at this point in the history 
* ✨ Change webhook field name

* ⬆️ Upgrade deps

* ✨ Change resource type

* ✨ Support webhook in daemon task handler

* ✨ Support blob gc webhook

* ✨ Support gc webhook
  • Loading branch information
@tosone
tosone authored Mar 3, 2024
1 parent 01781a4 commit fb0ce2e
Show file tree
Hide file tree
Showing 56 changed files with 1,882 additions and 975 deletions.
2 changes: 1 addition & 1 deletion docker-compose.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ services:
- /var/run/docker.sock:/var/run/docker.sock
- ${PWD}/conf:/etc/sigma
healthcheck:
test: ["CMD", "sh", "-c", "nc -z localhost 3000"]
test: ["CMD", "curl", "--fail", "-s", "http://localhost:3000/healthz"]
interval: 10s
timeout: 5s
retries: 10
Expand Down
14 changes: 7 additions & 7 deletions go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,9 +9,9 @@ require (
github.com/Masterminds/sprig/v3 v3.2.3
github.com/alicebob/miniredis/v2 v2.31.1
github.com/aliyun/aliyun-oss-go-sdk v3.0.2+incompatible
github.com/anchore/syft v0.105.0
github.com/anchore/syft v0.105.1
github.com/aquasecurity/trivy v0.49.1
github.com/aws/aws-sdk-go v1.50.25
github.com/aws/aws-sdk-go v1.50.27
github.com/caarlos0/env/v9 v9.0.0
github.com/casbin/casbin/v2 v2.82.0
github.com/casbin/gorm-adapter/v3 v3.21.0
Expand Down Expand Up @@ -63,9 +63,9 @@ require (
github.com/tencentyun/cos-go-sdk-v5 v0.7.46
github.com/tidwall/gjson v1.17.1
github.com/wagslane/go-password-validator v0.3.0
github.com/xanzy/go-gitlab v0.97.0
github.com/xanzy/go-gitlab v0.98.0
go.uber.org/mock v0.4.0
golang.org/x/crypto v0.19.0
golang.org/x/crypto v0.20.0
golang.org/x/exp v0.0.0-20240222234643-814bf88cf225
golang.org/x/net v0.21.0
golang.org/x/oauth2 v0.17.0
Expand All @@ -75,7 +75,7 @@ require (
gorm.io/driver/sqlite v1.5.5
gorm.io/gen v0.3.25
gorm.io/gorm v1.25.7
gorm.io/plugin/dbresolver v1.5.0
gorm.io/plugin/dbresolver v1.5.1
gorm.io/plugin/soft_delete v1.2.1
k8s.io/api v0.29.2
k8s.io/apimachinery v0.29.2
Expand Down Expand Up @@ -103,7 +103,7 @@ require (
github.com/anchore/go-logger v0.0.0-20230725134548-c21dafa1ec5a // indirect
github.com/anchore/go-struct-converter v0.0.0-20230627203149-c72ef8859ca9 // indirect
github.com/anchore/packageurl-go v0.1.1-0.20240202171727-877e1747d426 // indirect
github.com/anchore/stereoscope v0.0.2-0.20240208195325-681f6715b0e3 // indirect
github.com/anchore/stereoscope v0.0.2-0.20240216182029-6171ee21e1d5 // indirect
github.com/andybalholm/brotli v1.0.6 // indirect
github.com/aquasecurity/trivy-db v0.0.0-20231005141211-4fc651f7ac8d // indirect
github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect
Expand Down Expand Up @@ -381,7 +381,7 @@ require (
modernc.org/libc v1.41.0 // indirect
modernc.org/mathutil v1.6.0 // indirect
modernc.org/memory v1.7.2 // indirect
modernc.org/sqlite v1.29.1 // indirect
modernc.org/sqlite v1.29.2 // indirect
sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect
sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect
sigs.k8s.io/yaml v1.4.0 // indirect
Expand Down
28 changes: 14 additions & 14 deletions go.sum
View file
Original file line number Diff line number Diff line change
Expand Up @@ -192,10 +192,10 @@ github.com/anchore/go-testutils v0.0.0-20200925183923-d5f45b0d3c04 h1:VzprUTpc0v
github.com/anchore/go-testutils v0.0.0-20200925183923-d5f45b0d3c04/go.mod h1:6dK64g27Qi1qGQZ67gFmBFvEHScy0/C8qhQhNe5B5pQ=
github.com/anchore/packageurl-go v0.1.1-0.20240202171727-877e1747d426 h1:agoiZchSf1Nnnos1azwIg5hk5Ao9TzZNBD9++AChGEg=
github.com/anchore/packageurl-go v0.1.1-0.20240202171727-877e1747d426/go.mod h1:Blo6OgJNiYF41ufcgHKkbCKF2MDOMlrqhXv/ij6ocR4=
github.com/anchore/stereoscope v0.0.2-0.20240208195325-681f6715b0e3 h1:gnf3+0bYP6hsk/sQHdnLpqmilVUr/y6kIxzGCP6kUWA=
github.com/anchore/stereoscope v0.0.2-0.20240208195325-681f6715b0e3/go.mod h1:o0TqYkefad6kIPtmbigFKss7P48z4bjd8Vp5Wklbf3Y=
github.com/anchore/syft v0.105.0 h1:CG6D1wF4gfwVpF0o085Ym1FaWSK7sMz3B62nOk+0wH8=
github.com/anchore/syft v0.105.0/go.mod h1:qa0A9aliWCp0xpVA4tsB/S+aM+7VB9Fvjy8aWlDhbGU=
github.com/anchore/stereoscope v0.0.2-0.20240216182029-6171ee21e1d5 h1:o//fhRcSpOYHC/xG/HiI6ddtSMiRgHlB96xJQXZawZM=
github.com/anchore/stereoscope v0.0.2-0.20240216182029-6171ee21e1d5/go.mod h1:o0TqYkefad6kIPtmbigFKss7P48z4bjd8Vp5Wklbf3Y=
github.com/anchore/syft v0.105.1 h1:l4rRZGLQoWUr6FmuZU1smlMiVKM2YYAsdFK64YK5sCQ=
github.com/anchore/syft v0.105.1/go.mod h1:zntgjdKXqx5RM/GpSoDctnsGwUksLXWSsv9W0Is1YhM=
github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8=
github.com/andybalholm/brotli v1.0.1/go.mod h1:loMXtMfwqflxFJPmdbJO0a3KNoPuLBgiu3qAvBg8x/Y=
github.com/andybalholm/brotli v1.0.6 h1:Yf9fFpf49Zrxb9NlQaluyE92/+X7UVHlhMNJN2sxfOI=
Expand All @@ -219,8 +219,8 @@ github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:l
github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 h1:DklsrG3dyBCFEj5IhUbnKptjxatkF07cF2ak3yi77so=
github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw=
github.com/aws/aws-sdk-go v1.15.11/go.mod h1:mFuSZ37Z9YOHbQEwBWztmVzqXrEkub65tZoCYDt7FT0=
github.com/aws/aws-sdk-go v1.50.25 h1:vhiHtLYybv1Nhx3Kv18BBC6L0aPJHaG9aeEsr92W99c=
github.com/aws/aws-sdk-go v1.50.25/go.mod h1:LF8svs817+Nz+DmiMQKTO3ubZ/6IaTpq3TjupRn3Eqk=
github.com/aws/aws-sdk-go v1.50.27 h1:96ifhrSuja+AzdP3W/T2337igqVQ2FcSIJYkk+0rCeA=
github.com/aws/aws-sdk-go v1.50.27/go.mod h1:LF8svs817+Nz+DmiMQKTO3ubZ/6IaTpq3TjupRn3Eqk=
github.com/aws/aws-sdk-go-v2 v1.24.1 h1:xAojnj+ktS95YZlDf0zxWBkbFtymPeDP+rvUQIH3uAU=
github.com/aws/aws-sdk-go-v2 v1.24.1/go.mod h1:LNh45Br1YAkEKaAqvmE1m8FUx6a5b/V0oAKV7of29b4=
github.com/aws/aws-sdk-go-v2/config v1.26.6 h1:Z/7w9bUqlRI0FFQpetVuFYEsjzE3h7fpU6HuGmfPL/o=
Expand Down Expand Up @@ -1695,8 +1695,8 @@ github.com/wagslane/go-password-validator v0.3.0 h1:vfxOPzGHkz5S146HDpavl0cw1DSV
github.com/wagslane/go-password-validator v0.3.0/go.mod h1:TI1XJ6T5fRdRnHqHt14pvy1tNVnrwe7m3/f1f2fDphQ=
github.com/willf/bitset v1.1.11-0.20200630133818-d5bec3311243/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4=
github.com/willf/bitset v1.1.11/go.mod h1:83CECat5yLh5zVOf4P1ErAgKA5UDvKtgyUABdr3+MjI=
github.com/xanzy/go-gitlab v0.97.0 h1:StMqJ1Kvt00X43pYIBBjj52dFlghwSeBhRDRfzaZ7xY=
github.com/xanzy/go-gitlab v0.97.0/go.mod h1:ETg8tcj4OhrB84UEgeE8dSuV/0h4BBL1uOV/qK0vlyI=
github.com/xanzy/go-gitlab v0.98.0 h1:psTMbnA0vSo512M8WUpM5YIFPxrdQ/11V0y/5SdzIIg=
github.com/xanzy/go-gitlab v0.98.0/go.mod h1:ETg8tcj4OhrB84UEgeE8dSuV/0h4BBL1uOV/qK0vlyI=
github.com/xanzy/ssh-agent v0.3.3 h1:+/15pJfg/RsTxqYcX6fHqOXZwwMP+2VyYWJeWM2qQFM=
github.com/xanzy/ssh-agent v0.3.3/go.mod h1:6dzNDKs0J9rVPHPhaGCukekBHKqfl+L3KghI1Bc68Uw=
github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI=
Expand Down Expand Up @@ -1826,8 +1826,8 @@ golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU
golang.org/x/crypto v0.9.0/go.mod h1:yrmDGqONDYtNj3tH8X9dzUun2m2lzPa9ngI6/RUPGR0=
golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4=
golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4=
golang.org/x/crypto v0.19.0 h1:ENy+Az/9Y1vSrlrvBSyna3PITt4tiZLf7sgCjZBX7Wo=
golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU=
golang.org/x/crypto v0.20.0 h1:jmAMJJZXr5KiCw05dfYK9QnqaqKLYXijU23lsEdcQqg=
golang.org/x/crypto v0.20.0/go.mod h1:Xwo95rrVNIoSMx9wa1JroENMToLWn3RNVrTBpLHgZPQ=
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
Expand Down Expand Up @@ -2463,8 +2463,8 @@ gorm.io/gorm v1.25.7 h1:VsD6acwRjz2zFxGO50gPO6AkNs7KKnvfzUjHQhZDz/A=
gorm.io/gorm v1.25.7/go.mod h1:hbnx/Oo0ChWMn1BIhpy1oYozzpM15i4YPuHDmfYtwg8=
gorm.io/hints v1.1.2 h1:b5j0kwk5p4+3BtDtYqqfY+ATSxjj+6ptPgVveuynn9o=
gorm.io/hints v1.1.2/go.mod h1:/ARdpUHAtyEMCh5NNi3tI7FsGh+Cj/MIUlvNxCNCFWg=
gorm.io/plugin/dbresolver v1.5.0 h1:XVHLxh775eP0CqVh3vcfJtYqja3uFl5Wr3cKlY8jgDY=
gorm.io/plugin/dbresolver v1.5.0/go.mod h1:l4Cn87EHLEYuqUncpEeTC2tTJQkjngPSD+lo8hIvcT0=
gorm.io/plugin/dbresolver v1.5.1 h1:s9Dj9f7r+1rE3nx/Ywzc85nXptUEaeOO0pt27xdopM8=
gorm.io/plugin/dbresolver v1.5.1/go.mod h1:l4Cn87EHLEYuqUncpEeTC2tTJQkjngPSD+lo8hIvcT0=
gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo=
gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw=
gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk=
Expand Down Expand Up @@ -2526,8 +2526,8 @@ modernc.org/mathutil v1.6.0 h1:fRe9+AmYlaej+64JsEEhoWuAYBkOtQiMEU7n/XgfYi4=
modernc.org/mathutil v1.6.0/go.mod h1:Ui5Q9q1TR2gFm0AQRqQUaBWFLAhQpCwNcuhBOSedWPo=
modernc.org/memory v1.7.2 h1:Klh90S215mmH8c9gO98QxQFsY+W451E8AnzjoE2ee1E=
modernc.org/memory v1.7.2/go.mod h1:NO4NVCQy0N7ln+T9ngWqOQfi7ley4vpwvARR+Hjw95E=
modernc.org/sqlite v1.29.1 h1:19GY2qvWB4VPw0HppFlZCPAbmxFU41r+qjKZQdQ1ryA=
modernc.org/sqlite v1.29.1/go.mod h1:hG41jCYxOAOoO6BRK66AdRlmOcDzXf7qnwlwjUIOqa0=
modernc.org/sqlite v1.29.2 h1:xgBSyA3gemwgP31PWFfFjtBorQNYpeypGdoSDjXhrgI=
modernc.org/sqlite v1.29.2/go.mod h1:hG41jCYxOAOoO6BRK66AdRlmOcDzXf7qnwlwjUIOqa0=
rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=
rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=
Expand Down
86 changes: 72 additions & 14 deletions pkg/daemon/gc/decorator.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,8 +25,12 @@ import (

"github.com/go-sigma/sigma/pkg/configs"
"github.com/go-sigma/sigma/pkg/dal/dao"
"github.com/go-sigma/sigma/pkg/modules/workq"
"github.com/go-sigma/sigma/pkg/modules/workq/definition"
"github.com/go-sigma/sigma/pkg/storage"
"github.com/go-sigma/sigma/pkg/types"
"github.com/go-sigma/sigma/pkg/types/enums"
"github.com/go-sigma/sigma/pkg/utils"
"github.com/go-sigma/sigma/pkg/utils/ptr"
)

Expand All @@ -42,19 +46,51 @@ type decoratorStatus struct {
Updates map[string]any
}

// decoratorWebhook used for webhook trigger
type decoratorWebhook struct {
NamespaceID *int64
Meta types.WebhookPayload
WebhookObj any
}

type inject struct {
daemonServiceFactory dao.DaemonServiceFactory

producerClient definition.WorkQueueProducer
}

// Runner ...
type Runner interface {
// Run ...
Run(runnerID int64) error
}

// decorator is a decorator for daemon gc task runners
func decorator(daemon enums.Daemon) func(context.Context, []byte) error {
func decorator(daemon enums.Daemon, injects ...inject) func(context.Context, []byte) error { // nolint: unparam
return func(ctx context.Context, payload []byte) error {
ctx = log.Logger.WithContext(ctx)
id := gjson.GetBytes(payload, "runner_id").Int()

producerClient := workq.ProducerClient
daemonServiceFactory := dao.NewDaemonServiceFactory()
if len(injects) > 0 {
ij := injects[0]
if ij.producerClient != nil {
producerClient = ij.producerClient
}
if ij.daemonServiceFactory != nil {
daemonServiceFactory = ij.daemonServiceFactory
}
}

var runnerChan = make(chan decoratorStatus, 3)
var gc = initGc(ctx, daemon, runnerChan)
var webhookChan = make(chan decoratorWebhook, 3)
var gc = initGc(ctx, daemon, runnerChan, webhookChan)
if gc == nil {
return fmt.Errorf("daemon %s not support", daemon.String())
}

daemonService := dao.NewDaemonServiceFactory().New()
daemonService := daemonServiceFactory.New()

var waitAllEvents = &sync.WaitGroup{}
waitAllEvents.Add(1)
Expand Down Expand Up @@ -101,6 +137,16 @@ func decorator(daemon enums.Daemon) func(context.Context, []byte) error {
}
}()

go func() {
defer waitAllEvents.Done()
for webhook := range webhookChan {
err := triggerWebhook(ctx, webhook, producerClient)
if err != nil {
log.Error().Err(err).Msg("Webhook event produce failed")
}
}
}()

err := gc.Run(id)
if err != nil {
return fmt.Errorf("gc runner(%s) failed: %v", daemon.String(), err)
Expand All @@ -112,13 +158,7 @@ func decorator(daemon enums.Daemon) func(context.Context, []byte) error {
}
}

// Runner ...
type Runner interface {
// Run ...
Run(runnerID int64) error
}

func initGc(ctx context.Context, daemon enums.Daemon, runnerChan chan decoratorStatus) Runner {
func initGc(ctx context.Context, daemon enums.Daemon, runnerChan chan decoratorStatus, webhookChan chan decoratorWebhook) Runner {
switch daemon {
case enums.DaemonGcRepository:
return &gcRepository{
Expand All @@ -139,7 +179,8 @@ func initGc(ctx context.Context, daemon enums.Daemon, runnerChan chan decoratorS
collectRecordChan: make(chan repositoryTaskCollectRecord, pagination),
collectRecordChanOnce: &sync.Once{},

runnerChan: runnerChan,
runnerChan: runnerChan,
webhookChan: webhookChan,

waitAllDone: &sync.WaitGroup{},
}
Expand All @@ -163,7 +204,8 @@ func initGc(ctx context.Context, daemon enums.Daemon, runnerChan chan decoratorS
collectRecordChan: make(chan artifactTaskCollectRecord, pagination),
collectRecordChanOnce: &sync.Once{},

runnerChan: runnerChan,
runnerChan: runnerChan,
webhookChan: webhookChan,

waitAllDone: &sync.WaitGroup{},
}
Expand All @@ -190,7 +232,8 @@ func initGc(ctx context.Context, daemon enums.Daemon, runnerChan chan decoratorS
collectRecordChan: make(chan tagTaskCollectRecord, pagination),
collectRecordChanOnce: &sync.Once{},

runnerChan: runnerChan,
runnerChan: runnerChan,
webhookChan: webhookChan,

waitAllDone: &sync.WaitGroup{},
}
Expand All @@ -208,11 +251,26 @@ func initGc(ctx context.Context, daemon enums.Daemon, runnerChan chan decoratorS
collectRecordChan: make(chan blobTaskCollectRecord, pagination),
collectRecordChanOnce: &sync.Once{},

runnerChan: runnerChan,
runnerChan: runnerChan,
webhookChan: webhookChan,

waitAllDone: &sync.WaitGroup{},
}
default:
return nil
}
}

func triggerWebhook(ctx context.Context, webhook decoratorWebhook, producerClient definition.WorkQueueProducer) error {
err := producerClient.Produce(ctx, enums.DaemonWebhook.String(), types.DaemonWebhookPayload{
NamespaceID: webhook.NamespaceID,
Action: webhook.Meta.Action,
Type: enums.WebhookTypeSend,
ResourceType: webhook.Meta.ResourceType,
Payload: utils.MustMarshal(webhook.WebhookObj),
}, definition.ProducerOption{})
if err != nil {
return fmt.Errorf("Webhook event produce failed: %v", err)
}
return nil
}
Loading

0 comments on commit fb0ce2e

Please sign in to comment.