mirror of
https://codeberg.org/forgejo/forgejo.git
synced 2025-05-25 11:22:16 +00:00
Store webhook event in database (#29145)
Refactor the webhook logic, to have the type-dependent processing happen only in one place. --- 1. An event happens 2. It is pre-processed (depending on the webhook type) and its body is added to a task queue 3. When the task is processed, some more logic (depending on the webhook type as well) is applied to make an HTTP request This means that webhook-type dependant logic is needed in step 2 and 3. This is cumbersome and brittle to maintain. Updated webhook flow with this PR: 1. An event happens 2. It is stored as-is and added to a task queue 3. When the task is processed, the event is processed (depending on the webhook type) to make an HTTP request So the only webhook-type dependent logic happens in one place (step 3) which should be much more robust. - the raw event must be stored in the hooktask (until now, the pre-processed body was stored) - to ensure that previous hooktasks are correctly sent, a `payload_version` is added (version 1: the body has already been pre-process / version 2: the body is the raw event) So future webhook additions will only have to deal with creating an http.Request based on the raw event (no need to adjust the code in multiple places, like currently). Moreover since this processing happens when fetching from the task queue, it ensures that the queuing of new events (upon a `git push` for instance) does not get slowed down by a slow webhook. As a concrete example, the PR #19307 for custom webhooks, should be substantially smaller: - no need to change `services/webhook/deliver.go` - minimal change in `services/webhook/webhook.go` (add the new webhook to the map) - no need to change all the individual webhook files (since with this refactor the `*webhook_model.Webhook` is provided as argument) (cherry picked from commit 26653b196bd1d15c532af41f60351596dd4330bd) Conflicts: services/webhook/deliver_test.go trivial context conflict
This commit is contained in:
oliverpool
Earl Warren
parent
7ea8993a0e
commit
53f6f62ad4
28 changed files with 1686 additions and 1518 deletions
|
|
@ -32,36 +32,17 @@ import (
|
|||
"github.com/gobwas/glob"
|
||||
)
|
||||
|
||||
// Deliver deliver hook task
|
||||
func Deliver(ctx context.Context, t *webhook_model.HookTask) error {
|
||||
w, err := webhook_model.GetWebhookByID(ctx, t.HookID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
defer func() {
|
||||
err := recover()
|
||||
if err == nil {
|
||||
return
|
||||
}
|
||||
// There was a panic whilst delivering a hook...
|
||||
log.Error("PANIC whilst trying to deliver webhook task[%d] to webhook %s Panic: %v\nStacktrace: %s", t.ID, w.URL, err, log.Stack(2))
|
||||
}()
|
||||
|
||||
t.IsDelivered = true
|
||||
|
||||
var req *http.Request
|
||||
|
||||
func newDefaultRequest(ctx context.Context, w *webhook_model.Webhook, t *webhook_model.HookTask) (req *http.Request, body []byte, err error) {
|
||||
switch w.HTTPMethod {
|
||||
case "":
|
||||
log.Info("HTTP Method for webhook %s empty, setting to POST as default", w.URL)
|
||||
log.Info("HTTP Method for %s webhook %s [ID: %d] is not set, defaulting to POST", w.Type, w.URL, w.ID)
|
||||
fallthrough
|
||||
case http.MethodPost:
|
||||
switch w.ContentType {
|
||||
case webhook_model.ContentTypeJSON:
|
||||
req, err = http.NewRequest("POST", w.URL, strings.NewReader(t.PayloadContent))
|
||||
if err != nil {
|
||||
return err
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
|
|
@ -72,50 +53,58 @@ func Deliver(ctx context.Context, t *webhook_model.HookTask) error {
|
|||
|
||||
req, err = http.NewRequest("POST", w.URL, strings.NewReader(forms.Encode()))
|
||||
if err != nil {
|
||||
return err
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
|
||||
default:
|
||||
return nil, nil, fmt.Errorf("invalid content type: %v", w.ContentType)
|
||||
}
|
||||
case http.MethodGet:
|
||||
u, err := url.Parse(w.URL)
|
||||
if err != nil {
|
||||
return fmt.Errorf("unable to deliver webhook task[%d] as cannot parse webhook url %s: %w", t.ID, w.URL, err)
|
||||
return nil, nil, fmt.Errorf("invalid URL: %w", err)
|
||||
}
|
||||
vals := u.Query()
|
||||
vals["payload"] = []string{t.PayloadContent}
|
||||
u.RawQuery = vals.Encode()
|
||||
req, err = http.NewRequest("GET", u.String(), nil)
|
||||
if err != nil {
|
||||
return fmt.Errorf("unable to deliver webhook task[%d] as unable to create HTTP request for webhook url %s: %w", t.ID, w.URL, err)
|
||||
return nil, nil, err
|
||||
}
|
||||
case http.MethodPut:
|
||||
switch w.Type {
|
||||
case webhook_module.MATRIX:
|
||||
case webhook_module.MATRIX: // used when t.Version == 1
|
||||
txnID, err := getMatrixTxnID([]byte(t.PayloadContent))
|
||||
if err != nil {
|
||||
return err
|
||||
return nil, nil, err
|
||||
}
|
||||
url := fmt.Sprintf("%s/%s", w.URL, url.PathEscape(txnID))
|
||||
req, err = http.NewRequest("PUT", url, strings.NewReader(t.PayloadContent))
|
||||
if err != nil {
|
||||
return fmt.Errorf("unable to deliver webhook task[%d] as cannot create matrix request for webhook url %s: %w", t.ID, w.URL, err)
|
||||
return nil, nil, err
|
||||
}
|
||||
default:
|
||||
return fmt.Errorf("invalid http method for webhook task[%d] in webhook %s: %v", t.ID, w.URL, w.HTTPMethod)
|
||||
return nil, nil, fmt.Errorf("invalid http method: %v", w.HTTPMethod)
|
||||
}
|
||||
default:
|
||||
return fmt.Errorf("invalid http method for webhook task[%d] in webhook %s: %v", t.ID, w.URL, w.HTTPMethod)
|
||||
return nil, nil, fmt.Errorf("invalid http method: %v", w.HTTPMethod)
|
||||
}
|
||||
|
||||
body = []byte(t.PayloadContent)
|
||||
return req, body, addDefaultHeaders(req, []byte(w.Secret), t, body)
|
||||
}
|
||||
|
||||
func addDefaultHeaders(req *http.Request, secret []byte, t *webhook_model.HookTask, payloadContent []byte) error {
|
||||
var signatureSHA1 string
|
||||
var signatureSHA256 string
|
||||
if len(w.Secret) > 0 {
|
||||
sig1 := hmac.New(sha1.New, []byte(w.Secret))
|
||||
sig256 := hmac.New(sha256.New, []byte(w.Secret))
|
||||
_, err = io.MultiWriter(sig1, sig256).Write([]byte(t.PayloadContent))
|
||||
if len(secret) > 0 {
|
||||
sig1 := hmac.New(sha1.New, secret)
|
||||
sig256 := hmac.New(sha256.New, secret)
|
||||
_, err := io.MultiWriter(sig1, sig256).Write(payloadContent)
|
||||
if err != nil {
|
||||
log.Error("prepareWebhooks.sigWrite: %v", err)
|
||||
// this error should never happen, since the hashes are writing to []byte and always return a nil error.
|
||||
return fmt.Errorf("prepareWebhooks.sigWrite: %w", err)
|
||||
}
|
||||
signatureSHA1 = hex.EncodeToString(sig1.Sum(nil))
|
||||
signatureSHA256 = hex.EncodeToString(sig256.Sum(nil))
|
||||
|
|
@ -140,15 +129,36 @@ func Deliver(ctx context.Context, t *webhook_model.HookTask) error {
|
|||
req.Header["X-GitHub-Delivery"] = []string{t.UUID}
|
||||
req.Header["X-GitHub-Event"] = []string{event}
|
||||
req.Header["X-GitHub-Event-Type"] = []string{eventType}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Add Authorization Header
|
||||
authorization, err := w.HeaderAuthorization()
|
||||
// Deliver creates the [http.Request] (depending on the webhook type), sends it
|
||||
// and records the status and response.
|
||||
func Deliver(ctx context.Context, t *webhook_model.HookTask) error {
|
||||
w, err := webhook_model.GetWebhookByID(ctx, t.HookID)
|
||||
if err != nil {
|
||||
log.Error("Webhook could not get Authorization header [%d]: %v", w.ID, err)
|
||||
return err
|
||||
}
|
||||
if authorization != "" {
|
||||
req.Header["Authorization"] = []string{authorization}
|
||||
|
||||
defer func() {
|
||||
err := recover()
|
||||
if err == nil {
|
||||
return
|
||||
}
|
||||
// There was a panic whilst delivering a hook...
|
||||
log.Error("PANIC whilst trying to deliver webhook task[%d] to webhook %s Panic: %v\nStacktrace: %s", t.ID, w.URL, err, log.Stack(2))
|
||||
}()
|
||||
|
||||
t.IsDelivered = true
|
||||
|
||||
newRequest := webhookRequesters[w.Type]
|
||||
if t.PayloadVersion == 1 || newRequest == nil {
|
||||
newRequest = newDefaultRequest
|
||||
}
|
||||
|
||||
req, body, err := newRequest(ctx, w, t)
|
||||
if err != nil {
|
||||
return fmt.Errorf("cannot create http request for webhook %s[%d %s]: %w", w.Type, w.ID, w.URL, err)
|
||||
}
|
||||
|
||||
// Record delivery information.
|
||||
|
|
@ -156,11 +166,22 @@ func Deliver(ctx context.Context, t *webhook_model.HookTask) error {
|
|||
URL: req.URL.String(),
|
||||
HTTPMethod: req.Method,
|
||||
Headers: map[string]string{},
|
||||
Body: string(body),
|
||||
}
|
||||
for k, vals := range req.Header {
|
||||
t.RequestInfo.Headers[k] = strings.Join(vals, ",")
|
||||
}
|
||||
|
||||
// Add Authorization Header
|
||||
authorization, err := w.HeaderAuthorization()
|
||||
if err != nil {
|
||||
return fmt.Errorf("cannot get Authorization header for webhook %s[%d %s]: %w", w.Type, w.ID, w.URL, err)
|
||||
}
|
||||
if authorization != "" {
|
||||
req.Header.Set("Authorization", authorization)
|
||||
t.RequestInfo.Headers["Authorization"] = "******"
|
||||
}
|
||||
|
||||
t.ResponseInfo = &webhook_model.HookResponse{
|
||||
Headers: map[string]string{},
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue