1
0
Fork 0
mirror of https://codeberg.org/forgejo/forgejo.git synced 2024-12-04 02:48:34 +00:00

Add PULL_LIMIT and PUSH_LIMIT to cron.update_mirror task ()

This commit is contained in:
zeripath 2021-11-23 03:09:35 +00:00 committed by GitHub
parent 9450410ff7
commit 188fd2dd1a
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
6 changed files with 70 additions and 17 deletions

View file

@ -1719,6 +1719,12 @@ PATH =
;RUN_AT_START = false
;; Notice if not success
;NO_SUCCESS_NOTICE = true
;; Limit the number of mirrors added to the queue to this number
;; (negative values mean no limit, 0 will result in no result in no mirrors being queued effectively disabling pull mirror updating.)
;PULL_LIMIT=50
;; Limit the number of mirrors added to the queue to this number
;; (negative values mean no limit, 0 will result in no mirrors being queued effectively disabling push mirror updating)
;PUSH_LIMIT=50
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;

View file

@ -828,6 +828,8 @@ NB: You must have `DISABLE_ROUTER_LOG` set to `false` for this option to take ef
- `SCHEDULE`: **@every 10m**: Cron syntax for scheduling update mirrors, e.g. `@every 3h`.
- `NO_SUCCESS_NOTICE`: **true**: The cron task for update mirrors success report is not very useful - as it just means that the mirrors have been queued. Therefore this is turned off by default.
- `PULL_LIMIT`: **50**: Limit the number of mirrors added to the queue to this number (negative values mean no limit, 0 will result in no mirrors being queued effectively disabling pull mirror updating).
- `PUSH_LIMIT`: **50**: Limit the number of mirrors added to the queue to this number (negative values mean no limit, 0 will result in no mirrors being queued effectively disabling push mirror updating).
#### Cron - Repository Health Check (`cron.repo_health_check`)

View file

@ -119,6 +119,7 @@ func MirrorsIterate(f func(idx int, bean interface{}) error) error {
return db.GetEngine(db.DefaultContext).
Where("next_update_unix<=?", time.Now().Unix()).
And("next_update_unix!=0").
OrderBy("updated_unix ASC").
Iterate(new(Mirror), f)
}

View file

@ -107,5 +107,6 @@ func PushMirrorsIterate(f func(idx int, bean interface{}) error) error {
return db.GetEngine(db.DefaultContext).
Where("last_update + (`interval` / ?) <= ?", time.Second, time.Now().Unix()).
And("`interval` != 0").
OrderBy("last_update ASC").
Iterate(new(PushMirror), f)
}

View file

@ -18,13 +18,24 @@ import (
)
func registerUpdateMirrorTask() {
RegisterTaskFatal("update_mirrors", &BaseConfig{
Enabled: true,
RunAtStart: false,
Schedule: "@every 10m",
NoSuccessNotice: true,
}, func(ctx context.Context, _ *models.User, _ Config) error {
return mirror_service.Update(ctx)
type UpdateMirrorTaskConfig struct {
BaseConfig
PullLimit int
PushLimit int
}
RegisterTaskFatal("update_mirrors", &UpdateMirrorTaskConfig{
BaseConfig: BaseConfig{
Enabled: true,
RunAtStart: false,
Schedule: "@every 10m",
NoSuccessNotice: true,
},
PullLimit: 50,
PushLimit: 50,
}, func(ctx context.Context, _ *models.User, cfg Config) error {
umtc := cfg.(*UpdateMirrorTaskConfig)
return mirror_service.Update(ctx, umtc.PullLimit, umtc.PushLimit)
})
}

View file

@ -45,15 +45,19 @@ func doMirrorSync(ctx context.Context, req *SyncRequest) {
}
}
var errLimit = fmt.Errorf("reached limit")
// Update checks and updates mirror repositories.
func Update(ctx context.Context) error {
func Update(ctx context.Context, pullLimit, pushLimit int) error {
if !setting.Mirror.Enabled {
log.Warn("Mirror feature disabled, but cron job enabled: skip update")
return nil
}
log.Trace("Doing: Update")
handler := func(idx int, bean interface{}) error {
requested := 0
handler := func(idx int, bean interface{}, limit int) error {
var item SyncRequest
if m, ok := bean.(*models.Mirror); ok {
if m.Repo == nil {
@ -78,21 +82,49 @@ func Update(ctx context.Context) error {
return nil
}
// Check we've not been cancelled
select {
case <-ctx.Done():
return fmt.Errorf("Aborted")
return fmt.Errorf("aborted")
default:
return mirrorQueue.Push(&item)
}
// Check if this request is already in the queue
has, err := mirrorQueue.Has(&item)
if err != nil {
return err
}
if has {
return nil
}
// Push to the Queue
if err := mirrorQueue.Push(&item); err != nil {
return err
}
requested++
if limit > 0 && requested > limit {
return errLimit
}
return nil
}
if err := models.MirrorsIterate(handler); err != nil {
log.Error("MirrorsIterate: %v", err)
return err
if pullLimit != 0 {
if err := models.MirrorsIterate(func(idx int, bean interface{}) error {
return handler(idx, bean, pullLimit)
}); err != nil && err != errLimit {
log.Error("MirrorsIterate: %v", err)
return err
}
}
if err := models.PushMirrorsIterate(handler); err != nil {
log.Error("PushMirrorsIterate: %v", err)
return err
if pushLimit != 0 {
if err := models.PushMirrorsIterate(func(idx int, bean interface{}) error {
return handler(idx, bean, pushLimit)
}); err != nil && err != errLimit {
log.Error("PushMirrorsIterate: %v", err)
return err
}
}
log.Trace("Finished: Update")
return nil