feat(global): add —delay flag (#551)

* feat(global, download): add —delay flag

* feat(forward): support —delay flag

* feat(upload): support --delay flag

* doc(en/zh): add `—delay` in global-config doc

* refactor(iter): use HasNext() in guard of Next()

* Revert "refactor(iter): use HasNext() in guard of Next()"

This reverts commit 7a110fa069.

* Revert "feat(upload): support --delay flag"

This reverts commit 24ab0b50d9.

* Revert "feat(forward): support —delay flag"

This reverts commit 12c4f060de.

* Revert "feat(global, download): add —delay flag"

This reverts commit 735bbc74

* feat(global): support set delay duration between each task

---------

Co-authored-by: iyear <ljyngup@gmail.com>

app/dl/dl.go

@@ -75,7 +75,7 @@ func Run(ctx context.Context, c *telegram.Client, kvd kv.KV, opts Options) (rerr
 
 	manager := peers.Options{Storage: storage.NewPeers(kvd)}.Build(pool.Default(ctx))
 
-	it, err := newIter(pool, manager, dialogs, opts)
+	it, err := newIter(pool, manager, dialogs, opts, viper.GetDuration(consts.FlagDelay))
 	if err != nil {
 		return err
 	}

app/dl/iter.go

@@ -43,6 +43,7 @@ type iter struct {
 	include map[string]struct{}
 	exclude map[string]struct{}
 	opts    Options
+	delay   time.Duration
 
 	mu          *sync.Mutex
 	finished    map[int]struct{}
@@ -53,7 +54,9 @@ type iter struct {
 	err         error
 }
 
-func newIter(pool dcpool.Pool, manager *peers.Manager, dialog [][]*tmessage.Dialog, opts Options) (*iter, error) {
+func newIter(pool dcpool.Pool, manager *peers.Manager, dialog [][]*tmessage.Dialog,
+	opts Options, delay time.Duration,
+) (*iter, error) {
 	tpl, err := template.New("dl").
 		Funcs(tplfunc.FuncMap(tplfunc.All...)).
 		Parse(opts.Template)
@@ -82,6 +85,7 @@ func newIter(pool dcpool.Pool, manager *peers.Manager, dialog [][]*tmessage.Dial
 		include: includeMap,
 		exclude: excludeMap,
 		tpl:     tpl,
+		delay:   delay,
 
 		mu:          &sync.Mutex{},
 		finished:    make(map[int]struct{}),
@@ -102,6 +106,11 @@ func (i *iter) Next(ctx context.Context) bool {
 	default:
 	}
 
+	// if delay is set, sleep for a while for each iteration
+	if i.delay > 0 && (i.i+i.j) > 0 { // skip first delay
+		time.Sleep(i.delay)
+	}
+
 	for {
 		ok, skip := i.process(ctx)
 		if skip {

app/forward/forward.go

@@ -95,6 +95,7 @@ func Run(ctx context.Context, c *telegram.Client, kvd kv.KV, opts Options) (rerr
 			silent:  opts.Silent,
 			dryRun:  opts.DryRun,
 			grouped: !opts.Single,
+			delay:   viper.GetDuration(consts.FlagDelay),
 		}),
 		Progress: newProgress(fwProgress),
 		PartSize: viper.GetInt(consts.FlagPartSize),

app/forward/iter.go

@@ -3,6 +3,7 @@ package forward
 import (
 	"context"
 	"strings"
+	"time"
 
 	"github.com/expr-lang/expr/vm"
 	"github.com/go-faster/errors"
@@ -29,6 +30,7 @@ type iterOptions struct {
 	silent  bool
 	dryRun  bool
 	grouped bool
+	delay   time.Duration
 }
 
 type iter struct {
@@ -93,6 +95,11 @@ func (i *iter) Next(ctx context.Context) bool {
 		return false
 	}
 
+	// if delay is set, sleep for a while for each iteration
+	if i.opts.delay > 0 && (i.i+i.j) > 0 { // skip first delay
+		time.Sleep(i.opts.delay)
+	}
+
 	p, m := i.opts.dialogs[i.i].Peer, i.opts.dialogs[i.i].Messages[i.j]
 
 	if i.j++; i.j >= len(i.opts.dialogs[i.i].Messages) {

app/up/iter.go

@@ -3,6 +3,7 @@ package up
 import (
 	"context"
 	"os"
+	"time"
 
 	"github.com/gabriel-vasile/mimetype"
 	"github.com/go-faster/errors"
@@ -22,18 +23,20 @@ type iter struct {
 	to     peers.Peer
 	photo  bool
 	remove bool
+	delay  time.Duration
 
 	cur  int
 	err  error
 	file uploader.Elem
 }
 
-func newIter(files []*file, to peers.Peer, photo, remove bool) *iter {
+func newIter(files []*file, to peers.Peer, photo, remove bool, delay time.Duration) *iter {
 	return &iter{
 		files:  files,
 		to:     to,
 		photo:  photo,
 		remove: remove,
+		delay:  delay,
 
 		cur:  0,
 		err:  nil,
@@ -53,6 +56,11 @@ func (i *iter) Next(ctx context.Context) bool {
 		return false
 	}
 
+	// if delay is set, sleep for a while for each iteration
+	if i.delay > 0 && i.cur > 0 { // skip first delay
+		time.Sleep(i.delay)
+	}
+
 	cur := i.files[i.cur]
 	i.cur++
 

app/up/up.go

@@ -57,7 +57,7 @@ func Run(ctx context.Context, c *telegram.Client, kvd kv.KV, opts Options) (rerr
 		Client:   pool.Default(ctx),
 		PartSize: viper.GetInt(consts.FlagPartSize),
 		Threads:  viper.GetInt(consts.FlagThreads),
-		Iter:     newIter(files, to, opts.Photo, opts.Remove),
+		Iter:     newIter(files, to, opts.Photo, opts.Remove, viper.GetDuration(consts.FlagDelay)),
 		Progress: newProgress(upProgress),
 	}
 

cmd/root.go

@@ -94,6 +94,7 @@ func New() *cobra.Command {
 	cmd.PersistentFlags().IntP(consts.FlagThreads, "t", 4, "max threads for transfer one item")
 	cmd.PersistentFlags().IntP(consts.FlagLimit, "l", 2, "max number of concurrent tasks")
 	cmd.PersistentFlags().Int(consts.FlagPoolSize, 8, "specify the size of the DC pool, zero means infinity")
+	cmd.PersistentFlags().Duration(consts.FlagDelay, 0, "delay between each task, zero means no delay")
 
 	cmd.PersistentFlags().String(consts.FlagNTP, "", "ntp server host, if not set, use system time")
 	cmd.PersistentFlags().Duration(consts.FlagReconnectTimeout, 5*time.Minute, "Telegram client reconnection backoff timeout, infinite if set to 0") // #158

docs/content/en/guide/global-config.md

@@ -87,3 +87,15 @@ Set higher timeout or 0(INF) if you want faster speed.
 {{< command >}}
 tdl --pool 2
 {{< /command >}}
+
+## `--delay`
+
+set the delay between each task. Default: `0s`.
+
+{{< hint info >}}
+Set higher delay time if you want to avoid Telegram's flood control.
+{{< /hint >}}
+
+{{< command >}}
+tdl --delay 5s
+{{< /command >}}

docs/content/zh/guide/global-config.md

@@ -86,3 +86,16 @@ tdl --debug
 {{< command >}}
 tdl --pool 2
 {{< /command >}}
+
+## `--delay`
+
+设置每个任务之间的延迟。默认值：`0s`。
+
+{{< hint info >}}
+如果你想避免因为短时间内产生大量请求被限流，请设置更长的延迟时间。
+{{< /hint >}}
+
+{{< command >}}
+tdl --delay 5s
+{{< /command >}}
+

pkg/consts/flag.go

@@ -9,6 +9,7 @@ const (
 	FlagThreads          = "threads"
 	FlagLimit            = "limit"
 	FlagPoolSize         = "pool"
+	FlagDelay            = "delay"
 	FlagNTP              = "ntp"
 	FlagReconnectTimeout = "reconnect-timeout"
 	FlagDlTemplate       = "template"