From 0a084ef513bde506a8c7e9e77fee408fe8b5f6e7 Mon Sep 17 00:00:00 2001 From: Junyu Liu Date: Thu, 7 Nov 2024 18:14:38 +0800 Subject: [PATCH] fix(prj): set part size to max by default * fix part size * remove useless --size flag in e2e test --- app/dl/dl.go | 2 -- app/forward/forward.go | 1 - app/up/up.go | 1 - cmd/root.go | 2 ++ core/downloader/downloader.go | 8 +++++--- core/forwarder/clone.go | 6 ++++-- core/forwarder/forwarder.go | 1 - core/uploader/uploader.go | 6 ++++-- pkg/consts/flag.go | 2 +- test/suite_test.go | 1 - 10 files changed, 16 insertions(+), 14 deletions(-) diff --git a/app/dl/dl.go b/app/dl/dl.go index 6b9dd66..0929e36 100644 --- a/app/dl/dl.go +++ b/app/dl/dl.go @@ -103,7 +103,6 @@ func Run(ctx context.Context, c *telegram.Client, kvd kv.KV, opts Options) (rerr options := downloader.Options{ Pool: pool, - PartSize: viper.GetInt(consts.FlagPartSize), Threads: viper.GetInt(consts.FlagThreads), Iter: it, Progress: newProgress(dlProgress, it, opts), @@ -114,7 +113,6 @@ func Run(ctx context.Context, c *telegram.Client, kvd kv.KV, opts Options) (rerr zap.String("dir", opts.Dir), zap.Bool("rewrite_ext", opts.RewriteExt), zap.Bool("skip_same", opts.SkipSame), - zap.Int("part_size", options.PartSize), zap.Int("threads", options.Threads), zap.Int("limit", limit)) diff --git a/app/forward/forward.go b/app/forward/forward.go index f0e6770..0b6a357 100644 --- a/app/forward/forward.go +++ b/app/forward/forward.go @@ -98,7 +98,6 @@ func Run(ctx context.Context, c *telegram.Client, kvd kv.KV, opts Options) (rerr delay: viper.GetDuration(consts.FlagDelay), }), Progress: newProgress(fwProgress), - PartSize: viper.GetInt(consts.FlagPartSize), Threads: viper.GetInt(consts.FlagThreads), }) diff --git a/app/up/up.go b/app/up/up.go index 935db9b..b1ad2fe 100644 --- a/app/up/up.go +++ b/app/up/up.go @@ -56,7 +56,6 @@ func Run(ctx context.Context, c *telegram.Client, kvd kv.KV, opts Options) (rerr options := uploader.Options{ Client: pool.Default(ctx), - PartSize: viper.GetInt(consts.FlagPartSize), Threads: viper.GetInt(consts.FlagThreads), Iter: newIter(files, to, opts.Photo, opts.Remove, viper.GetDuration(consts.FlagDelay)), Progress: newProgress(upProgress), diff --git a/cmd/root.go b/cmd/root.go index 39d7b7c..c45c4f4 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -156,6 +156,8 @@ func New() *cobra.Command { cmd.PersistentFlags().Bool(consts.FlagDebug, false, "enable debug mode") cmd.PersistentFlags().IntP(consts.FlagPartSize, "s", 512*1024, "part size for transfer") + _ = cmd.PersistentFlags().MarkDeprecated(consts.FlagPartSize, "part size has been set to maximum by default, this flag will be removed in the future") + cmd.PersistentFlags().IntP(consts.FlagThreads, "t", 4, "max threads for transfer one item") cmd.PersistentFlags().IntP(consts.FlagLimit, "l", 2, "max number of concurrent tasks") cmd.PersistentFlags().Int(consts.FlagPoolSize, 8, "specify the size of the DC pool, zero means infinity") diff --git a/core/downloader/downloader.go b/core/downloader/downloader.go index fded2b5..6d34beb 100644 --- a/core/downloader/downloader.go +++ b/core/downloader/downloader.go @@ -13,13 +13,15 @@ import ( "github.com/iyear/tdl/core/util/tutil" ) +// MaxPartSize refer to https://core.telegram.org/api/files#downloading-files +const MaxPartSize = 1024 * 1024 + type Downloader struct { opts Options } type Options struct { Pool dcpool.Pool - PartSize int Threads int Iter Iter Progress Progress @@ -77,10 +79,10 @@ func (d *Downloader) download(ctx context.Context, elem Elem) error { client = d.opts.Pool.Takeout(ctx, elem.File().DC()) } - _, err := downloader.NewDownloader().WithPartSize(d.opts.PartSize). + _, err := downloader.NewDownloader().WithPartSize(MaxPartSize). Download(client, elem.File().Location()). WithThreads(tutil.BestThreads(elem.File().Size(), d.opts.Threads)). - Parallel(ctx, newWriteAt(elem, d.opts.Progress, d.opts.PartSize)) + Parallel(ctx, newWriteAt(elem, d.opts.Progress, MaxPartSize)) if err != nil { return errors.Wrap(err, "download") } diff --git a/core/forwarder/clone.go b/core/forwarder/clone.go index 78ea2c9..6dd9f4e 100644 --- a/core/forwarder/clone.go +++ b/core/forwarder/clone.go @@ -12,7 +12,9 @@ import ( "go.uber.org/atomic" "go.uber.org/multierr" + tdownloader "github.com/iyear/tdl/core/downloader" "github.com/iyear/tdl/core/tmedia" + tuploader "github.com/iyear/tdl/core/uploader" "github.com/iyear/tdl/core/util/tutil" ) @@ -47,7 +49,7 @@ func (f *Forwarder) cloneMedia(ctx context.Context, opts cloneOptions, dryRun bo threads := tutil.BestThreads(opts.media.Size, f.opts.Threads) _, err = downloader.NewDownloader(). - WithPartSize(f.opts.PartSize). + WithPartSize(tdownloader.MaxPartSize). Download(f.opts.Pool.Client(ctx, opts.media.DC), opts.media.InputFileLoc). WithThreads(threads). Parallel(ctx, writeAt{ @@ -66,7 +68,7 @@ func (f *Forwarder) cloneMedia(ctx context.Context, opts cloneOptions, dryRun bo upload := uploader.NewUpload(opts.media.Name, temp, opts.media.Size) file, err = uploader.NewUploader(f.opts.Pool.Default(ctx)). - WithPartSize(f.opts.PartSize). + WithPartSize(tuploader.MaxPartSize). WithThreads(threads). WithProgress(uploaded{ opts: opts, diff --git a/core/forwarder/forwarder.go b/core/forwarder/forwarder.go index 0569533..51a677a 100644 --- a/core/forwarder/forwarder.go +++ b/core/forwarder/forwarder.go @@ -26,7 +26,6 @@ type Mode int type Options struct { Pool dcpool.Pool - PartSize int Threads int Iter Iter Progress Progress diff --git a/core/uploader/uploader.go b/core/uploader/uploader.go index f513524..11d5da8 100644 --- a/core/uploader/uploader.go +++ b/core/uploader/uploader.go @@ -17,13 +17,15 @@ import ( "github.com/iyear/tdl/core/util/mediautil" ) +// MaxPartSize refer to https://core.telegram.org/api/files#uploading-files +const MaxPartSize = 512 * 1024 + type Uploader struct { opts Options } type Options struct { Client *tg.Client - PartSize int Threads int Iter Iter Progress Progress @@ -72,7 +74,7 @@ func (u *Uploader) upload(ctx context.Context, elem Elem) error { } up := uploader.NewUploader(u.opts.Client). - WithPartSize(u.opts.PartSize). + WithPartSize(MaxPartSize). WithThreads(u.opts.Threads). WithProgress(&wrapProcess{ elem: elem, diff --git a/pkg/consts/flag.go b/pkg/consts/flag.go index 5b31f69..1df7b7d 100644 --- a/pkg/consts/flag.go +++ b/pkg/consts/flag.go @@ -5,7 +5,7 @@ const ( FlagProxy = "proxy" FlagNamespace = "ns" FlagDebug = "debug" - FlagPartSize = "size" + FlagPartSize = "size" // Deprecated: all part size should be set to maximum by default FlagThreads = "threads" FlagLimit = "limit" FlagPoolSize = "pool" diff --git a/test/suite_test.go b/test/suite_test.go index 7053128..9e410d3 100644 --- a/test/suite_test.go +++ b/test/suite_test.go @@ -53,7 +53,6 @@ func exec(cmd *cobra.Command, args []string, success bool) { log.Printf("args: %s\n", args) cmd.SetArgs(append([]string{ - "-s", "131072", // self-hosted Telegram server don't support 1MiB "-n", testAccount, "--storage", fmt.Sprintf("type=file,path=%s", sessionFile), }, args...))