feat(dl): files mode

app/dl/dl.go

@@ -11,7 +11,7 @@ import (
 	"github.com/spf13/viper"
 )
 
-func Run(ctx context.Context, urls []string) error {
+func Run(ctx context.Context, urls, files []string) error {
 	kvd, err := kv.New(kv.Options{
 		Path: consts.KVPath,
 		NS:   viper.GetString(consts.FlagNamespace),
@@ -38,9 +38,13 @@ func Run(ctx context.Context, urls []string) error {
 		if err != nil {
 			return err
 		}
-		// TODO(iyear): files msgs
 
-		return downloader.New(c.API(), viper.GetInt(consts.FlagPartSize), viper.GetInt(consts.FlagThreads), newIter(c.API(), umsgs)).
+		fmsgs, err := parseFiles(ctx, c.API(), files)
+		if err != nil {
+			return err
+		}
+
+		return downloader.New(c.API(), viper.GetInt(consts.FlagPartSize), viper.GetInt(consts.FlagThreads), newIter(c.API(), umsgs, fmsgs)).
 			Download(ctx, viper.GetInt(consts.FlagLimit))
 	})
 }

app/dl/files.go

@@ -0,0 +1,172 @@
+package dl
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"github.com/bcicen/jstream"
+	"github.com/gotd/td/telegram/peers"
+	"github.com/gotd/td/tg"
+	"github.com/mitchellh/mapstructure"
+	"io"
+	"os"
+)
+
+// https://github.com/telegramdesktop/tdesktop/blob/dev/Telegram/SourceFiles/export/output/export_output_json.cpp#L1112-L1124
+var typeMap = map[string]uint32{
+	"saved_messages":     tg.InputPeerSelfTypeID,
+	"personal_chat":      tg.InputPeerUserTypeID,
+	"bot_chat":           tg.InputPeerUserTypeID,
+	"private_group":      tg.InputPeerChatTypeID,
+	"private_supergroup": tg.InputPeerChannelTypeID,
+	"public_supergroup":  tg.InputPeerChannelTypeID,
+	"private_channel":    tg.InputPeerChannelTypeID,
+	"public_channel":     tg.InputPeerChannelTypeID,
+}
+
+const (
+	keyID       = "id"
+	keyType     = "type"
+	typeMessage = "message"
+)
+
+type fMessage struct {
+	ID     int         `mapstructure:"id"`
+	Type   string      `mapstructure:"type"`
+	Time   string      `mapstructure:"date_unixtime"`
+	File   string      `mapstructure:"file"`
+	Photo  string      `mapstructure:"photo"`
+	FromID string      `mapstructure:"from_id"`
+	From   string      `mapstructure:"from"`
+	Text   interface{} `mapstructure:"text"`
+}
+
+func parseFiles(ctx context.Context, client *tg.Client, files []string) ([]*dialog, error) {
+	dialogs := make([]*dialog, 0, len(files))
+
+	for _, file := range files {
+		d, err := parseFile(ctx, client, file)
+		if err != nil {
+			return nil, err
+		}
+
+		dialogs = append(dialogs, d)
+	}
+
+	return dialogs, nil
+}
+
+func parseFile(ctx context.Context, client *tg.Client, file string) (*dialog, error) {
+	f, err := os.Open(file)
+	if err != nil {
+		return nil, err
+	}
+	defer func(f *os.File) {
+		_ = f.Close()
+	}(f)
+
+	peer, err := getChatInfo(ctx, client, f)
+	if err != nil {
+		return nil, err
+	}
+
+	if _, err = f.Seek(0, io.SeekStart); err != nil {
+		return nil, err
+	}
+
+	return collect(ctx, f, peer)
+}
+
+func collect(ctx context.Context, r io.Reader, peer tg.InputPeerClass) (*dialog, error) {
+	d := jstream.NewDecoder(r, 2)
+
+	m := &dialog{
+		peer: peer,
+		msgs: make([]int, 0),
+	}
+
+	for mv := range d.Stream() {
+		select {
+		case <-ctx.Done():
+			return nil, ctx.Err()
+		default:
+			fm := fMessage{}
+
+			if mv.ValueType != jstream.Object {
+				continue
+			}
+
+			if err := mapstructure.WeakDecode(mv.Value, &fm); err != nil {
+				return nil, err
+			}
+
+			if fm.ID < 0 || fm.Type != typeMessage {
+				continue
+			}
+
+			if fm.File == "" && fm.Photo == "" {
+				continue
+			}
+
+			m.msgs = append(m.msgs, fm.ID)
+		}
+	}
+
+	return m, nil
+}
+
+func getChatInfo(ctx context.Context, client *tg.Client, r io.Reader) (tg.InputPeerClass, error) {
+	d := jstream.NewDecoder(r, 1).EmitKV()
+
+	chatType, chatID := uint32(0), int64(0)
+
+	for mv := range d.Stream() {
+		kv, ok := mv.Value.(jstream.KV)
+		if !ok {
+			continue
+		}
+
+		if kv.Key == keyType {
+			v := kv.Value.(string)
+			chatType, ok = typeMap[v]
+			if !ok {
+				return nil, fmt.Errorf("unsupported dialog type: %s", v)
+			}
+		}
+
+		if kv.Key == keyID {
+			chatID = int64(kv.Value.(float64))
+		}
+
+		if chatType != 0 && chatID != 0 {
+			break
+		}
+	}
+
+	if chatType == 0 || chatID == 0 {
+		return nil, errors.New("can't get chat type or chat id")
+	}
+
+	var (
+		peer peers.Peer
+		err  error
+	)
+	manager := peers.Options{}.Build(client)
+
+	switch chatType {
+	case tg.InputPeerSelfTypeID:
+		return &tg.InputPeerSelf{}, nil
+	case tg.InputPeerUserTypeID:
+		peer, err = manager.ResolveUserID(ctx, chatID)
+	case tg.InputPeerChatTypeID:
+		peer, err = manager.ResolveChatID(ctx, chatID)
+	case tg.InputPeerChannelTypeID:
+		peer, err = manager.ResolveChannelID(ctx, chatID)
+	}
+
+	if err != nil {
+		return nil, err
+	}
+
+	return peer.InputPeer(), nil
+}

cmd/dl/dl.go

@@ -7,7 +7,7 @@ import (
 )
 
 var (
-	urls []string
+	urls, files []string
 )
 
 var Cmd = &cobra.Command{
@@ -16,10 +16,11 @@ var Cmd = &cobra.Command{
 	Short:   "Download anything from Telegram (protected) chat",
 	Example: "tdl dl -n iyear --proxy socks5://localhost:1080 -u https://t.me/tdl/1 -u https://t.me/tdl/2 -s 262144 -t 16 -l 3",
 	RunE: func(cmd *cobra.Command, args []string) error {
-		return dl.Run(cmd.Context(), urls)
+		return dl.Run(cmd.Context(), urls, files)
 	},
 }
 
 func init() {
-	Cmd.Flags().StringSliceVarP(&urls, consts.FlagDlUrl, "u", []string{}, "telegram message links to be downloaded")
+	Cmd.Flags().StringSliceVarP(&urls, consts.FlagDlUrl, "u", []string{}, "telegram message links")
+	Cmd.Flags().StringSliceVarP(&files, consts.FlagDlFile, "f", []string{}, "official client export files")
 }

go.mod

@@ -18,6 +18,7 @@ require (
 )
 
 require (
+	github.com/bcicen/jstream v1.0.1 // indirect
 	github.com/beevik/ntp v0.3.0 // indirect
 	github.com/cenkalti/backoff/v4 v4.1.3 // indirect
 	github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect

go.sum

@@ -38,6 +38,8 @@ cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3f
 dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
+github.com/bcicen/jstream v1.0.1 h1:BXY7Cu4rdmc0rhyTVyT3UkxAiX3bnLpKLas9btbH5ck=
+github.com/bcicen/jstream v1.0.1/go.mod h1:9ielPxqFry7Y4Tg3j4BfjPocfJ3TbsRtXOAYXYmRuAQ=
 github.com/beevik/ntp v0.3.0 h1:xzVrPrE4ziasFXgBVBZJDP0Wg/KpMwk2KHJ4Ba8GrDw=
 github.com/beevik/ntp v0.3.0/go.mod h1:hIHWr+l3+/clUnF44zdK+CWW7fO8dR5cIylAQ76NRpg=
 github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8=

pkg/consts/flag.go

@@ -12,4 +12,5 @@ const (
 	FlagUpExcludes   = "excludes"
 	FlagLoginDesktop = "desktop"
 	FlagDlUrl        = "url"
+	FlagDlFile       = "file"
 )