Minor fixes

Signed-off-by: anasty17 <e.anastayyar@gmail.com>
This commit is contained in:
anasty17 2022-09-20 18:24:57 +03:00
parent c53167c9fe
commit 484ca384d9
8 changed files with 112 additions and 76 deletions

View File

@ -24,12 +24,12 @@ In each single file there is a major change from base code, it's almost totaly d
- Upload all files to specific superGroup/channel.
### Google
- Stop duplicates for all tasks except yt-dlp tasks
- Download G-Drive links
- Counting files/folders from Google Drive link
- Download from Google Drive
- Counting Google Drive files/folders
- Search in multiple Drive folder/TeamDrive
- Recursive Search (only with `root` or TeamDrive ID, folder ids will be listed with non-recursive method)
- Use Token.pickle if file not found with Service Account, for all Gdrive functions
- List result in html file instead of telegraph or telegram message to avoid limits by @junedkh
- List result in html file instead of telegraph or telegram message to avoid limits by [junedkh](https://github.com/junedkh)
- Random Service Account for each task
### Status
- Clone Status
@ -134,21 +134,22 @@ cp config_sample.env config.env
```
_____REMOVE_THIS_LINE_____=True
```
Fill up rest of the fields. Meaning of each field is discussed below:
Fill up rest of the fields. Meaning of each field is discussed below. **NOTE**: All values must be filled between quotes, even if `Int` or `Bool`.
**1. Required Fields**
- `BOT_TOKEN`: The Telegram Bot Token that you got from [@BotFather](https://t.me/BotFather). `Str`
- `GDRIVE_FOLDER_ID`: This is the Folder/TeamDrive ID of the Google Drive Folder or `root` to which you want to upload all the mirrors. `Str`
- `OWNER_ID`: The Telegram User ID (not username) of the Owner of the bot. `Int`
- `DOWNLOAD_DIR`: The path to the local folder where the downloads should be downloaded to. `Str`
- `DOWNLOAD_STATUS_UPDATE_INTERVAL`: Time in seconds after which the progress/status message will be updated. Recommended `10` seconds at least. `Int`
- `AUTO_DELETE_MESSAGE_DURATION`: Interval of time (in seconds), after which the bot deletes it's message and command message which is expected to be viewed instantly. **NOTE**: Set to `-1` to disable auto message deletion. `Int`
- `TELEGRAM_API`: This is to authenticate your Telegram account for downloading Telegram files. You can get this from https://my.telegram.org. `Int`
- `TELEGRAM_HASH`: This is to authenticate your Telegram account for downloading Telegram files. You can get this from https://my.telegram.org. `Str`
**2. Optional Fields**
- `GDRIVE_FOLDER_ID`: This is the Folder/TeamDrive ID of the Google Drive Folder or `root` to which you want to upload all the mirrors. Required for `Google Drive` upload. `Str`
- `IS_TEAM_DRIVE`: Set `True` if uploading to TeamDrive. Default is `False`. `Bool`
- `DOWNLOAD_DIR`: The path to the local folder where the downloads should be downloaded to. `Str`
- `DOWNLOAD_STATUS_UPDATE_INTERVAL`: Time in seconds after which the progress/status message will be updated. Recommended `10` seconds at least. `Int`
- `AUTO_DELETE_MESSAGE_DURATION`: Interval of time (in seconds), after which the bot deletes it's message and command message which is expected to be viewed instantly. **NOTE**: Set to `-1` to disable auto message deletion. `Int`
- `DATABASE_URL`: Your SQL Database URL. Follow this [Generate Database](https://github.com/anasty17/mirror-leech-telegram-bot/tree/master#generate-database) to generate database. Data will be saved in Database: auth and sudo users, leech settings including thumbnails for each user, rss data and incomplete tasks. **NOTE**: If deploying on heroku and using heroku postgresql delete this variable from **config.env** file. **DATABASE_URL** will be grabbed from heroku variables. `Str`
- `AUTHORIZED_CHATS`: Fill user_id and chat_id of groups/users you want to authorize. Separate them by space. `Str`
- `SUDO_USERS`: Fill user_id of users whom you want to give sudo permission. Separate them by space. `Str`

View File

@ -61,12 +61,12 @@ QbInterval = []
DRIVES_NAMES = []
DRIVES_IDS = []
INDEX_URLS = []
def getConfig(name: str):
return environ[name]
AS_DOC_USERS = set()
AS_MEDIA_USERS = set()
EXTENSION_FILTER = {'.aria2'}
try:
if bool(getConfig('_____REMOVE_THIS_LINE_____')):
if bool(environ.get('_____REMOVE_THIS_LINE_____')):
log_error('The README.md file there to be read! Exiting now!')
exit()
except:
@ -95,25 +95,52 @@ download_dict = {}
# value: [rss_feed, last_link, last_title, filter]
rss_dict = {}
AS_DOC_USERS = set()
AS_MEDIA_USERS = set()
EXTENSION_FILTER = set(['.aria2'])
BOT_TOKEN = environ.get('BOT_TOKEN', '')
if len(BOT_TOKEN) == 0:
log_error("BOT_TOKEN variable is missing! Exiting now")
exit(1)
OWNER_ID = environ.get('OWNER_ID', '')
if len(OWNER_ID) == 0:
log_error("OWNER_ID variable is missing! Exiting now")
exit(1)
else:
OWNER_ID = int(OWNER_ID)
try:
BOT_TOKEN = getConfig('BOT_TOKEN')
parent_id = getConfig('GDRIVE_FOLDER_ID')
DOWNLOAD_DIR = getConfig('DOWNLOAD_DIR')
TELEGRAM_API = environ.get('TELEGRAM_API', '')
if len(TELEGRAM_API) == 0:
log_error("TELEGRAM_API variable is missing! Exiting now")
exit(1)
else:
TELEGRAM_API = int(TELEGRAM_API)
TELEGRAM_HASH = environ.get('TELEGRAM_HASH', '')
if len(TELEGRAM_HASH) == 0:
log_error("TELEGRAM_HASH variable is missing! Exiting now")
exit(1)
PARENT_ID = environ.get('GDRIVE_FOLDER_ID', '')
if len(PARENT_ID) == 0:
PARENT_ID = None
DOWNLOAD_DIR = environ.get('DOWNLOAD_DIR', '')
if len(DOWNLOAD_DIR) == 0:
DOWNLOAD_DIR = '/usr/src/app/downloads/'
else:
if not DOWNLOAD_DIR.endswith("/"):
DOWNLOAD_DIR = DOWNLOAD_DIR + '/'
DOWNLOAD_STATUS_UPDATE_INTERVAL = int(getConfig('DOWNLOAD_STATUS_UPDATE_INTERVAL'))
OWNER_ID = int(getConfig('OWNER_ID'))
AUTO_DELETE_MESSAGE_DURATION = int(getConfig('AUTO_DELETE_MESSAGE_DURATION'))
TELEGRAM_API = getConfig('TELEGRAM_API')
TELEGRAM_HASH = getConfig('TELEGRAM_HASH')
except:
log_error("One or more env variables missing! Exiting now")
exit(1)
DOWNLOAD_STATUS_UPDATE_INTERVAL = environ.get('DOWNLOAD_STATUS_UPDATE_INTERVAL', '')
if len(DOWNLOAD_STATUS_UPDATE_INTERVAL) == 0:
DOWNLOAD_STATUS_UPDATE_INTERVAL = 10
else:
DOWNLOAD_STATUS_UPDATE_INTERVAL = int(DOWNLOAD_STATUS_UPDATE_INTERVAL)
AUTO_DELETE_MESSAGE_DURATION = environ.get('AUTO_DELETE_MESSAGE_DURATION', '')
if len(AUTO_DELETE_MESSAGE_DURATION) == 0:
AUTO_DELETE_MESSAGE_DURATION = 30
else:
AUTO_DELETE_MESSAGE_DURATION = int(AUTO_DELETE_MESSAGE_DURATION)
aid = environ.get('AUTHORIZED_CHATS', '')
if len(aid) != 0:
@ -133,15 +160,14 @@ if len(fx) > 0:
for x in fx:
EXTENSION_FILTER.add(x.strip().lower())
IS_PREMIUM_USER = False
USER_SESSION_STRING = environ.get('USER_SESSION_STRING', '')
if len(USER_SESSION_STRING) == 0:
log_info("Creating client from BOT_TOKEN")
app = Client(name='pyrogram', api_id=int(TELEGRAM_API), api_hash=TELEGRAM_HASH, bot_token=BOT_TOKEN, parse_mode=enums.ParseMode.HTML, no_updates=True)
app = Client(name='pyrogram', api_id=TELEGRAM_API, api_hash=TELEGRAM_HASH, bot_token=BOT_TOKEN, parse_mode=enums.ParseMode.HTML, no_updates=True)
else:
log_info("Creating client from USER_SESSION_STRING")
app = Client(name='pyrogram', api_id=int(TELEGRAM_API), api_hash=TELEGRAM_HASH, session_string=USER_SESSION_STRING, parse_mode=enums.ParseMode.HTML, no_updates=True)
app = Client(name='pyrogram', api_id=TELEGRAM_API, api_hash=TELEGRAM_HASH, session_string=USER_SESSION_STRING, parse_mode=enums.ParseMode.HTML, no_updates=True)
with app:
IS_PREMIUM_USER = app.me.is_premium
@ -150,7 +176,7 @@ if len(RSS_USER_SESSION_STRING) == 0:
rss_session = None
else:
log_info("Creating client from RSS_USER_SESSION_STRING")
rss_session = Client(name='rss_session', api_id=int(TELEGRAM_API), api_hash=TELEGRAM_HASH, session_string=RSS_USER_SESSION_STRING, parse_mode=enums.ParseMode.HTML, no_updates=True)
rss_session = Client(name='rss_session', api_id=TELEGRAM_API, api_hash=TELEGRAM_HASH, session_string=RSS_USER_SESSION_STRING, parse_mode=enums.ParseMode.HTML, no_updates=True)
def aria2c_init():
try:
@ -319,7 +345,7 @@ if len(YT_COOKIES_URL) != 0:
log_error(f"YT_COOKIES_URL: {e}")
DRIVES_NAMES.append("Main")
DRIVES_IDS.append(parent_id)
DRIVES_IDS.append(PARENT_ID)
if ospath.exists('drive_folder'):
with open('drive_folder', 'r+') as f:
lines = f.readlines()

View File

@ -46,8 +46,8 @@ class setInterval:
def __setInterval(self):
nextTime = time() + self.interval
while not self.stopEvent.wait(nextTime - time()):
nextTime += self.interval
self.action()
nextTime = time() + self.interval
def cancel(self):
self.stopEvent.set()

View File

@ -7,7 +7,8 @@ def get_download(gid):
try:
return aria2.get_download(gid)
except Exception as e:
LOGGER.error(f'{e}: while getting torrent info')
LOGGER.error(f'{e}: Aria2c, Error while getting torrent info')
return get_download(gid)
class AriaDownloadStatus:
@ -22,7 +23,9 @@ class AriaDownloadStatus:
def __update(self):
self.__download = self.__download.live
if self.__download.followed_by_ids:
if self.__download is None:
self.__download = get_download(self.__gid)
elif self.__download.followed_by_ids:
self.__gid = self.__download.followed_by_ids[0]
self.__download = get_download(self.__gid)

View File

@ -7,7 +7,7 @@ def get_download(client, hash_):
try:
return client.torrents_info(torrent_hashes=hash_)[0]
except Exception as e:
LOGGER.error(f'{e}: while getting torrent info')
LOGGER.error(f'{e}: Qbittorrent, Error while getting torrent info')
client = get_client()
return get_download(client, hash_)

View File

@ -15,7 +15,7 @@ from googleapiclient.http import MediaFileUpload, MediaIoBaseDownload
from tenacity import retry, wait_exponential, stop_after_attempt, retry_if_exception_type, RetryError
from bot.helper.telegram_helper.button_build import ButtonMaker
from bot import parent_id, IS_TEAM_DRIVE, INDEX_URL, USE_SERVICE_ACCOUNTS, VIEW_LINK, \
from bot import PARENT_ID, IS_TEAM_DRIVE, INDEX_URL, USE_SERVICE_ACCOUNTS, VIEW_LINK, \
DRIVES_NAMES, DRIVES_IDS, INDEX_URLS, EXTENSION_FILTER
from bot.helper.ext_utils.bot_utils import get_readable_file_size, setInterval
from bot.helper.ext_utils.fs_utils import get_mime_type
@ -203,7 +203,7 @@ class GoogleDriveHelper:
try:
if ospath.isfile(file_path):
mime_type = get_mime_type(file_path)
link = self.__upload_file(file_path, file_name, mime_type, parent_id)
link = self.__upload_file(file_path, file_name, mime_type, PARENT_ID)
if self.__is_cancelled:
return
if link is None:
@ -211,7 +211,7 @@ class GoogleDriveHelper:
LOGGER.info(f"Uploaded To G-Drive: {file_path}")
else:
mime_type = 'Folder'
dir_id = self.__create_directory(ospath.basename(ospath.abspath(file_name)), parent_id)
dir_id = self.__create_directory(ospath.basename(ospath.abspath(file_name)), PARENT_ID)
result = self.__upload_dir(file_path, dir_id)
if result is None:
raise Exception('Upload has been manually cancelled!')
@ -237,38 +237,38 @@ class GoogleDriveHelper:
return
self.__listener.onUploadComplete(link, size, self.__total_files, self.__total_folders, mime_type, self.name)
def __upload_dir(self, input_directory, parent_id):
def __upload_dir(self, input_directory, PARENT_ID):
list_dirs = listdir(input_directory)
if len(list_dirs) == 0:
return parent_id
return PARENT_ID
new_id = None
for item in list_dirs:
current_file_name = ospath.join(input_directory, item)
if ospath.isdir(current_file_name):
current_dir_id = self.__create_directory(item, parent_id)
current_dir_id = self.__create_directory(item, PARENT_ID)
new_id = self.__upload_dir(current_file_name, current_dir_id)
self.__total_folders += 1
elif not item.lower().endswith(tuple(EXTENSION_FILTER)):
mime_type = get_mime_type(current_file_name)
file_name = current_file_name.split("/")[-1]
# current_file_name will have the full path
self.__upload_file(current_file_name, file_name, mime_type, parent_id)
self.__upload_file(current_file_name, file_name, mime_type, PARENT_ID)
self.__total_files += 1
new_id = parent_id
new_id = PARENT_ID
if self.__is_cancelled:
break
return new_id
@retry(wait=wait_exponential(multiplier=2, min=3, max=6), stop=stop_after_attempt(3),
retry=retry_if_exception_type(GCError))
def __create_directory(self, directory_name, parent_id):
def __create_directory(self, directory_name, PARENT_ID):
file_metadata = {
"name": directory_name,
"description": "Uploaded by Mirror-leech-telegram-bot",
"mimeType": self.__G_DRIVE_DIR_MIME_TYPE
}
if parent_id is not None:
file_metadata["parents"] = [parent_id]
if PARENT_ID is not None:
file_metadata["parents"] = [PARENT_ID]
file = self.__service.files().create(supportsTeamDrives=True, body=file_metadata).execute()
file_id = file.get("id")
if not IS_TEAM_DRIVE:
@ -278,15 +278,15 @@ class GoogleDriveHelper:
@retry(wait=wait_exponential(multiplier=2, min=3, max=6), stop=stop_after_attempt(3),
retry=(retry_if_exception_type(GCError) | retry_if_exception_type(IOError)))
def __upload_file(self, file_path, file_name, mime_type, parent_id):
def __upload_file(self, file_path, file_name, mime_type, PARENT_ID):
# File body description
file_metadata = {
'name': file_name,
'description': 'Uploaded by Mirror-leech-telegram-bot',
'mimeType': mime_type,
}
if parent_id is not None:
file_metadata['parents'] = [parent_id]
if PARENT_ID is not None:
file_metadata['parents'] = [PARENT_ID]
if ospath.getsize(file_path) == 0:
media_body = MediaFileUpload(file_path,
@ -326,7 +326,7 @@ class GoogleDriveHelper:
if USE_SERVICE_ACCOUNTS:
self.__switchServiceAccount()
LOGGER.info(f"Got: {reason}, Trying Again.")
return self.__upload_file(file_path, file_name, mime_type, parent_id)
return self.__upload_file(file_path, file_name, mime_type, PARENT_ID)
else:
LOGGER.error(f"Got: {reason}")
raise err
@ -362,7 +362,7 @@ class GoogleDriveHelper:
meta = self.__getFileMetadata(file_id)
mime_type = meta.get("mimeType")
if mime_type == self.__G_DRIVE_DIR_MIME_TYPE:
dir_id = self.__create_directory(meta.get('name'), parent_id)
dir_id = self.__create_directory(meta.get('name'), PARENT_ID)
self.__cloneFolder(meta.get('name'), meta.get('name'), meta.get('id'), dir_id)
durl = self.__G_DRIVE_DIR_BASE_DOWNLOAD_URL.format(dir_id)
if self.__is_cancelled:
@ -381,7 +381,7 @@ class GoogleDriveHelper:
url = f'{INDEX_URL}/{url_path}/'
buttons.buildbutton("⚡ Index Link", url)
else:
file = self.__copyFile(meta.get('id'), parent_id)
file = self.__copyFile(meta.get('id'), PARENT_ID)
msg += f'<b>Name: </b><code>{file.get("name")}</code>'
durl = self.__G_DRIVE_BASE_DOWNLOAD_URL.format(file.get("id"))
buttons = ButtonMaker()
@ -415,21 +415,21 @@ class GoogleDriveHelper:
return msg, ""
return msg, buttons.build_menu(2)
def __cloneFolder(self, name, local_path, folder_id, parent_id):
def __cloneFolder(self, name, local_path, folder_id, PARENT_ID):
LOGGER.info(f"Syncing: {local_path}")
files = self.__getFilesByFolderId(folder_id)
if len(files) == 0:
return parent_id
return PARENT_ID
for file in files:
if file.get('mimeType') == self.__G_DRIVE_DIR_MIME_TYPE:
self.__total_folders += 1
file_path = ospath.join(local_path, file.get('name'))
current_dir_id = self.__create_directory(file.get('name'), parent_id)
current_dir_id = self.__create_directory(file.get('name'), PARENT_ID)
self.__cloneFolder(file.get('name'), file_path, file.get('id'), current_dir_id)
elif not file.get('name').lower().endswith(tuple(EXTENSION_FILTER)):
self.__total_files += 1
self.transferred_size += int(file.get('size', 0))
self.__copyFile(file.get('id'), parent_id)
self.__copyFile(file.get('id'), PARENT_ID)
if self.__is_cancelled:
break
@ -487,7 +487,7 @@ class GoogleDriveHelper:
rtnlist.reverse()
return rtnlist
def __drive_query(self, parent_id, fileName, stopDup, isRecursive, itemType):
def __drive_query(self, PARENT_ID, fileName, stopDup, isRecursive, itemType):
try:
if isRecursive:
if stopDup:
@ -504,7 +504,7 @@ class GoogleDriveHelper:
elif itemType == "folders":
query += "mimeType = 'application/vnd.google-apps.folder' and "
query += "trashed = false"
if parent_id == "root":
if PARENT_ID == "root":
return (
self.__service.files()
.list(q=f"{query} and 'me' in owners",
@ -520,7 +520,7 @@ class GoogleDriveHelper:
self.__service.files()
.list(supportsTeamDrives=True,
includeTeamDriveItems=True,
teamDriveId=parent_id,
teamDriveId=PARENT_ID,
q=query,
corpora='drive',
spaces='drive',
@ -532,9 +532,9 @@ class GoogleDriveHelper:
)
else:
if stopDup:
query = f"'{parent_id}' in parents and name = '{fileName}' and "
query = f"'{PARENT_ID}' in parents and name = '{fileName}' and "
else:
query = f"'{parent_id}' in parents and "
query = f"'{PARENT_ID}' in parents and "
fileName = fileName.split()
for name in fileName:
if name != '':
@ -571,9 +571,9 @@ class GoogleDriveHelper:
token_service = self.__alt_authorize()
if token_service is not None:
self.__service = token_service
for index, parent_id in enumerate(DRIVES_IDS):
isRecur = False if isRecursive and len(parent_id) > 23 else isRecursive
response = self.__drive_query(parent_id, fileName, stopDup, isRecur, itemType)
for index, PARENT_ID in enumerate(DRIVES_IDS):
isRecur = False if isRecursive and len(PARENT_ID) > 23 else isRecursive
response = self.__drive_query(PARENT_ID, fileName, stopDup, isRecur, itemType)
if not response["files"]:
if noMulti:
break
@ -596,7 +596,7 @@ class GoogleDriveHelper:
f'<span> <a class="forhover" href="{furl}">Drive Link</a></span>'
if INDEX_URLS[index] is not None:
if isRecur:
url_path = "/".join([rquote(n, safe='') for n in self.__get_recursive_list(file, parent_id)])
url_path = "/".join([rquote(n, safe='') for n in self.__get_recursive_list(file, PARENT_ID)])
else:
url_path = rquote(f'{file.get("name")}', safe='')
url = f'{INDEX_URLS[index]}/{url_path}/'
@ -617,7 +617,7 @@ class GoogleDriveHelper:
f'<span> <a class="forhover" href="{furl}">Drive Link</a></span>'
if INDEX_URLS[index] is not None:
if isRecur:
url_path = "/".join(rquote(n, safe='') for n in self.__get_recursive_list(file, parent_id))
url_path = "/".join(rquote(n, safe='') for n in self.__get_recursive_list(file, PARENT_ID))
else:
url_path = rquote(f'{file.get("name")}')
url = f'{INDEX_URLS[index]}/{url_path}'

View File

@ -1,7 +1,7 @@
from telegram.ext import CommandHandler, CallbackQueryHandler
from os import remove, path as ospath
from bot import aria2, BASE_URL, download_dict, dispatcher, download_dict_lock, SUDO_USERS, OWNER_ID
from bot import aria2, BASE_URL, download_dict, dispatcher, download_dict_lock, SUDO_USERS, OWNER_ID, LOGGER
from bot.helper.telegram_helper.bot_commands import BotCommands
from bot.helper.telegram_helper.filters import CustomFilters
from bot.helper.telegram_helper.message_utils import sendMessage, sendMarkup, sendStatusMessage
@ -50,7 +50,10 @@ def select(update, context):
client.torrents_pause(torrent_hashes=id_)
else:
id_ = dl.gid()
aria2.client.force_pause(id_)
try:
aria2.client.force_pause(id_)
except Exception as e:
LOGGER.error(f"{e} Error in pause, this mostly happens after abuse aria2")
listener.select = True
except:
sendMessage("This is not a bittorrent task!", context.bot, update.message)
@ -105,7 +108,10 @@ def get_confirm(update, context):
remove(f['path'])
except:
pass
aria2.client.unpause(id_)
try:
aria2.client.unpause(id_)
except Exception as e:
LOGGER.error(f"{e} Error in resume, this mostly happens after abuse aria2. Try to use select cmd again!")
sendStatusMessage(listener.message, listener.bot)
query.message.delete()

View File

@ -2,15 +2,15 @@
_____REMOVE_THIS_LINE_____=True
# REQUIRED CONFIG
BOT_TOKEN = ""
GDRIVE_FOLDER_ID = ""
OWNER_ID =
DOWNLOAD_DIR = "/usr/src/app/downloads"
DOWNLOAD_STATUS_UPDATE_INTERVAL = 10
AUTO_DELETE_MESSAGE_DURATION = 20
TELEGRAM_API =
OWNER_ID = ""
TELEGRAM_API = ""
TELEGRAM_HASH = ""
# OPTIONAL CONFIG
GDRIVE_FOLDER_ID = ""
IS_TEAM_DRIVE = ""
DOWNLOAD_DIR = "/usr/src/app/downloads/"
DOWNLOAD_STATUS_UPDATE_INTERVAL = "10"
AUTO_DELETE_MESSAGE_DURATION = "30"
DATABASE_URL = ""
AUTHORIZED_CHATS = ""
SUDO_USERS = ""