WindyMadman/mirror-leech-telegram-bot
1
0
Fork 0
mirror of https://github.com/anasty17/mirror-leech-telegram-bot.git synced 2025-01-05 10:36:52 +08:00
Code Issues Packages Projects Releases Wiki Activity

Improve jdownloader

Browse Source 
- Improve queue
- handle invalid directory in jdownloader incase file name is too long
- Otherr minor fixes

Signed-off-by: anasty17 <e.anastayyar@gmail.com>
This commit is contained in:
anasty17 2024-10-23 22:53:25 +03:00
parent 862831a68a
commit 010fb337a7
No known key found for this signature in database
GPG Key ID: EAAC8A2DF2861DE4
22 changed files with 196 additions and 703 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
bot/__main__.py
View File

@ -264,7 +264,7 @@ async def main():
if config_dict["DATABASE_URL"]:
await database.db_load()
await gather(
jdownloader.initiate(),
jdownloader.boot(),
sync_to_async(clean_all),
bot_settings.initiate_search_tools(),
restart_notification(),

6
bot/helper/common.py
View File

@ -1053,6 +1053,9 @@ class TaskConfig:
else:
res = ""
name = sub(rf"{pattern}", res, name, flags=I if sen else 0)
if len(name.encode()) > 255:
LOGGER.error(f"Substitute: {name} is too long")
return dl_path
new_path = ospath.join(up_dir, name)
await move(dl_path, new_path)
return new_path
@ -1074,5 +1077,8 @@ class TaskConfig:
else:
res = ""
file_ = sub(rf"{pattern}", res, file_, flags=I if sen else 0)
if len(file_.encode()) > 255:
LOGGER.error(f"Substitute: {file_} is too long")
continue
await move(f_path, ospath.join(dirpath, file_))
return dl_path

8
bot/helper/ext_utils/bot_utils.py
View File

@ -171,14 +171,6 @@ def update_user_ldata(id_, key, value):
user_data[id_][key] = value
async def retry_function(func, *args, **kwargs):
try:
return await func(*args, **kwargs)
except:
await sleep(0.2)
return await retry_function(func, *args, **kwargs)
async def cmd_exec(cmd, shell=False):
if shell:
proc = await create_subprocess_shell(cmd, stdout=PIPE, stderr=PIPE)

114
bot/helper/ext_utils/jdownloader_booter.py
View File

@ -2,19 +2,11 @@ from aiofiles.os import path, makedirs, listdir, rename
from aioshutil import rmtree
from json import dump
from random import randint
from asyncio import sleep, wait_for
from re import match
from bot import config_dict, LOGGER, jd_lock, bot_name
from .bot_utils import cmd_exec, new_task, retry_function
from bot import config_dict, LOGGER, bot_name
from .bot_utils import cmd_exec, new_task
from myjd import MyJdApi
from myjd.exception import (
MYJDException,
MYJDAuthFailedException,
MYJDEmailForbiddenException,
MYJDEmailInvalidException,
MYJDErrorEmailNotConfirmedException,
)
class JDownloader(MyJdApi):
@ -23,23 +15,16 @@ class JDownloader(MyJdApi):
self._username = ""
self._password = ""
self._device_name = ""
self.is_connected = False
self.error = "JDownloader Credentials not provided!"
self.device = None
self.set_app_key("mltb")
@new_task
async def initiate(self):
self.device = None
async with jd_lock:
is_connected = await self.jdconnect()
if is_connected:
await self.boot()
await self.connectToDevice()
@new_task
async def boot(self):
await cmd_exec(["pkill", "-9", "-f", "java"])
self.device = None
if not config_dict["JD_EMAIL"] or not config_dict["JD_PASS"]:
self.is_connected = False
self.error = "JDownloader Credentials not provided!"
return
self.error = "Connecting... Try agin after couple of seconds"
self._device_name = f"{randint(0, 1000)}@{bot_name}"
if await path.exists("/JDownloader/logs"):
@ -56,6 +41,20 @@ class JDownloader(MyJdApi):
"devicename": f"{self._device_name}",
"email": config_dict["JD_EMAIL"],
}
remote_data = {
"localapiserverheaderaccesscontrollalloworigin": "",
"deprecatedapiport": 3128,
"localapiserverheaderxcontenttypeoptions": "nosniff",
"localapiserverheaderxframeoptions": "DENY",
"externinterfaceenabled": True,
"deprecatedapilocalhostonly": True,
"localapiserverheaderreferrerpolicy": "no-referrer",
"deprecatedapienabled": True,
"localapiserverheadercontentsecuritypolicy": "default-src 'self'",
"jdanywhereapienabled": True,
"externinterfacelocalhostonly": False,
"localapiserverheaderxxssprotection": "1; mode=block",
}
await makedirs("/JDownloader/cfg", exist_ok=True)
with open(
"/JDownloader/cfg/org.jdownloader.api.myjdownloader.MyJDownloaderSettings.json",
@ -63,6 +62,12 @@ class JDownloader(MyJdApi):
) as sf:
sf.truncate(0)
dump(jdata, sf)
with open(
"/JDownloader/cfg/org.jdownloader.api.RemoteAPIConfig.json",
"w",
) as rf:
rf.truncate(0)
dump(remote_data, rf)
if not await path.exists("/JDownloader/JDownloader.jar"):
pattern = r"JDownloader\.jar\.backup.\d$"
for filename in await listdir("/JDownloader"):
@ -74,72 +79,11 @@ class JDownloader(MyJdApi):
await rmtree("/JDownloader/update")
await rmtree("/JDownloader/tmp")
cmd = "java -Dsun.jnu.encoding=UTF-8 -Dfile.encoding=UTF-8 -Djava.awt.headless=true -jar /JDownloader/JDownloader.jar"
self.is_connected = True
_, __, code = await cmd_exec(cmd, shell=True)
self.is_connected = False
if code != -9:
await self.boot()
async def jdconnect(self):
if not config_dict["JD_EMAIL"] or not config_dict["JD_PASS"]:
return False
try:
await self.connect(config_dict["JD_EMAIL"], config_dict["JD_PASS"])
LOGGER.info("MYJDownloader is connected!")
return True
except (
MYJDAuthFailedException,
MYJDEmailForbiddenException,
MYJDEmailInvalidException,
MYJDErrorEmailNotConfirmedException,
) as err:
self.error = f"{err}".strip()
LOGGER.error(f"Failed to connect with jdownloader! ERROR: {self.error}")
self.device = None
return False
except MYJDException as e:
self.error = f"{e}".strip()
LOGGER.error(
f"Failed to connect with jdownloader! Retrying... ERROR: {self.error}"
)
return await self.jdconnect()
async def connectToDevice(self):
self.error = "Connecting to device..."
await sleep(0.5)
while True:
self.device = None
if not config_dict["JD_EMAIL"] or not config_dict["JD_PASS"]:
self.error = "JDownloader Credentials not provided!"
await cmd_exec(["pkill", "-9", "-f", "java"])
return False
try:
await self.update_devices()
if not (devices := self.list_devices()):
continue
for device in devices:
if self._device_name == device["name"]:
self.device = self.get_device(f"{self._device_name}")
break
else:
continue
except:
continue
break
await self.device.enable_direct_connection()
self.error = ""
LOGGER.info("JDownloader Device have been Connected!")
return True
async def check_jdownloader_state(self):
try:
await wait_for(retry_function(self.device.jd.version), timeout=10)
except:
is_connected = await self.jdconnect()
if not is_connected:
raise MYJDException(self.error)
await self.boot()
isDeviceConnected = await self.connectToDevice()
if not isDeviceConnected:
raise MYJDException(self.error)
jdownloader = JDownloader()

6
bot/helper/ext_utils/task_manager.py
View File

@ -1,4 +1,4 @@
from asyncio import Event, sleep
from asyncio import Event
from bot import (
config_dict,
@ -95,13 +95,13 @@ async def check_running_tasks(listener, state="dl"):
async def start_dl_from_queued(mid: int):
queued_dl[mid].set()
del queued_dl[mid]
await sleep(0.7)
non_queued_dl.add(mid)
async def start_up_from_queued(mid: int):
queued_up[mid].set()
del queued_up[mid]
await sleep(0.7)
non_queued_up.add(mid)
async def start_from_queued():

19
bot/helper/listeners/jdownloader_listener.py
View File

@ -1,7 +1,7 @@
from asyncio import sleep
from bot import intervals, jd_lock, jd_downloads
from ..ext_utils.bot_utils import new_task, retry_function
from ..ext_utils.bot_utils import new_task
from ..ext_utils.jdownloader_booter import jdownloader
from ..ext_utils.status_utils import get_task_by_gid
@ -10,9 +10,8 @@ from ..ext_utils.status_utils import get_task_by_gid
async def remove_download(gid):
if intervals["stopAll"]:
return
await retry_function(
jdownloader.device.downloads.remove_links,
package_ids=jd_downloads[gid]["ids"],
await jdownloader.device.downloads.remove_links(
package_ids=jd_downloads[gid]["ids"]
)
if task := await get_task_by_gid(gid):
await task.listener.on_download_error("Download removed manually!")
@ -25,8 +24,7 @@ async def _on_download_complete(gid):
if task := await get_task_by_gid(gid):
if task.listener.select:
async with jd_lock:
await retry_function(
jdownloader.device.downloads.cleanup,
await jdownloader.device.downloads.cleanup(
"DELETE_DISABLED",
"REMOVE_LINKS_AND_DELETE_FILES",
"SELECTED",
@ -37,8 +35,7 @@ async def _on_download_complete(gid):
return
async with jd_lock:
if gid in jd_downloads:
await retry_function(
jdownloader.device.downloads.remove_links,
await jdownloader.device.downloads.remove_links(
package_ids=jd_downloads[gid]["ids"],
)
del jd_downloads[gid]
@ -52,10 +49,6 @@ async def _jd_listener():
if len(jd_downloads) == 0:
intervals["jd"] = ""
break
try:
await jdownloader.check_jdownloader_state()
except:
continue
try:
packages = await jdownloader.device.downloads.query_packages(
[{"finished": True, "saveTo": True}]
@ -64,8 +57,6 @@ async def _jd_listener():
continue
all_packages = {pack["uuid"]: pack for pack in packages}
if not all_packages:
continue
for d_gid, d_dict in list(jd_downloads.items()):
if d_dict["status"] == "down":
for index, pid in enumerate(d_dict["ids"]):

2
bot/helper/listeners/task_listener.py
View File

@ -228,8 +228,6 @@ class TaskListener(TaskConfig):
await event.wait()
if self.is_cancelled:
return
async with queue_dict_lock:
non_queued_up.add(self.mid)
LOGGER.info(f"Start from Queued/Upload: {self.name}")
self.size = await get_path_size(up_dir)

4
bot/helper/mirror_leech_utils/download_utils/aria2_download.py
View File

@ -8,8 +8,6 @@ from bot import (
config_dict,
aria2_options,
aria2c_global,
non_queued_dl,
queue_dict_lock,
)
from ...ext_utils.bot_utils import bt_selection_buttons, sync_to_async
from ...ext_utils.task_manager import check_running_tasks
@ -83,8 +81,6 @@ async def add_aria2c_download(listener, dpath, header, ratio, seed_time):
await event.wait()
if listener.is_cancelled:
return
async with queue_dict_lock:
non_queued_dl.add(listener.mid)
async with task_dict_lock:
task = task_dict[listener.mid]
task.queued = False

4
bot/helper/mirror_leech_utils/download_utils/direct_downloader.py
View File

@ -6,8 +6,6 @@ from bot import (
aria2c_global,
task_dict,
task_dict_lock,
non_queued_dl,
queue_dict_lock,
)
from ...ext_utils.bot_utils import sync_to_async
from ...ext_utils.task_manager import check_running_tasks, stop_duplicate_check
@ -45,8 +43,6 @@ async def add_direct_download(listener, path):
await event.wait()
if listener.is_cancelled:
return
async with queue_dict_lock:
non_queued_dl.add(listener.mid)
a2c_opt = {**aria2_options}
[a2c_opt.pop(k) for k in aria2c_global if k in aria2_options]

4
bot/helper/mirror_leech_utils/download_utils/gd_download.py
View File

@ -1,6 +1,6 @@
from secrets import token_urlsafe
from bot import task_dict, task_dict_lock, LOGGER, non_queued_dl, queue_dict_lock
from bot import task_dict, task_dict_lock, LOGGER
from ...ext_utils.bot_utils import sync_to_async
from ...ext_utils.task_manager import check_running_tasks, stop_duplicate_check
from ...mirror_leech_utils.gdrive_utils.count import GoogleDriveCount
@ -38,8 +38,6 @@ async def add_gd_download(listener, path):
await event.wait()
if listener.is_cancelled:
return
async with queue_dict_lock:
non_queued_dl.add(listener.mid)
drive = GoogleDriveDownload(listener, path)
async with task_dict_lock:

161
bot/helper/mirror_leech_utils/download_utils/jd_download.py
View File

@ -13,12 +13,10 @@ from bot import (
task_dict,
task_dict_lock,
LOGGER,
non_queued_dl,
queue_dict_lock,
jd_lock,
jd_downloads,
)
from ...ext_utils.bot_utils import retry_function, new_task
from ...ext_utils.bot_utils import new_task
from ...ext_utils.jdownloader_booter import jdownloader
from ...ext_utils.task_manager import check_running_tasks, stop_duplicate_check
from ...listeners.jdownloader_listener import on_download_start
@ -86,14 +84,13 @@ class JDownloaderHelper:
async def get_online_packages(path, state="grabbing"):
if state == "grabbing":
queued_downloads = await retry_function(
jdownloader.device.linkgrabber.query_packages, [{"saveTo": True}]
queued_downloads = await jdownloader.device.linkgrabber.query_packages(
[{"saveTo": True}]
)
return [qd["uuid"] for qd in queued_downloads if qd["saveTo"].startswith(path)]
else:
download_packages = await retry_function(
jdownloader.device.downloads.query_packages,
[{"saveTo": True}],
download_packages = await jdownloader.device.downloads.query_packages(
[{"saveTo": True}]
)
return [dl["uuid"] for dl in download_packages if dl["saveTo"].startswith(path)]
@ -109,35 +106,35 @@ def trim_path(path):
return "/".join(trimmed_components)
async def get_jd_download_directory():
res = await jdownloader.device.config.get(
"org.jdownloader.settings.GeneralSettings", None, "DefaultDownloadFolder"
)
return f'/{res.strip("/")}/'
async def add_jd_download(listener, path):
try:
async with jd_lock:
if jdownloader.device is None:
if not jdownloader.is_connected:
raise MYJDException(jdownloader.error)
await jdownloader.check_jdownloader_state()
default_path = await get_jd_download_directory()
if not jd_downloads:
await retry_function(jdownloader.device.linkgrabber.clear_list)
if odl := await retry_function(
jdownloader.device.downloads.query_packages, [{}]
):
await jdownloader.device.linkgrabber.clear_list()
if odl := await jdownloader.device.downloads.query_packages([{}]):
odl_list = [od["uuid"] for od in odl]
await retry_function(
jdownloader.device.downloads.remove_links,
package_ids=odl_list,
await jdownloader.device.downloads.remove_links(
package_ids=odl_list
)
elif odl := await retry_function(
jdownloader.device.linkgrabber.query_packages, [{}]
):
elif odl := await jdownloader.device.linkgrabber.query_packages([{}]):
if odl_list := [
od["uuid"]
for od in odl
if od.get("saveTo", "").startswith("/root/Downloads/")
if od.get("saveTo", "").startswith(default_path)
]:
await retry_function(
jdownloader.device.linkgrabber.remove_links,
package_ids=odl_list,
await jdownloader.device.linkgrabber.remove_links(
package_ids=odl_list
)
gid = token_urlsafe(12)
@ -147,14 +144,11 @@ async def add_jd_download(listener, path):
async with aiopen(listener.link, "rb") as dlc:
content = await dlc.read()
content = b64encode(content)
await retry_function(
jdownloader.device.linkgrabber.add_container,
"DLC",
f";base64,{content.decode()}",
await jdownloader.device.linkgrabber.add_container(
"DLC", f";base64,{content.decode()}"
)
else:
await retry_function(
jdownloader.device.linkgrabber.add_links,
await jdownloader.device.linkgrabber.add_links(
[
{
"autoExtract": False,
@ -165,7 +159,7 @@ async def add_jd_download(listener, path):
)
await sleep(1)
while await retry_function(jdownloader.device.linkgrabber.is_collecting):
while await jdownloader.device.linkgrabber.is_collecting():
pass
start_time = time()
online_packages = []
@ -174,8 +168,7 @@ async def add_jd_download(listener, path):
name = ""
error = ""
while (time() - start_time) < 60:
queued_downloads = await retry_function(
jdownloader.device.linkgrabber.query_packages,
queued_downloads = await jdownloader.device.linkgrabber.query_packages(
[
{
"bytesTotal": True,
@ -189,8 +182,7 @@ async def add_jd_download(listener, path):
)
if not online_packages and corrupted_packages and error:
await retry_function(
jdownloader.device.linkgrabber.remove_links,
await jdownloader.device.linkgrabber.remove_links(
package_ids=corrupted_packages,
)
raise MYJDException(error)
@ -203,10 +195,8 @@ async def add_jd_download(listener, path):
continue
save_to = pack["saveTo"]
if not name:
if save_to.startswith("/root/Downloads/"):
name = save_to.replace("/root/Downloads/", "", 1).split(
"/", 1
)[0]
if save_to.startswith(default_path):
name = save_to.replace(default_path, "", 1).split("/", 1)[0]
else:
name = save_to.replace(f"{path}/", "", 1).split("/", 1)[0]
name = name[:255]
@ -220,19 +210,17 @@ async def add_jd_download(listener, path):
listener.size += pack.get("bytesTotal", 0)
online_packages.append(pack["uuid"])
if save_to.startswith("/root/Downloads/"):
if save_to.startswith(default_path):
save_to = trim_path(save_to)
await retry_function(
jdownloader.device.linkgrabber.set_download_directory,
save_to.replace("/root/Downloads", path, 1),
await jdownloader.device.linkgrabber.set_download_directory(
save_to.replace(default_path, f"{path}/", 1),
[pack["uuid"]],
)
if online_packages:
if listener.join and len(online_packages) > 1:
listener.name = listener.folder_name
await retry_function(
jdownloader.device.linkgrabber.move_to_new_package,
await jdownloader.device.linkgrabber.move_to_new_package(
listener.name,
f"{path}/{listener.name}",
package_ids=online_packages,
@ -246,8 +234,7 @@ async def add_jd_download(listener, path):
)
if corrupted_packages or online_packages:
packages_to_remove = corrupted_packages + online_packages
await retry_function(
jdownloader.device.linkgrabber.remove_links,
await jdownloader.device.linkgrabber.remove_links(
package_ids=packages_to_remove,
)
raise MYJDException(error)
@ -256,8 +243,7 @@ async def add_jd_download(listener, path):
corrupted_links = []
if remove_unknown:
links = await retry_function(
jdownloader.device.linkgrabber.query_links,
links = await jdownloader.device.linkgrabber.query_links(
[{"packageUUIDs": online_packages, "availability": True}],
)
corrupted_links = [
@ -266,8 +252,7 @@ async def add_jd_download(listener, path):
if link["availability"].lower() != "online"
]
if corrupted_packages or corrupted_links:
await retry_function(
jdownloader.device.linkgrabber.remove_links,
await jdownloader.device.linkgrabber.remove_links(
corrupted_links,
corrupted_packages,
)
@ -276,8 +261,8 @@ async def add_jd_download(listener, path):
msg, button = await stop_duplicate_check(listener)
if msg:
await retry_function(
jdownloader.device.linkgrabber.remove_links, package_ids=online_packages
await jdownloader.device.linkgrabber.remove_links(
package_ids=online_packages
)
await listener.on_download_error(msg, button)
async with jd_lock:
@ -286,9 +271,8 @@ async def add_jd_download(listener, path):
if listener.select:
if not await JDownloaderHelper(listener).wait_for_configurations():
await retry_function(
jdownloader.device.linkgrabber.remove_links,
package_ids=online_packages,
await jdownloader.device.linkgrabber.remove_links(
package_ids=online_packages
)
await listener.remove_from_same_dir()
async with jd_lock:
@ -297,7 +281,9 @@ async def add_jd_download(listener, path):
else:
online_packages = await get_online_packages(path)
if not online_packages:
raise MYJDException("This Download have been removed manually!")
raise MYJDException(
"Select: This Download have been removed manually!"
)
async with jd_lock:
jd_downloads[gid]["ids"] = online_packages
@ -312,45 +298,41 @@ async def add_jd_download(listener, path):
await event.wait()
if listener.is_cancelled:
return
async with queue_dict_lock:
non_queued_dl.add(listener.mid)
await jdownloader.check_jdownloader_state()
online_packages = await get_online_packages(path)
if not online_packages:
raise MYJDException("This Download have been removed manually!")
raise MYJDException("Queue: This Download have been removed manually!")
async with jd_lock:
jd_downloads[gid]["ids"] = online_packages
await retry_function(
jdownloader.device.linkgrabber.move_to_downloadlist,
package_ids=online_packages,
await jdownloader.device.linkgrabber.move_to_downloadlist(
package_ids=online_packages
)
await sleep(1)
await sleep(0.5)
online_packages = await get_online_packages(path, "down")
if not online_packages:
online_packages = await get_online_packages(path)
if not online_packages:
raise MYJDException("This Download have been removed manually!")
await retry_function(
jdownloader.device.linkgrabber.move_to_downloadlist,
package_ids=online_packages,
raise MYJDException(
"Linkgrabber: This Download have been removed manually!"
)
await jdownloader.device.linkgrabber.move_to_downloadlist(
package_ids=online_packages
)
await sleep(1)
await sleep(0.5)
online_packages = await get_online_packages(path, "down")
if not online_packages:
raise MYJDException("This Download have been removed manually!")
raise MYJDException(
"Download List: This Download have been removed manually!"
)
async with jd_lock:
jd_downloads[gid]["status"] = "down"
jd_downloads[gid]["ids"] = online_packages
await retry_function(
jdownloader.device.downloads.force_download,
package_ids=online_packages,
)
await jdownloader.device.downloads.force_download(package_ids=online_packages)
async with task_dict_lock:
task_dict[listener.mid] = JDownloaderStatus(listener, gid)
@ -371,3 +353,30 @@ async def add_jd_download(listener, path):
finally:
if await aiopath.exists(listener.link):
await remove(listener.link)
await sleep(2)
links = await jdownloader.device.downloads.query_links(
[
{
"packageUUIDs": online_packages,
"status": True,
}
],
)
links_to_remove = []
force_download = False
for dlink in links:
if dlink["status"] == "Invalid download directory":
force_download = True
new_name, ext = dlink["name"].rsplit(".", 1)
new_name = new_name[: 250 - len(f".{ext}".encode())]
new_name = f"{new_name}.{ext}"
await jdownloader.device.downloads.rename_link(dlink["uuid"], new_name)
elif dlink["status"] == "HLS stream broken?":
links_to_remove.append(dlink["uuid"])
if links_to_remove:
await jdownloader.device.downloads.remove_links(links_to_remove)
if force_download:
await jdownloader.device.downloads.force_download(package_ids=online_packages)

4
bot/helper/mirror_leech_utils/download_utils/nzb_downloader.py
View File

@ -8,8 +8,6 @@ from bot import (
sabnzbd_client,
LOGGER,
config_dict,
non_queued_dl,
queue_dict_lock,
)
from ...ext_utils.bot_utils import bt_selection_buttons
from ...ext_utils.task_manager import check_running_tasks
@ -153,8 +151,6 @@ async def add_nzb(listener, path):
await event.wait()
if listener.is_cancelled:
return
async with queue_dict_lock:
non_queued_dl.add(listener.mid)
async with task_dict_lock:
task_dict[listener.mid].queued = False

27
bot/helper/mirror_leech_utils/download_utils/qbit_download.py
View File

@ -8,8 +8,6 @@ from bot import (
qbittorrent_client,
LOGGER,
config_dict,
non_queued_dl,
queue_dict_lock,
)
from ...ext_utils.bot_utils import bt_selection_buttons, sync_to_async
from ...ext_utils.task_manager import check_running_tasks
@ -65,6 +63,8 @@ async def add_qb_torrent(listener, path, ratio, seed_time):
start_time = time()
if len(tor_info) == 0:
while (time() - start_time) <= 60:
if add_to_queue and event.is_set():
add_to_queue = False
tor_info = await sync_to_async(
qbittorrent_client.torrents_info, tag=f"{listener.mid}"
)
@ -130,21 +130,20 @@ async def add_qb_torrent(listener, path, ratio, seed_time):
elif listener.multi <= 1:
await send_status_message(listener.message)
if add_to_queue:
await event.wait()
if listener.is_cancelled:
return
async with queue_dict_lock:
non_queued_dl.add(listener.mid)
async with task_dict_lock:
task_dict[listener.mid].queued = False
if event is not None:
if not event.is_set():
await event.wait()
if listener.is_cancelled:
return
async with task_dict_lock:
task_dict[listener.mid].queued = False
LOGGER.info(
f"Start Queued Download from Qbittorrent: {tor_info.name} - Hash: {ext_hash}"
)
await sync_to_async(
qbittorrent_client.torrents_resume, torrent_hashes=ext_hash
)
LOGGER.info(
f"Start Queued Download from Qbittorrent: {tor_info.name} - Hash: {ext_hash}"
)
except Exception as e:
await listener.on_download_error(f"{e}")
finally:

4
bot/helper/mirror_leech_utils/download_utils/rclone_download.py
View File

@ -3,7 +3,7 @@ from json import loads
from secrets import token_urlsafe
from aiofiles.os import remove
from bot import task_dict, task_dict_lock, queue_dict_lock, non_queued_dl, LOGGER
from bot import task_dict, task_dict_lock, LOGGER
from ...ext_utils.bot_utils import cmd_exec
from ...ext_utils.task_manager import check_running_tasks, stop_duplicate_check
from ...mirror_leech_utils.rclone_utils.transfer import RcloneTransferHelper
@ -115,8 +115,6 @@ async def add_rclone_download(listener, path):
await event.wait()
if listener.is_cancelled:
return
async with queue_dict_lock:
non_queued_dl.add(listener.mid)
RCTransfer = RcloneTransferHelper(listener)
async with task_dict_lock:

4
bot/helper/mirror_leech_utils/download_utils/telegram_download.py
View File

@ -6,8 +6,6 @@ from bot import (
LOGGER,
task_dict,
task_dict_lock,
non_queued_dl,
queue_dict_lock,
bot,
user,
)
@ -151,8 +149,6 @@ class TelegramDownloadHelper:
await event.wait()
if self._listener.is_cancelled:
return
async with queue_dict_lock:
non_queued_dl.add(self._listener.mid)
await self._on_download_start(gid, add_to_queue)
await self._download(message, path)

4
bot/helper/mirror_leech_utils/download_utils/yt_dlp_download.py
View File

@ -4,7 +4,7 @@ from re import search as re_search
from secrets import token_urlsafe
from yt_dlp import YoutubeDL, DownloadError
from bot import task_dict_lock, task_dict, non_queued_dl, queue_dict_lock
from bot import task_dict_lock, task_dict
from ...ext_utils.bot_utils import sync_to_async, async_to_sync
from ...ext_utils.task_manager import check_running_tasks, stop_duplicate_check
from ...mirror_leech_utils.status_utils.queue_status import QueueStatus
@ -325,8 +325,6 @@ class YoutubeDLHelper:
await event.wait()
if self._listener.is_cancelled:
return
async with queue_dict_lock:
non_queued_dl.add(self._listener.mid)
LOGGER.info(f"Start Queued Download from YT_DLP: {self._listener.name}")
await self._on_download_start(True)

7
bot/helper/mirror_leech_utils/status_utils/jdownloader_status.py
View File

@ -1,5 +1,5 @@
from bot import LOGGER, jd_lock, jd_downloads
from ...ext_utils.bot_utils import retry_function, async_to_sync
from ...ext_utils.bot_utils import async_to_sync
from ...ext_utils.jdownloader_booter import jdownloader
from ...ext_utils.status_utils import (
MirrorStatus,
@ -106,10 +106,7 @@ class JDownloaderStatus:
async def cancel_task(self):
self.listener.is_cancelled = True
LOGGER.info(f"Cancelling Download: {self.name()}")
await retry_function(
jdownloader.device.downloads.remove_links,
package_ids=jd_downloads[self._gid]["ids"],
)
await jdownloader.device.downloads.remove_links(package_ids=jd_downloads[self._gid]["ids"])
async with jd_lock:
del jd_downloads[self._gid]
await self.listener.on_download_error("Download cancelled by user!")

27
bot/modules/bot_settings.py
View File

@ -6,7 +6,6 @@ from asyncio import (
create_subprocess_shell,
sleep,
gather,
wait_for,
)
from dotenv import load_dotenv
from functools import partial
@ -35,7 +34,6 @@ from bot import (
qbittorrent_client,
sabnzbd_client,
bot,
jd_downloads,
nzb_options,
get_nzb_options,
get_qb_options,
@ -44,7 +42,6 @@ from bot import (
from ..helper.ext_utils.bot_utils import (
SetInterval,
sync_to_async,
retry_function,
new_task,
)
from ..helper.ext_utils.db_handler import database
@ -334,7 +331,7 @@ async def edit_variable(_, message, pre_message, key):
]:
await rclone_serve_booter()
elif key in ["JD_EMAIL", "JD_PASS"]:
await jdownloader.initiate()
await jdownloader.boot()
elif key == "RSS_DELAY":
add_job()
elif key == "USET_SERVERS":
@ -444,27 +441,9 @@ async def edit_nzb_server(_, message, pre_message, key, index=0):
async def sync_jdownloader():
async with jd_lock:
if not config_dict["DATABASE_URL"] or jdownloader.device is None:
if not config_dict["DATABASE_URL"] or not jdownloader.is_connected:
return
try:
await wait_for(retry_function(jdownloader.update_devices), timeout=10)
except:
is_connected = await jdownloader.jdconnect()
if not is_connected:
LOGGER.error(jdownloader.error)
return
isDeviceConnected = await jdownloader.connectToDevice()
if not isDeviceConnected:
LOGGER.error(jdownloader.error)
return
await jdownloader.device.system.exit_jd()
is_connected = await jdownloader.jdconnect()
if not is_connected:
LOGGER.error(jdownloader.error)
return
isDeviceConnected = await jdownloader.connectToDevice()
if not isDeviceConnected:
LOGGER.error(jdownloader.error)
if await aiopath.exists("cfg.zip"):
await remove("cfg.zip")
await (
@ -664,8 +643,6 @@ async def edit_bot_settings(client, query):
elif data[2] == "INCOMPLETE_TASK_NOTIFIER" and config_dict["DATABASE_URL"]:
await database.trunc_table("tasks")
elif data[2] in ["JD_EMAIL", "JD_PASS"]:
jdownloader.device = None
jdownloader.error = "JDownloader Credentials not provided!"
await create_subprocess_exec("pkill", "-9", "-f", "java")
elif data[2] == "USENET_SERVERS":
for s in config_dict["USENET_SERVERS"]:

4
config_sample.env
View File

@ -38,7 +38,7 @@ RCLONE_SERVE_PASS = ""
JD_EMAIL = ""
JD_PASS = ""
# Sabnzbd
USENET_SERVERS = [{'name': 'main', 'host': '', 'port': 5126, 'timeout': 60, 'username': '', 'password': '', 'connections': 8, 'ssl': 1, 'ssl_verify': 2, 'ssl_ciphers': '', 'enable': 1, 'required': 0, 'optional': 0, 'retention': 0, 'send_group': 0, 'priority': 0}]
USENET_SERVERS = "[{'name': 'main', 'host': '', 'port': 5126, 'timeout': 60, 'username': '', 'password': '', 'connections': 8, 'ssl': 1, 'ssl_verify': 2, 'ssl_ciphers': '', 'enable': 1, 'required': 0, 'optional': 0, 'retention': 0, 'send_group': 0, 'priority': 0}]"
# Update
UPSTREAM_REPO = ""
UPSTREAM_BRANCH = ""
@ -83,4 +83,4 @@ SEARCH_PLUGINS = '["https://raw.githubusercontent.com/qbittorrent/search-plugins
"https://raw.githubusercontent.com/v1k45/1337x-qBittorrent-search-plugin/master/leetx.py",
"https://raw.githubusercontent.com/nindogo/qbtSearchScripts/master/magnetdl.py",
"https://raw.githubusercontent.com/msagca/qbittorrent_plugins/main/uniondht.py",
"https://raw.githubusercontent.com/khensolomon/leyts/master/yts.py"]'
"https://raw.githubusercontent.com/khensolomon/leyts/master/yts.py"]'

485
myjd/myjdapi.py
View File

@ -1,34 +1,15 @@
# -*- encoding: utf-8 -*-
from Crypto.Cipher import AES
from base64 import b64encode, b64decode
from hashlib import sha256
from hmac import new
from json import dumps, loads, JSONDecodeError
from httpx import AsyncClient, RequestError
from httpx import AsyncHTTPTransport
from time import time
from urllib.parse import quote
from functools import wraps
from .exception import (
MYJDApiException,
MYJDConnectionException,
MYJDDecodeException,
MYJDDeviceNotFoundException,
)
BS = 16
def PAD(s):
return s + ((BS - len(s) % BS) * chr(BS - len(s) % BS)).encode()
def UNPAD(s):
return s[: -s[-1]]
class System:
def __init__(self, device):
self.device = device
@ -254,7 +235,7 @@ class Linkgrabber:
self.url = "/linkgrabberv2"
async def clear_list(self):
return await self.device.action(f"{self.url}/clearList", http_action="POST")
return await self.device.action(f"{self.url}/clearList")
async def move_to_downloadlist(self, link_ids=None, package_ids=None):
"""
@ -679,9 +660,13 @@ class Downloads:
async def move_to_new_package(
self, link_ids, package_ids, new_pkg_name, download_path
):
params = link_ids, package_ids, new_pkg_name, download_path
params = [link_ids, package_ids, new_pkg_name, download_path]
return await self.device.action(f"{self.url}/movetoNewPackage", params)
async def rename_link(self, link_id: list, new_name: str):
params = [link_id, new_name]
return await self.device.action(f"{self.url}/renameLink", params)
class Captcha:
@ -701,15 +686,12 @@ class Captcha:
class Jddevice:
def __init__(self, jd, device_dict):
def __init__(self, jd):
"""This functions initializates the device instance.
It uses the provided dictionary to create the device.
:param device_dict: Device dictionary
"""
self.name = device_dict["name"]
self.device_id = device_dict["id"]
self.device_type = device_dict["type"]
self.myjd = jd
self.config = Config(self)
self.linkgrabber = Linkgrabber(self)
@ -719,105 +701,22 @@ class Jddevice:
self.extensions = Extension(self)
self.jd = Jd(self)
self.system = System(self)
self.__direct_connection_info = None
self.__direct_connection_enabled = False
self.__direct_connection_cooldown = 0
self.__direct_connection_consecutive_failures = 0
async def __refresh_direct_connections(self):
response = await self.myjd.request_api(
"/device/getDirectConnectionInfos", "POST", None, self.__action_url()
)
if (
response is not None
and "data" in response
and "infos" in response["data"]
and len(response["data"]["infos"]) != 0
):
self.__update_direct_connections(response["data"]["infos"])
def __update_direct_connections(self, direct_info):
"""
Updates the direct_connections info keeping the order.
"""
tmp = []
if self.__direct_connection_info is None:
tmp.extend({"conn": conn, "cooldown": 0} for conn in direct_info)
self.__direct_connection_info = tmp
return
# We remove old connections not available anymore.
for i in self.__direct_connection_info:
if i["conn"] not in direct_info:
tmp.remove(i)
else:
direct_info.remove(i["conn"])
# We add new connections
tmp.extend({"conn": conn, "cooldown": 0} for conn in direct_info)
self.__direct_connection_info = tmp
async def ping(self):
return await self.action("/device/ping")
async def enable_direct_connection(self):
self.__direct_connection_enabled = True
await self.__refresh_direct_connections()
def disable_direct_connection(self):
self.__direct_connection_enabled = False
self.__direct_connection_info = None
async def action(self, path, params=(), http_action="POST"):
action_url = self.__action_url()
if (
self.__direct_connection_enabled
and self.__direct_connection_info is not None
and time() >= self.__direct_connection_cooldown
):
return await self.__direct_connect(path, http_action, params, action_url)
response = await self.myjd.request_api(path, http_action, params, action_url)
async def action(self, path, params=()):
response = await self.myjd.request_api(path, params)
if response is None:
raise (MYJDConnectionException("No connection established\n"))
if (
self.__direct_connection_enabled
and time() >= self.__direct_connection_cooldown
):
await self.__refresh_direct_connections()
return response["data"]
async def __direct_connect(self, path, http_action, params, action_url):
for conn in self.__direct_connection_info:
if time() > conn["cooldown"]:
connection = conn["conn"]
api = "http://" + connection["ip"] + ":" + str(connection["port"])
response = await self.myjd.request_api(
path, http_action, params, action_url, api
)
if response is not None:
self.__direct_connection_info.remove(conn)
self.__direct_connection_info.insert(0, conn)
self.__direct_connection_consecutive_failures = 0
return response["data"]
else:
conn["cooldown"] = time() + 60
self.__direct_connection_consecutive_failures += 1
self.__direct_connection_cooldown = time() + (
60 * self.__direct_connection_consecutive_failures
)
response = await self.myjd.request_api(path, http_action, params, action_url)
if response is None:
raise (MYJDConnectionException("No connection established\n"))
await self.__refresh_direct_connections()
return response["data"]
def __action_url(self):
return f"/t_{self.myjd.get_session_token()}_{self.device_id}"
class clientSession(AsyncClient):
@wraps(AsyncClient.request)
async def request(self, method: str, url: str, **kwargs):
kwargs.setdefault("timeout", 1.5)
kwargs.setdefault("timeout", 3)
kwargs.setdefault("follow_redirects", True)
return await super().request(method, url, **kwargs)
@ -825,19 +724,9 @@ class clientSession(AsyncClient):
class MyJdApi:
def __init__(self):
self.__request_id = int(time() * 1000)
self.__api_url = "https://api.jdownloader.org"
self.__app_key = "mltb"
self.__api_version = 1
self.__devices = None
self.__login_secret = None
self.__device_secret = None
self.__session_token = None
self.__regain_token = None
self.__server_encryption_token = None
self.__device_encryption_token = None
self.__connected = False
self.__api_url = "http://127.0.0.1:3128"
self._http_session = None
self.device = Jddevice(self)
def _session(self):
if self._http_session is not None:
@ -851,337 +740,51 @@ class MyJdApi:
return self._http_session
def get_session_token(self):
return self.__session_token
def is_connected(self):
"""
Indicates if there is a connection established.
"""
return self.__connected
def set_app_key(self, app_key):
"""
Sets the APP Key.
"""
self.__app_key = app_key
def __secret_create(self, email, password, domain):
"""
Calculates the login_secret and device_secret
:param email: My.Jdownloader User email
:param password: My.Jdownloader User password
:param domain: The domain , if is for Server (login_secret) or Device (device_secret)
:return: secret hash
"""
secret_hash = sha256()
secret_hash.update(
email.lower().encode("utf-8")
+ password.encode("utf-8")
+ domain.lower().encode("utf-8")
)
return secret_hash.digest()
def __update_encryption_tokens(self):
"""
Updates the server_encryption_token and device_encryption_token
"""
if self.__server_encryption_token is None:
old_token = self.__login_secret
else:
old_token = self.__server_encryption_token
new_token = sha256()
new_token.update(old_token + bytearray.fromhex(self.__session_token))
self.__server_encryption_token = new_token.digest()
new_token = sha256()
new_token.update(self.__device_secret + bytearray.fromhex(self.__session_token))
self.__device_encryption_token = new_token.digest()
def __signature_create(self, key, data):
"""
Calculates the signature for the data given a key.
:param key:
:param data:
"""
signature = new(key, data.encode("utf-8"), sha256)
return signature.hexdigest()
def __decrypt(self, secret_token, data):
"""
Decrypts the data from the server using the provided token
:param secret_token:
:param data:
"""
init_vector = secret_token[: len(secret_token) // 2]
key = secret_token[len(secret_token) // 2 :]
decryptor = AES.new(key, AES.MODE_CBC, init_vector)
return UNPAD(decryptor.decrypt(b64decode(data)))
def __encrypt(self, secret_token, data):
"""
Encrypts the data from the server using the provided token
:param secret_token:
:param data:
"""
data = PAD(data.encode("utf-8"))
init_vector = secret_token[: len(secret_token) // 2]
key = secret_token[len(secret_token) // 2 :]
encryptor = AES.new(key, AES.MODE_CBC, init_vector)
encrypted_data = b64encode(encryptor.encrypt(data))
return encrypted_data.decode("utf-8")
def update_request_id(self):
"""
Updates Request_Id
"""
self.__request_id = int(time())
async def connect(self, email, password):
"""Establish connection to api
:param email: My.Jdownloader User email
:param password: My.Jdownloader User password
:returns: boolean -- True if succesful, False if there was any error.
"""
self.__clean_resources()
self.__login_secret = self.__secret_create(email, password, "server")
self.__device_secret = self.__secret_create(email, password, "device")
response = await self.request_api(
"/my/connect", "GET", [("email", email), ("appkey", self.__app_key)]
)
self.__connected = True
self.update_request_id()
self.__session_token = response["sessiontoken"]
self.__regain_token = response["regaintoken"]
self.__update_encryption_tokens()
return response
async def reconnect(self):
"""
Reestablish connection to API.
:returns: boolean -- True if successful, False if there was any error.
"""
response = await self.request_api(
"/my/reconnect",
"GET",
[
("sessiontoken", self.__session_token),
("regaintoken", self.__regain_token),
],
)
self.update_request_id()
self.__session_token = response["sessiontoken"]
self.__regain_token = response["regaintoken"]
self.__update_encryption_tokens()
return response
async def disconnect(self):
"""
Disconnects from API
:returns: boolean -- True if successful, False if there was any error.
"""
response = await self.request_api(
"/my/disconnect", "GET", [("sessiontoken", self.__session_token)]
)
self.__clean_resources()
if self._http_session is not None:
self._http_session = None
await self._http_session.aclose()
return response
def __clean_resources(self):
self.update_request_id()
self.__login_secret = None
self.__device_secret = None
self.__session_token = None
self.__regain_token = None
self.__server_encryption_token = None
self.__device_encryption_token = None
self.__devices = None
self.__connected = False
async def update_devices(self):
"""
Updates available devices. Use list_devices() to get the devices list.
:returns: boolean -- True if successful, False if there was any error.
"""
response = await self.request_api(
"/my/listdevices", "GET", [("sessiontoken", self.__session_token)]
)
self.update_request_id()
self.__devices = response["list"]
def list_devices(self):
"""
Returns available devices. Use getDevices() to update the devices list.
Each device in the list is a dictionary like this example:
{
'name': 'Device',
'id': 'af9d03a21ddb917492dc1af8a6427f11',
'type': 'jd'
}
:returns: list -- list of devices.
"""
return self.__devices
def get_device(self, device_name=None, device_id=None):
"""
Returns a jddevice instance of the device
:param deviceid:
"""
if not self.is_connected():
raise (MYJDConnectionException("No connection established\n"))
if device_id is not None:
for device in self.__devices:
if device["id"] == device_id:
return Jddevice(self, device)
elif device_name is not None:
for device in self.__devices:
if device["name"] == device_name:
return Jddevice(self, device)
raise (MYJDDeviceNotFoundException("Device not found\n"))
async def request_api(
self, path, http_method="GET", params=None, action=None, api=None
):
"""
Makes a request to the API to the 'path' using the 'http_method' with parameters,'params'.
Ex:
http_method=GET
params={"test":"test"}
post_params={"test2":"test2"}
action=True
This would make a request to "https://api.jdownloader.org"
"""
async def request_api(self, path, params=None):
session = self._session()
if not api:
api = self.__api_url
data = None
if not self.is_connected() and path != "/my/connect":
raise (MYJDConnectionException("No connection established\n"))
if http_method == "GET":
query = [f"{path}?"]
if params is not None:
for param in params:
if param[0] != "encryptedLoginSecret":
query += [f"{param[0]}={quote(param[1])}"]
else:
query += [f"&{param[0]}={param[1]}"]
query += [f"rid={str(self.__request_id)}"]
if self.__server_encryption_token is None:
query += [
"signature="
+ str(
self.__signature_create(
self.__login_secret, query[0] + "&".join(query[1:])
)
)
]
else:
query += [
"signature="
+ str(
self.__signature_create(
self.__server_encryption_token,
query[0] + "&".join(query[1:]),
)
)
]
query = query[0] + "&".join(query[1:])
res = await session.request(http_method, api + query)
encrypted_response = res.text
else:
params_request = []
if params is not None:
for param in params:
if isinstance(param, (str, list)):
params_request += [param]
elif isinstance(param, (dict, bool)):
params_request += [dumps(param)]
else:
params_request += [str(param)]
params_request = {
"apiVer": self.__api_version,
"url": path,
"params": params_request,
"rid": self.__request_id,
}
data = dumps(params_request)
# Removing quotes around null elements.
data = data.replace('"null"', "null")
data = data.replace("'null'", "null")
encrypted_data = self.__encrypt(self.__device_encryption_token, data)
request_url = api + action + path if action is not None else api + path
try:
res = await session.request(
http_method,
request_url,
headers={"Content-Type": "application/aesjson-jd; charset=utf-8"},
content=encrypted_data,
)
encrypted_response = res.text
except RequestError:
return None
# Prepare params_request based on the input params
params_request = params if params is not None else []
# Construct the request payload
params_request = {
"params": params_request,
}
data = dumps(params_request)
# Removing quotes around null elements.
data = data.replace('"null"', "null")
data = data.replace("'null'", "null")
request_url = self.__api_url + path
try:
res = await session.request(
"POST",
request_url,
headers={"Content-Type": "application/json; charset=utf-8"},
content=data,
)
response = res.text
except RequestError:
return None
if res.status_code != 200:
try:
error_msg = loads(encrypted_response)
except JSONDecodeError:
try:
error_msg = loads(
self.__decrypt(
self.__device_encryption_token, encrypted_response
)
)
except JSONDecodeError as exc:
raise MYJDDecodeException(
"Failed to decode response: {}", encrypted_response
) from exc
error_msg = loads(response)
except JSONDecodeError as exc:
raise MYJDDecodeException(
"Failed to decode response: {}", response
) from exc
msg = (
"\n\tSOURCE: "
+ error_msg["src"]
+ "\n\tTYPE: "
+ error_msg["type"]
+ "\n------\nREQUEST_URL: "
+ api
+ (path if http_method != "GET" else "")
+ self.__api_url
+ path
)
if http_method == "GET":
msg += query
msg += "\n"
if data is not None:
msg += "DATA:\n" + data
raise (
MYJDApiException.get_exception(error_msg["src"], error_msg["type"], msg)
)
if action is None:
if not self.__server_encryption_token:
response = self.__decrypt(self.__login_secret, encrypted_response)
else:
response = self.__decrypt(
self.__server_encryption_token, encrypted_response
)
else:
response = self.__decrypt(
self.__device_encryption_token, encrypted_response
)
jsondata = loads(response.decode("utf-8"))
if jsondata["rid"] != self.__request_id:
self.update_request_id()
return None
self.update_request_id()
return jsondata
return loads(response)

2
qBittorrent/config/qBittorrent.conf
View File

@ -63,4 +63,4 @@ WebUI\MaxAuthenticationFailCount=1000
WebUI\Port=8090
WebUI\SecureCookie=false
WebUI\UseUPnP=false
WebUI\Username=mltb
WebUI\Username=mltb

1
requirements.txt
View File

@ -19,7 +19,6 @@ motor
natsort
pillow
psutil
pycryptodome
pymongo
pyrofork
python-dotenv