diff --git a/alas.py b/alas.py index b1b0cde09..90173a17e 100644 --- a/alas.py +++ b/alas.py @@ -9,7 +9,7 @@ from cached_property import cached_property from module.base.decorator import del_cached_property from module.config.config import AzurLaneConfig, TaskEnd -from module.config.utils import deep_get, deep_set +from module.config.deep import deep_get, deep_set from module.exception import * from module.logger import logger from module.notify import handle_notify diff --git a/module/config/config.py b/module/config/config.py index f57aa0be0..97631fca1 100644 --- a/module/config/config.py +++ b/module/config/config.py @@ -1,16 +1,17 @@ import copy -import datetime import operator import threading +from datetime import datetime, timedelta import pywebio from module.base.filter import Filter from module.config.config_generated import GeneratedConfig from module.config.config_manual import ManualConfig, OutputConfig -from module.config.config_updater import ConfigUpdater +from module.config.config_updater import ConfigUpdater, ensure_time, get_server_next_update, nearest_future +from module.config.deep import deep_get, deep_set +from module.config.utils import DEFAULT_TIME, dict_to_kv, filepath_config, get_os_reset_remain, path_to_arg from module.config.watcher import ConfigWatcher -from module.config.utils import * from module.exception import RequestHumanTakeover, ScriptError from module.logger import logger from module.map.map_grids import SelectedGrids diff --git a/module/config/config_updater.py b/module/config/config_updater.py index 2d2c395a2..9fa8f8381 100644 --- a/module/config/config_updater.py +++ b/module/config/config_updater.py @@ -6,10 +6,11 @@ from cached_property import cached_property from deploy.utils import DEPLOY_TEMPLATE, poor_yaml_read, poor_yaml_write from module.base.timer import timer +from module.config.deep import deep_default, deep_get, deep_iter, deep_pop, deep_set from module.config.env import IS_ON_PHONE_CLOUD -from module.config.redirect_utils.utils import * from module.config.server import VALID_CHANNEL_PACKAGE, VALID_PACKAGE, VALID_SERVER_LIST, to_package, to_server from module.config.utils import * +from module.config.redirect_utils.utils import * CONFIG_IMPORT = ''' import datetime diff --git a/module/config/deep.py b/module/config/deep.py new file mode 100644 index 000000000..3f4bb9e04 --- /dev/null +++ b/module/config/deep.py @@ -0,0 +1,533 @@ +from collections import deque + +# deep_* functions are used for access nested dictionary. +# They target for high performance so code are complicated to read +# In general performance practise, time costs are as below: +# - When key exists +# try: dict[key] except KeyError << dict.get(key) < if key in dict: dict[key] +# - When not key exists +# if key in dict: dict[key] < dict.get(key) <<< try: dict[key] except KeyError + +OP_ADD = 'add' +OP_SET = 'set' +OP_DEL = 'del' + + +def deep_get(d, keys, default=None): + """ + Get value from nested dict and list + https://stackoverflow.com/questions/25833613/safe-method-to-get-value-of-nested-dictionary + + Args: + d (dict): + keys (list[str], str): Such as ['Scheduler', 'NextRun', 'value'] + default: Default return if key not found. + + Returns: + Value on given keys + """ + # 240 + 30 * depth (ns) + if type(keys) is str: + keys = keys.split('.') + + try: + for k in keys: + d = d[k] + return d + # No such key + except KeyError: + return default + # No such key + except IndexError: + return default + # Input `keys` is not iterable or input `d` is not dict + # list indices must be integers or slices, not str + except TypeError: + return default + + +def deep_get_with_error(d, keys): + """ + Get value from nested dict and list, raise KeyError if key not exists + + Args: + d (dict): + keys (list[str], str): Such as ['Scheduler', 'NextRun', 'value'] + + Returns: + Value on given keys + + Raises: + KeyError: If key not exists + """ + # 240 + 30 * depth (ns) + if type(keys) is str: + keys = keys.split('.') + + try: + for k in keys: + d = d[k] + return d + # No such key + # except KeyError: + # raise + # No such key + except IndexError: + raise KeyError + # Input `keys` is not iterable or input `d` is not dict + # list indices must be integers or slices, not str + except TypeError: + raise KeyError + + +def deep_exist(d, keys): + """ + Check if keys exists in nested dict or list + + Args: + d (dict): + keys (str, list): Such as `Scheduler.NextRun.value` + + Returns: + bool: If key exists + """ + # 240 + 30 * depth (ns) + if type(keys) is str: + keys = keys.split('.') + + try: + for k in keys: + d = d[k] + return True + # No such key + except KeyError: + return False + # No such key + except IndexError: + return False + # Input `keys` is not iterable or input `d` is not dict + # list indices must be integers or slices, not str + except TypeError: + return False + + +def deep_set(d, keys, value): + """ + Set value into nested dict safely, imitating deep_get(). + Can only set dict + """ + # 150 * depth (ns) + if type(keys) is str: + keys = keys.split('.') + + first = True + exist = True + prev_d = None + prev_k = None + prev_k2 = None + try: + for k in keys: + if first: + prev_d = d + prev_k = k + first = False + continue + try: + # if key in dict: dict[key] > dict.get > dict.setdefault > try dict[key] except + if exist and prev_k in d: + prev_d = d + d = d[prev_k] + else: + exist = False + new = {} + d[prev_k] = new + d = new + except TypeError: + # `d` is not dict + exist = False + d = {} + prev_d[prev_k2] = {prev_k: d} + + prev_k2 = prev_k + prev_k = k + # prev_k2, prev_k = prev_k, k + # Input `keys` is not iterable + except TypeError: + return + + # Last key, set value + try: + d[prev_k] = value + return + # Last value `d` is not dict + except TypeError: + prev_d[prev_k2] = {prev_k: value} + return + + +def deep_default(d, keys, value): + """ + Set value into nested dict safely, imitating deep_get(). + Can only set dict + """ + # 150 * depth (ns) + if type(keys) is str: + keys = keys.split('.') + + first = True + exist = True + prev_d = None + prev_k = None + prev_k2 = None + try: + for k in keys: + if first: + prev_d = d + prev_k = k + first = False + continue + try: + # if key in dict: dict[key] > dict.get > dict.setdefault > try dict[key] except + if exist and prev_k in d: + prev_d = d + d = d[prev_k] + else: + exist = False + new = {} + d[prev_k] = new + d = new + except TypeError: + # `d` is not dict + exist = False + d = {} + prev_d[prev_k2] = {prev_k: d} + + prev_k2 = prev_k + prev_k = k + # prev_k2, prev_k = prev_k, k + # Input `keys` is not iterable + except TypeError: + return + + # Last key, set value + try: + d.setdefault(prev_k, value) + return + # Last value `d` is not dict + except AttributeError: + prev_d[prev_k2] = {prev_k: value} + return + + +def deep_pop(d, keys, default=None): + """ + Pop value from nested dict and list + """ + if type(keys) is str: + keys = keys.split('.') + + try: + for k in keys[:-1]: + d = d[k] + # No `pop(k, default)` so it can pop list + return d.pop(keys[-1]) + # No such key + except KeyError: + return default + # Input `keys` is not iterable or input `d` is not dict + # list indices must be integers or slices, not str + except TypeError: + return default + # Input `keys` out of index + except IndexError: + return default + # Last `d` is not dict + except AttributeError: + return default + + +def deep_iter_depth1(data): + """ + Equivalent to data.items() but suppress error if data is not a dict + + Args: + data: + + Yields: + Any: Key + Any: Value + """ + try: + for k, v in data.items(): + yield k, v + return + except AttributeError: + # `data` is not dict + return + + +def deep_iter_depth2(data): + """ + Iter key and value in nested dict of depth 2 + A simplified deep_iter + + Args: + data: + + Yields: + Any: Key1 + Any: Key2 + Any: Value + """ + try: + for k1, v1 in data.items(): + if type(v1) is dict: + for k2, v2 in v1.items(): + yield k1, k2, v2 + except AttributeError: + # `data` is not dict + return + + +def deep_iter(data, min_depth=None, depth=3): + """ + Iter key and value in nested dict + 300us on alas.json depth=3 (530+ rows) + Can only iter dict + + Args: + data: + min_depth: + depth: + + Yields: + list[str]: Key path + Any: Value + """ + if min_depth is None: + min_depth = depth + assert 1 <= min_depth <= depth + + # Equivalent to dict.items() + try: + if depth == 1: + for k, v in data.items(): + yield [k], v + return + # Iter first depth + elif min_depth == 1: + q = deque() + for k, v in data.items(): + key = [k] + if type(v) is dict: + q.append((key, v)) + else: + yield key, v + # Iter target depth only + else: + q = deque() + for k, v in data.items(): + key = [k] + if type(v) is dict: + q.append((key, v)) + except AttributeError: + # `data` is not dict + return + + # Iter depths + current = 2 + while current <= depth: + new_q = deque() + # max depth + if current == depth: + for key, data in q: + for k, v in data.items(): + yield key + [k], v + # in target depth + elif min_depth <= current < depth: + for key, data in q: + for k, v in data.items(): + subkey = key + [k] + if type(v) is dict: + new_q.append((subkey, v)) + else: + yield subkey, v + # Haven't reached min depth + else: + for key, data in q: + for k, v in data.items(): + subkey = key + [k] + if type(v) is dict: + new_q.append((subkey, v)) + q = new_q + current += 1 + + +def deep_values(data, min_depth=None, depth=3): + """ + Iter value in nested dict + 300us on alas.json depth=3 (530+ rows) + Can only iter dict + + Args: + data: + min_depth: + depth: + + Yields: + Any: Value + """ + if min_depth is None: + min_depth = depth + assert 1 <= min_depth <= depth + + # Equivalent to dict.items() + try: + if depth == 1: + for v in data.values(): + yield v + return + # Iter first depth + elif min_depth == 1: + q = deque() + for v in data.values(): + if type(v) is dict: + q.append(v) + else: + yield v + # Iter target depth only + else: + q = deque() + for v in data.values(): + if type(v) is dict: + q.append(v) + except AttributeError: + # `data` is not dict + return + + # Iter depths + current = 2 + while current <= depth: + new_q = deque() + # max depth + if current == depth: + for data in q: + for v in data.values(): + yield v + # in target depth + elif min_depth <= current < depth: + for data in q: + for v in data.values(): + if type(v) is dict: + new_q.append(v) + else: + yield v + # Haven't reached min depth + else: + for data in q: + for v in data.values(): + if type(v) is dict: + new_q.append(v) + q = new_q + current += 1 + + +def deep_iter_diff(before, after): + """ + Iter diff between 2 dict. + Pretty fast to compare 2 deeply nested dict, + time cost increases with the number of differences. + + Args: + before: + after: + + Yields: + list[str]: Key path + Any: Value in before, or None if not exists + Any: Value in after, or None if not exists + """ + if before == after: + return + if type(before) is not dict or type(after) is not dict: + yield [], before, after + return + + queue = deque([([], before, after)]) + while True: + new_queue = deque() + for path, d1, d2 in queue: + keys1 = set(d1.keys()) + keys2 = set(d2.keys()) + for key in keys1.union(keys2): + try: + val2 = d2[key] + except KeyError: + # Safe to access d1[key], because key came from the union of both + # If it's not in d2 then it's in d1 + yield path + [key], d1[key], None + continue + try: + val1 = d1[key] + except KeyError: + yield path + [key], None, val2 + continue + # Compare dict first, which is pretty fast + if val1 != val2: + if type(val1) is dict and type(val2) is dict: + new_queue.append((path + [key], val1, val2)) + else: + yield path + [key], val1, val2 + queue = new_queue + if not queue: + break + + +def deep_iter_patch(before, after): + """ + Iter patch event from before to after, like creating a json-patch + Pretty fast to compare 2 deeply nested dict, + time cost increases with the number of differences. + + Args: + before: + after: + + Yields: + str: OP_ADD, OP_SET, OP_DEL + list[str]: Key path + Any: Value in after, + or None of event is OP_DEL + """ + if before == after: + return + if type(before) is not dict or type(after) is not dict: + yield OP_SET, [], after + return + + queue = deque([([], before, after)]) + while True: + new_queue = deque() + for path, d1, d2 in queue: + keys1 = set(d1.keys()) + keys2 = set(d2.keys()) + for key in keys1.union(keys2): + try: + val2 = d2[key] + except KeyError: + yield OP_DEL, path + [key], None + continue + try: + val1 = d1[key] + except KeyError: + yield OP_ADD, path + [key], val2 + continue + # Compare dict first, which is pretty fast + if val1 != val2: + if type(val1) is dict and type(val2) is dict: + new_queue.append((path + [key], val1, val2)) + else: + yield OP_SET, path + [key], val2 + queue = new_queue + if not queue: + break diff --git a/module/config/utils.py b/module/config/utils.py index d42269448..423e6fa22 100644 --- a/module/config/utils.py +++ b/module/config/utils.py @@ -181,101 +181,6 @@ def alas_instance(): return out -def deep_get(d, keys, default=None): - """ - Get values in dictionary safely. - https://stackoverflow.com/questions/25833613/safe-method-to-get-value-of-nested-dictionary - - Args: - d (dict): - keys (str, list): Such as `Scheduler.NextRun.value` - default: Default return if key not found. - - Returns: - - """ - if isinstance(keys, str): - keys = keys.split('.') - assert type(keys) is list - if d is None: - return default - if not keys: - return d - return deep_get(d.get(keys[0]), keys[1:], default) - - -def deep_set(d, keys, value): - """ - Set value into dictionary safely, imitating deep_get(). - """ - if isinstance(keys, str): - keys = keys.split('.') - assert type(keys) is list - if not keys: - return value - if not isinstance(d, dict): - d = {} - d[keys[0]] = deep_set(d.get(keys[0], {}), keys[1:], value) - return d - - -def deep_pop(d, keys, default=None): - """ - Pop value from dictionary safely, imitating deep_get(). - """ - if isinstance(keys, str): - keys = keys.split('.') - assert type(keys) is list - if not isinstance(d, dict): - return default - if not keys: - return default - elif len(keys) == 1: - return d.pop(keys[0], default) - return deep_pop(d.get(keys[0]), keys[1:], default) - - -def deep_default(d, keys, value): - """ - Set default value into dictionary safely, imitating deep_get(). - Value is set only when the dict doesn't contain such keys. - """ - if isinstance(keys, str): - keys = keys.split('.') - assert type(keys) is list - if not keys: - if d: - return d - else: - return value - if not isinstance(d, dict): - d = {} - d[keys[0]] = deep_default(d.get(keys[0], {}), keys[1:], value) - return d - - -def deep_iter(data, depth=0, current_depth=1): - """ - Iter a dictionary safely. - - Args: - data (dict): - depth (int): Maximum depth to iter - current_depth (int): - - Returns: - list: Key path - Any: - """ - if isinstance(data, dict) \ - and (depth and current_depth <= depth): - for key, value in data.items(): - for child_path, child_value in deep_iter(value, depth=depth, current_depth=current_depth + 1): - yield [key] + child_path, child_value - else: - yield [], data - - def parse_value(value, data): """ Convert a string to float, int, datetime, if possible. diff --git a/module/device/connection_attr.py b/module/device/connection_attr.py index 0814b9524..ef8b343dc 100644 --- a/module/device/connection_attr.py +++ b/module/device/connection_attr.py @@ -8,7 +8,7 @@ from adbutils import AdbClient, AdbDevice from module.base.decorator import cached_property from module.config.config import AzurLaneConfig from module.config.env import IS_ON_PHONE_CLOUD -from module.config.utils import deep_iter +from module.config.deep import deep_iter from module.device.method.utils import get_serial_pair from module.exception import RequestHumanTakeover from module.logger import logger diff --git a/module/device/method/nemu_ipc.py b/module/device/method/nemu_ipc.py index b91a6b531..e0a8c927a 100644 --- a/module/device/method/nemu_ipc.py +++ b/module/device/method/nemu_ipc.py @@ -11,7 +11,7 @@ import numpy as np from module.base.decorator import cached_property, del_cached_property, has_cached_property from module.base.timer import Timer from module.base.utils import ensure_time -from module.config.utils import deep_get +from module.config.deep import deep_get from module.device.env import IS_WINDOWS from module.device.method.minitouch import insert_swipe, random_rectangle_point from module.device.method.pool import JobTimeout, WORKER_POOL diff --git a/module/statistics/azurstats.py b/module/statistics/azurstats.py index c815479dd..695da5f91 100644 --- a/module/statistics/azurstats.py +++ b/module/statistics/azurstats.py @@ -10,7 +10,7 @@ from requests.adapters import HTTPAdapter from module.base.utils import save_image from module.config.config import AzurLaneConfig -from module.config.utils import deep_get +from module.config.deep import deep_get from module.exception import ScriptError from module.logger import logger from module.statistics.utils import pack diff --git a/module/webui/app.py b/module/webui/app.py index d7d6ba1f4..0d8d0f31b 100644 --- a/module/webui/app.py +++ b/module/webui/app.py @@ -37,17 +37,15 @@ from pywebio.output import ( use_scope, ) from pywebio.pin import pin, pin_on_change -from pywebio.session import (download, go_app, info, local, register_thread, run_js, set_env) +from pywebio.session import download, go_app, info, local, register_thread, run_js, set_env import module.webui.lang as lang from module.config.config import AzurLaneConfig, Function +from module.config.deep import deep_get, deep_iter, deep_set from module.config.env import IS_ON_PHONE_CLOUD from module.config.utils import ( alas_instance, alas_template, - deep_get, - deep_iter, - deep_set, dict_to_kv, filepath_args, filepath_config, diff --git a/module/webui/lang.py b/module/webui/lang.py index 0a5cd3999..a47331333 100644 --- a/module/webui/lang.py +++ b/module/webui/lang.py @@ -1,8 +1,9 @@ from typing import Dict -from module.config.utils import * -from module.webui.setting import State +from module.config.deep import deep_iter +from module.config.utils import LANGUAGES, filepath_i18n, read_file from module.submodule.utils import list_mod_dir +from module.webui.setting import State LANG = "zh-CN" TRANSLATE_MODE = False diff --git a/module/webui/translate.py b/module/webui/translate.py index 7953673b3..970c76433 100644 --- a/module/webui/translate.py +++ b/module/webui/translate.py @@ -6,8 +6,8 @@ from pywebio.output import put_buttons, put_markdown from pywebio.session import defer_call, hold, run_js, set_env import module.webui.lang as lang -from module.config.utils import (LANGUAGES, deep_get, deep_iter, deep_set, - filepath_i18n, read_file, write_file) +from module.config.deep import deep_get, deep_iter, deep_set +from module.config.utils import LANGUAGES, filepath_i18n, read_file, write_file def translate(): diff --git a/module/webui/utils.py b/module/webui/utils.py index e813c99ac..6426e76fa 100644 --- a/module/webui/utils.py +++ b/module/webui/utils.py @@ -9,17 +9,15 @@ from queue import Queue from typing import Callable, Generator, List import pywebio -from module.config.utils import deep_iter -from module.logger import logger -from module.webui.setting import State from pywebio.input import PASSWORD, input from pywebio.output import PopupSize, popup, put_html, toast -from pywebio.session import eval_js -from pywebio.session import info as session_info -from pywebio.session import register_thread, run_js -from rich.console import Console, ConsoleOptions +from pywebio.session import eval_js, info as session_info, register_thread, run_js +from rich.console import Console from rich.terminal_theme import TerminalTheme +from module.config.deep import deep_iter +from module.logger import logger +from module.webui.setting import State RE_DATETIME = ( r"\d{4}\-(0\d|1[0-2])\-([0-2]\d|[3][0-1]) " diff --git a/submodule/AlasFpyBridge/module/config/config_updater.py b/submodule/AlasFpyBridge/module/config/config_updater.py index 490408ebe..bbfd3d4ec 100644 --- a/submodule/AlasFpyBridge/module/config/config_updater.py +++ b/submodule/AlasFpyBridge/module/config/config_updater.py @@ -2,6 +2,7 @@ from cached_property import cached_property from module.base.timer import timer from module.config import config_updater +from module.config.deep import deep_get, deep_set, deep_iter from module.config.utils import * diff --git a/submodule/AlasMaaBridge/module/config/config_updater.py b/submodule/AlasMaaBridge/module/config/config_updater.py index 6e777ae3b..21f066e13 100644 --- a/submodule/AlasMaaBridge/module/config/config_updater.py +++ b/submodule/AlasMaaBridge/module/config/config_updater.py @@ -2,6 +2,7 @@ from cached_property import cached_property from module.base.timer import timer from module.config import config_updater +from module.config.deep import deep_get, deep_iter, deep_set from module.config.utils import * diff --git a/submodule/AlasMaaBridge/module/handler/handler.py b/submodule/AlasMaaBridge/module/handler/handler.py index bb48e7c3d..b61dd03d2 100644 --- a/submodule/AlasMaaBridge/module/handler/handler.py +++ b/submodule/AlasMaaBridge/module/handler/handler.py @@ -10,7 +10,8 @@ from cached_property import cached_property from deploy.config import DeployConfig from module.base.timer import Timer -from module.config.utils import read_file, deep_get, get_server_last_update +from module.config.deep import deep_get +from module.config.utils import read_file, get_server_last_update from module.device.connection_attr import ConnectionAttr from module.exception import RequestHumanTakeover from module.logger import logger diff --git a/submodule/AlasMaaBridge/module/logger.py b/submodule/AlasMaaBridge/module/logger.py index a3139a9bb..2d2c06480 100644 --- a/submodule/AlasMaaBridge/module/logger.py +++ b/submodule/AlasMaaBridge/module/logger.py @@ -1,7 +1,7 @@ import typing as t from module.base.decorator import cached_property -from module.config.utils import deep_get +from module.config.deep import deep_get from module.logger import logger