feat: headless environment support (#2782)

* chore(linux): config template for Linux

* feat: Linux configuration template updater

* feat: headless requirements updater

* chore: Sorted by dictionary order

* chore: Remove useless dependencies.

* chore: remove cache operation.

config/deploy.template-linux-cn.yaml

@@ -0,0 +1,159 @@
+Deploy:
+  Git:
+    # URL of AzurLaneAutoScript repository
+    # [CN user] Use 'git://git.lyoko.io/AzurLaneAutoScript' for faster and more stable download
+    # [Other] Use 'https://github.com/LmeSzinc/AzurLaneAutoScript'
+    Repository: git://git.lyoko.io/AzurLaneAutoScript
+    # Branch of Alas
+    # [Developer] Use 'dev', 'app', etc, to try new features
+    # [Other] Use 'master', the stable branch
+    Branch: master
+    # Filepath of git executable `git.exe`
+    # [Easy installer] Use './toolkit/Git/mingw64/bin/git.exe'
+    # [Other] Use you own git
+    GitExecutable: /usr/bin/git
+    # Set git proxy
+    # [CN user] Use your local http proxy (http://127.0.0.1:{port}) or socks5 proxy (socks5://127.0.0.1:{port})
+    # [Other] Use null
+    GitProxy: null
+    # Set SSL Verify
+    # [In most cases] Use true
+    # [Other] Use false to when connected to an untrusted network
+    SSLVerify: true
+    # Update Alas at startup
+    # [In most cases] Use true
+    AutoUpdate: true
+    # Whether to keep local changes during update
+    # User settings, logs and screenshots will be kept, no mather this is true or false
+    # [Developer] Use true, if you modified the code
+    # [Other] Use false
+    KeepLocalChanges: false
+
+  Python:
+    # Filepath of python executable `python.exe`
+    # [Easy installer] Use './toolkit/python.exe'
+    # [Other] Use you own python, and its version should be 3.7.6 64bit
+    PythonExecutable: python
+    # URL of pypi mirror
+    # [CN user] Use 'https://pypi.tuna.tsinghua.edu.cn/simple' for faster and more stable download
+    # [Other] Use null
+    PypiMirror: https://pypi.tuna.tsinghua.edu.cn/simple
+    # Install dependencies at startup
+    # [In most cases] Use true
+    InstallDependencies: true
+    # Path to requirements.txt
+    # [In most cases] Use 'requirements.txt'
+    # [In AidLux] Use './deploy/AidLux/{version}/requirements.txt', version is default to 0.92
+    RequirementsFile: ./deploy/headless/requirements.txt
+
+  Adb:
+    # Filepath of ADB executable `adb.exe`
+    # [Easy installer] Use './toolkit/Lib/site-packages/adbutils/binaries/adb.exe'
+    # [Other] Use you own latest ADB, but not the ADB in your emulator
+    AdbExecutable: /usr/bin/adb
+    # Whether to replace ADB
+    # Chinese emulators (NoxPlayer, LDPlayer, MemuPlayer, MuMuPlayer) use their own ADB, instead of the latest.
+    # Different ADB servers will terminate each other at startup, resulting in disconnection.
+    # For compatibility, we have to replace them all.
+    # This will do:
+    #   1. Terminate current ADB server
+    #   2. Rename ADB from all emulators to *.bak and replace them by the AdbExecutable set above
+    #   3. Brute-force connect to all available emulator instances
+    # [In most cases] Use true
+    # [In few cases] Use false, if you have other programs using ADB.
+    ReplaceAdb: false
+    # Brute-force connect to all available emulator instances
+    # [In most cases] Use true
+    AutoConnect: true
+    # Re-install uiautomator2
+    # [In most cases] Use true
+    InstallUiautomator2: true
+
+  Ocr:
+    # Run Ocr as a service, can reduce memory usage by not import mxnet everytime you start an alas instance
+
+    # Whether to use ocr server
+    # [Default] false
+    UseOcrServer: false
+    # Whether to start ocr server when start GUI
+    # [Default] false
+    StartOcrServer: false
+    # Port of ocr server runs by GUI
+    # [Default] 22268
+    OcrServerPort: 22268
+    # Address of ocr server for alas instance to connect
+    # [Default] 127.0.0.1:22268
+    OcrClientAddress: 127.0.0.1:22268
+
+  Update:
+    # Use auto update and builtin updater feature
+    # This may cause problem https://github.com/LmeSzinc/AzurLaneAutoScript/issues/876
+    EnableReload: true
+    # Check update every X minute
+    # [Disable] 0
+    # [Default] 5
+    CheckUpdateInterval: 5
+    # Scheduled restart time
+    # If there are updates, Alas will automatically restart and update at this time every day
+    # and run all alas instances that running before restarted
+    # [Disable] null
+    # [Default] 03:50
+    AutoRestartTime: 03:50
+
+  Misc:
+    # Enable discord rich presence
+    DiscordRichPresence: false
+
+  RemoteAccess:
+    # Enable remote access (using ssh reverse tunnel serve by https://github.com/wang0618/localshare)
+    # ! You need to set Password below to enable remote access since everyone can access to your alas if they have your url.
+    # See here (http://app.azurlane.cloud/en.html) for more infomation.
+    EnableRemoteAccess: false
+    # Username when login into ssh server
+    # [Default] null (will generate a random one when startup)
+    SSHUser: null
+    # Server to connect
+    # [Default] null
+    # [Format] host:port
+    SSHServer: null
+    # Filepath of SSH executable `ssh.exe`
+    # [Default] ssh (find ssh in system PATH)
+    # If you don't have one, install OpenSSH or download it here (https://github.com/PowerShell/Win32-OpenSSH/releases)
+    SSHExecutable: /usr/bin/ssh
+
+  Webui:
+    # --host. Host to listen
+    # [Use IPv6] '::'
+    # [In most cases] Default to '0.0.0.0'
+    WebuiHost: 0.0.0.0
+    # --port. Port to listen
+    # You will be able to access webui via `http://{host}:{port}`
+    # [In most cases] Default to 22267
+    WebuiPort: 22267
+    # Language to use on web ui
+    # 'zh-CN' for Chinese simplified
+    # 'en-US' for English
+    # 'ja-JP' for Japanese
+    # 'zh-TW' for Chinese traditional
+    Language: zh-CN
+    # Theme of web ui
+    # 'default' for light theme
+    # 'dark' for dark theme
+    Theme: default
+    # Follow system DPI scaling
+    # [In most cases] true
+    # [In few cases] false to make Alas smaller, if you have a low resolution but high DPI scaling.
+    DpiScaling: true
+    # --key. Password of web ui
+    # Useful when expose Alas to the public network
+    Password: null
+    # --cdn. Use jsdelivr cdn for pywebio static files (css, js).
+    # 'true' for jsdelivr cdn
+    # 'false' for self host cdn (automatically)
+    # 'https://path.to.your/cdn' to use custom cdn
+    CDN: false
+    # --run. Auto-run specified config when startup
+    # 'null' default no specified config
+    # '["alas"]' specified "alas" config
+    # '["alas","alas2"]' specified "alas" "alas2" configs
+    Run: null

config/deploy.template-linux.yaml

@@ -0,0 +1,159 @@
+Deploy:
+  Git:
+    # URL of AzurLaneAutoScript repository
+    # [CN user] Use 'git://git.lyoko.io/AzurLaneAutoScript' for faster and more stable download
+    # [Other] Use 'https://github.com/LmeSzinc/AzurLaneAutoScript'
+    Repository: https://github.com/LmeSzinc/AzurLaneAutoScript
+    # Branch of Alas
+    # [Developer] Use 'dev', 'app', etc, to try new features
+    # [Other] Use 'master', the stable branch
+    Branch: master
+    # Filepath of git executable `git.exe`
+    # [Easy installer] Use './toolkit/Git/mingw64/bin/git.exe'
+    # [Other] Use you own git
+    GitExecutable: /usr/bin/git
+    # Set git proxy
+    # [CN user] Use your local http proxy (http://127.0.0.1:{port}) or socks5 proxy (socks5://127.0.0.1:{port})
+    # [Other] Use null
+    GitProxy: null
+    # Set SSL Verify
+    # [In most cases] Use true
+    # [Other] Use false to when connected to an untrusted network
+    SSLVerify: true
+    # Update Alas at startup
+    # [In most cases] Use true
+    AutoUpdate: true
+    # Whether to keep local changes during update
+    # User settings, logs and screenshots will be kept, no mather this is true or false
+    # [Developer] Use true, if you modified the code
+    # [Other] Use false
+    KeepLocalChanges: false
+
+  Python:
+    # Filepath of python executable `python.exe`
+    # [Easy installer] Use './toolkit/python.exe'
+    # [Other] Use you own python, and its version should be 3.7.6 64bit
+    PythonExecutable: python
+    # URL of pypi mirror
+    # [CN user] Use 'https://pypi.tuna.tsinghua.edu.cn/simple' for faster and more stable download
+    # [Other] Use null
+    PypiMirror: null
+    # Install dependencies at startup
+    # [In most cases] Use true
+    InstallDependencies: true
+    # Path to requirements.txt
+    # [In most cases] Use 'requirements.txt'
+    # [In AidLux] Use './deploy/AidLux/{version}/requirements.txt', version is default to 0.92
+    RequirementsFile: ./deploy/headless/requirements.txt
+
+  Adb:
+    # Filepath of ADB executable `adb.exe`
+    # [Easy installer] Use './toolkit/Lib/site-packages/adbutils/binaries/adb.exe'
+    # [Other] Use you own latest ADB, but not the ADB in your emulator
+    AdbExecutable: /usr/bin/adb
+    # Whether to replace ADB
+    # Chinese emulators (NoxPlayer, LDPlayer, MemuPlayer, MuMuPlayer) use their own ADB, instead of the latest.
+    # Different ADB servers will terminate each other at startup, resulting in disconnection.
+    # For compatibility, we have to replace them all.
+    # This will do:
+    #   1. Terminate current ADB server
+    #   2. Rename ADB from all emulators to *.bak and replace them by the AdbExecutable set above
+    #   3. Brute-force connect to all available emulator instances
+    # [In most cases] Use true
+    # [In few cases] Use false, if you have other programs using ADB.
+    ReplaceAdb: false
+    # Brute-force connect to all available emulator instances
+    # [In most cases] Use true
+    AutoConnect: true
+    # Re-install uiautomator2
+    # [In most cases] Use true
+    InstallUiautomator2: true
+
+  Ocr:
+    # Run Ocr as a service, can reduce memory usage by not import mxnet everytime you start an alas instance
+
+    # Whether to use ocr server
+    # [Default] false
+    UseOcrServer: false
+    # Whether to start ocr server when start GUI
+    # [Default] false
+    StartOcrServer: false
+    # Port of ocr server runs by GUI
+    # [Default] 22268
+    OcrServerPort: 22268
+    # Address of ocr server for alas instance to connect
+    # [Default] 127.0.0.1:22268
+    OcrClientAddress: 127.0.0.1:22268
+
+  Update:
+    # Use auto update and builtin updater feature
+    # This may cause problem https://github.com/LmeSzinc/AzurLaneAutoScript/issues/876
+    EnableReload: true
+    # Check update every X minute
+    # [Disable] 0
+    # [Default] 5
+    CheckUpdateInterval: 5
+    # Scheduled restart time
+    # If there are updates, Alas will automatically restart and update at this time every day
+    # and run all alas instances that running before restarted
+    # [Disable] null
+    # [Default] 03:50
+    AutoRestartTime: 03:50
+
+  Misc:
+    # Enable discord rich presence
+    DiscordRichPresence: false
+
+  RemoteAccess:
+    # Enable remote access (using ssh reverse tunnel serve by https://github.com/wang0618/localshare)
+    # ! You need to set Password below to enable remote access since everyone can access to your alas if they have your url.
+    # See here (http://app.azurlane.cloud/en.html) for more infomation.
+    EnableRemoteAccess: false
+    # Username when login into ssh server
+    # [Default] null (will generate a random one when startup)
+    SSHUser: null
+    # Server to connect
+    # [Default] null
+    # [Format] host:port
+    SSHServer: null
+    # Filepath of SSH executable `ssh.exe`
+    # [Default] ssh (find ssh in system PATH)
+    # If you don't have one, install OpenSSH or download it here (https://github.com/PowerShell/Win32-OpenSSH/releases)
+    SSHExecutable: /usr/bin/ssh
+
+  Webui:
+    # --host. Host to listen
+    # [Use IPv6] '::'
+    # [In most cases] Default to '0.0.0.0'
+    WebuiHost: 0.0.0.0
+    # --port. Port to listen
+    # You will be able to access webui via `http://{host}:{port}`
+    # [In most cases] Default to 22267
+    WebuiPort: 22267
+    # Language to use on web ui
+    # 'zh-CN' for Chinese simplified
+    # 'en-US' for English
+    # 'ja-JP' for Japanese
+    # 'zh-TW' for Chinese traditional
+    Language: en-US
+    # Theme of web ui
+    # 'default' for light theme
+    # 'dark' for dark theme
+    Theme: default
+    # Follow system DPI scaling
+    # [In most cases] true
+    # [In few cases] false to make Alas smaller, if you have a low resolution but high DPI scaling.
+    DpiScaling: true
+    # --key. Password of web ui
+    # Useful when expose Alas to the public network
+    Password: null
+    # --cdn. Use jsdelivr cdn for pywebio static files (css, js).
+    # 'true' for jsdelivr cdn
+    # 'false' for self host cdn (automatically)
+    # 'https://path.to.your/cdn' to use custom cdn
+    CDN: false
+    # --run. Auto-run specified config when startup
+    # 'null' default no specified config
+    # '["alas"]' specified "alas" config
+    # '["alas","alas2"]' specified "alas" "alas2" configs
+    Run: null

deploy/AidLux/0.92/requirements.txt

@@ -1,30 +1,30 @@
-numpy
-scipy==1.7.1
-pillow
-imageio==2.27.0
-lz4
-tqdm
 adbutils==0.11.0
-uiautomator2==2.16.17
-retrying
-mxnet==1.6.0
-cnocr==1.2.2
-jellyfish
-pyyaml
-inflection
-pywebio==1.6.2
-starlette==0.14.2
-anyio==1.3.1
-uvicorn[standard]==0.17.6
 aiofiles
-wrapt==1.13.1
-prettytable==2.2.1
-pypresence==4.2.1
-rich==11.2.0
-zerorpc==0.6.3
-pyzmq==22.3.0
-onepush
-pydantic
-psutil==5.9.3
+anyio==1.3.1
 av==10.0.0
-uiautomator2cache==0.3.0.1
+cnocr==1.2.2
+imageio==2.27.0
+inflection
+jellyfish
+lz4
+mxnet==1.6.0
+numpy
+onepush
+pillow
+prettytable==2.2.1
+psutil==5.9.3
+pydantic
+pypresence==4.2.1
+pywebio==1.6.2
+pyyaml
+pyzmq==22.3.0
+retrying
+rich==11.2.0
+scipy==1.7.1
+starlette==0.14.2
+tqdm
+uiautomator2==2.16.17
+uiautomator2cache==0.3.0.1
+uvicorn[standard]==0.17.6
+wrapt==1.13.1
+zerorpc==0.6.3

deploy/docker/Dockerfile

@@ -9,10 +9,9 @@ COPY requirements.txt /tmp/requirements.txt
 
 # Initial download of UiAutomator2 is slow outside of China using appetizer mirror, switch to GitHub
 RUN apt update \
- && apt install -y git adb netcat libgomp1 \
+ && apt install -y git adb libgomp1 \
  && git config --global --add safe.directory '*' \
  && pip install -r /tmp/requirements.txt \
- && sed -i "s#path = mirror_download(url,#path = cache_download(url,#" /usr/local/lib/python3.7/site-packages/uiautomator2/init.py \
  && rm /tmp/requirements.txt \
  && rm -r ~/.cache/pip
 

deploy/docker/Dockerfile.cn

@@ -19,7 +19,7 @@ deb https://mirrors.aliyun.com/debian/ bullseye-backports main non-free contrib\
 deb-src https://mirrors.aliyun.com/debian/ bullseye-backports main non-free contrib" \
 > /etc/apt/sources.list \
  && apt update \
- && apt install -y git adb netcat libgomp1 \
+ && apt install -y git adb libgomp1 \
  && git config --global --add safe.directory '*' \
  && ln -sf /usr/share/zoneinfo/Asia/Shanghai /etc/localtime \
  && echo 'Asia/Shanghai' > /etc/timezone \

deploy/docker/requirements.txt

@@ -1,31 +1,31 @@
-numpy==1.16.6
-scipy==1.4.1
-pillow
-opencv-python-headless
-imageio==2.27.0
-lz4
-tqdm
 adbutils==0.11.0
-uiautomator2==2.16.17
-retrying
-mxnet==1.6.0
-cnocr==1.2.2
-jellyfish
-pyyaml
-inflection
-pywebio==1.6.2
-starlette==0.14.2
-anyio==1.3.1
-uvicorn[standard]==0.17.6
 aiofiles
-wrapt==1.13.1
-prettytable==2.2.1
-pypresence==4.2.1
-rich==11.2.0
-zerorpc==0.6.3
-pyzmq==22.3.0
-onepush
-pydantic
-psutil==5.9.3
+anyio==1.3.1
 av==10.0.0
-uiautomator2cache==0.3.0.1
+cnocr==1.2.2
+imageio==2.27.0
+inflection
+jellyfish
+lz4
+mxnet==1.6.0
+numpy==1.16.6
+onepush
+opencv-python-headless
+pillow
+prettytable==2.2.1
+psutil==5.9.3
+pydantic
+pypresence==4.2.1
+pywebio==1.6.2
+pyyaml
+pyzmq==22.3.0
+retrying
+rich==11.2.0
+scipy==1.4.1
+starlette==0.14.2
+tqdm
+uiautomator2==2.16.17
+uiautomator2cache==0.3.0.1
+uvicorn[standard]==0.17.6
+wrapt==1.13.1
+zerorpc==0.6.3

deploy/headless/requirements_generator.py

@@ -0,0 +1,70 @@
+import os
+
+from deploy.logger import logger
+
+BASE_FOLDER = os.path.dirname(os.path.abspath(__file__))
+logger.info(BASE_FOLDER)
+
+
+def read_file(file):
+    out = {}
+    with open(file, 'r', encoding='utf-8') as f:
+        for line in f.readlines():
+            res = [s.strip() for s in line.split('==')]
+            if len(res) > 1:
+                name, version = res
+            else:
+                name, version = res[0], None
+            out[name] = version
+
+    return out
+
+
+def write_file(file, data):
+    lines = []
+    for name, version in data.items():
+        if version:
+            lines.append(f'{name}=={version}')
+        else:
+            lines.append(str(name))
+
+    with open(file, 'w', encoding='utf-8', newline='') as f:
+        f.write('\n'.join(lines))
+
+
+def headless_requirements_generate(requirements_in='requirements-in.txt'):
+    requirements = read_file(requirements_in)
+
+    logger.info(f'Generate requirements for headless environment')
+    lock = {
+        'aiofiles': '23.1.0',
+        'inflection': '0.5.1',
+        'jellyfish': '0.11.2',
+        'lz4': '4.3.2',
+        'numpy': '1.17.4',
+        'onepush': '1.2.0',
+        'opencv-python': {
+            'name': 'opencv-python-headless',
+            'version': '4.7.0.72'
+        },
+        'pillow': '9.5.0',
+        'pydantic': '1.10.9',
+        'pyyaml': '6.0',
+        'retrying': '1.3.4',
+        'tqdm': '4.65.0',
+        'wrapt': '1.15.0'
+    }
+    new = {}
+    for name, version in requirements.items():
+        if name == 'alas-webapp':
+            continue
+        if name in lock:
+            version = lock[name] if not isinstance(lock[name], dict) else lock[name]['version']
+            name = name if not isinstance(lock[name], dict) else lock[name]['name']
+        new[name] = version
+
+    write_file(os.path.join(BASE_FOLDER, f'./requirements.txt'), data=new)
+
+
+if __name__ == '__main__':
+    headless_requirements_generate()

dev_tools/requirements_updater.py

@@ -2,6 +2,7 @@ import os
 
 from deploy.AidLux.requirements_generator import aidlux_requirements_generate
 from deploy.docker.requirements_generator import docker_requirements_generate
+from deploy.headless.requirements_generator import headless_requirements_generate
 
 # Ensure running in Alas root folder
 os.chdir(os.path.join(os.path.dirname(__file__), '../'))
@@ -39,3 +40,4 @@ if __name__ == '__main__':
     requirements_modify()
     aidlux_requirements_generate()
     docker_requirements_generate()
+    headless_requirements_generate()

module/config/config_updater.py

@@ -427,6 +427,15 @@ class ConfigGenerator:
             'AdbExecutable': '/usr/bin/adb',
         }
 
+        linux = {
+            'GitExecutable': '/usr/bin/git',
+            'PythonExecutable': 'python',
+            'RequirementsFile': './deploy/headless/requirements.txt',
+            'AdbExecutable': '/usr/bin/adb',
+            'SSHExecutable': '/usr/bin/ssh',
+            'ReplaceAdb': 'false'
+        }
+
         def update(suffix, *args):
             file = f'./config/deploy.{suffix}.yaml'
             new = deepcopy(template)
@@ -440,6 +449,8 @@ class ConfigGenerator:
         update('template-AidLux-cn', aidlux, cn)
         update('template-docker', docker)
         update('template-docker-cn', docker, cn)
+        update('template-linux', linux)
+        update('template-linux-cn', linux, cn)
 
     def insert_package(self):
         option = deep_get(self.argument, keys='Emulator.PackageName.option')

requirements-in.txt

@@ -1,32 +1,32 @@
-numpy==1.16.6
-scipy==1.4.1
-pillow
-opencv-python
-imageio==2.27.0
-lz4
-tqdm
 adbutils==0.11.0
-uiautomator2==2.16.17
-retrying
-mxnet==1.6.0
-cnocr==1.2.2
-jellyfish
-pyyaml
-inflection
-pywebio==1.6.2
-starlette==0.14.2
-anyio==1.3.1
-uvicorn[standard]==0.17.6
 aiofiles
-wrapt==1.13.1
-prettytable==2.2.1
-pypresence==4.2.1
 alas-webapp==0.3.7
-rich==11.2.0
-zerorpc==0.6.3
-pyzmq==22.3.0
-onepush
-pydantic
-psutil==5.9.3
+anyio==1.3.1
 av==10.0.0
-uiautomator2cache==0.3.0.1
+cnocr==1.2.2
+imageio==2.27.0
+inflection
+jellyfish
+lz4
+mxnet==1.6.0
+numpy==1.16.6
+onepush
+opencv-python
+pillow
+prettytable==2.2.1
+psutil==5.9.3
+pydantic
+pypresence==4.2.1
+pywebio==1.6.2
+pyyaml
+pyzmq==22.3.0
+retrying
+rich==11.2.0
+scipy==1.4.1
+starlette==0.14.2
+tqdm
+uiautomator2==2.16.17
+uiautomator2cache==0.3.0.1
+uvicorn[standard]==0.17.6
+wrapt==1.13.1
+zerorpc==0.6.3