Use paramiko module to upload archives.

.github/scripts/common.py

@@ -1,6 +1,6 @@
 import os
 from os import environ as _environ
-from pathlib import Path
+from pathlib import Path, PurePosixPath
 from datetime import date
 import tarfile
 import zipfile
@@ -197,55 +197,112 @@ def run_kiwix_build(
     subprocess.check_call(command, cwd=str(HOME), env=env)
     print_message("Build ended")
 
+try:
+    import paramiko
 
-def upload(file_to_upload, host, dest_path):
+    def upload(file_to_upload, host, dest_path):
-    if not file_to_upload.exists():
+        if not file_to_upload.exists():
-        print_message("No {} to upload!", file_to_upload)
+            print_message("No {} to upload!", file_to_upload)
-        return
+            return
 
-    if ":" in host:
+        if ":" in host:
-        host, port = host.split(":", 1)
+            host, port = host.split(":", 1)
-    else:
+        else:
-        port = "22"
+            port = "22"
+        if '@' in host:
+            user, host = host.split('@', 1)
+        else:
+            user = None
 
-    # sending SFTP mkdir command to the sftp interactive mode and not batch (-b) mode
+        from contextlib import contextmanager
-    # as the latter would exit on any mkdir error while it is most likely
+
-    # the first parts of the destination is already present and thus can't be created
+        @contextmanager
-    sftp_commands = "\n".join(
+        def get_client():
-        [
+            client = paramiko.client.SSHClient()
-            f"mkdir {part}"
+            client.set_missing_host_key_policy(paramiko.client.WarningPolicy)
-            for part in list(reversed(Path(dest_path).parents)) + [dest_path]
+            print_message(f"Connect to {host}:{port}")
+            client.connect(host, port=port, username=user, key_filename=_environ.get("SSH_KEY"), look_for_keys=False, compress=True)
+            try:
+                yield client
+            finally:
+                client.close()
+
+        @contextmanager
+        def get_sftp():
+            with get_client() as client:
+                sftp = client.open_sftp()
+                try:
+                    yield sftp
+                finally:
+                    sftp.close()
+
+        dest_path = PurePosixPath(dest_path)
+        remote_file = dest_path.joinpath(file_to_upload.name)
+
+        with get_sftp() as sftp:
+            for part in list(reversed(dest_path.parents)) + [dest_path]:
+                part = str(part)
+                try:
+                    sftp.stat(part)
+                except FileNotFoundError:
+                    sftp.mkdir(part)
+
+            print_message(f"Sending archive {file_to_upload} to {remote_file}")
+            sftp.put(str(file_to_upload), str(remote_file), confirm=True)
+
+
+except ModuleNotFoundError:
+    # On old system (bionic) paramiko is really complex to install
+    # Keep the old implementaion on sush system.
+
+    def upload(file_to_upload, host, dest_path):
+        if not file_to_upload.exists():
+            print_message("No {} to upload!", file_to_upload)
+            return
+
+        if ":" in host:
+            host, port = host.split(":", 1)
+        else:
+            port = "22"
+
+        # sending SFTP mkdir command to the sftp interactive mode and not batch (-b) mode
+        # as the latter would exit on any mkdir error while it is most likely
+        # the first parts of the destination is already present and thus can't be created
+        sftp_commands = "\n".join(
+            [
+                f"mkdir {part}"
+                for part in list(reversed(Path(dest_path).parents)) + [dest_path]
+            ]
+        )
+        command = [
+            "sftp",
+            "-i",
+            _environ.get("SSH_KEY"),
+            "-P",
+            port,
+            "-o",
+            "StrictHostKeyChecking=no",
+            host,
         ]
-    )
+        print_message("Creating dest path {}", dest_path)
-    command = [
+        subprocess.run(command, input=sftp_commands.encode("utf-8"), check=True)
-        "sftp",
-        "-i",
-        _environ.get("SSH_KEY"),
-        "-P",
-        port,
-        "-o",
-        "StrictHostKeyChecking=no",
-        host,
-    ]
-    print_message("Creating dest path {}", dest_path)
-    subprocess.run(command, input=sftp_commands.encode("utf-8"), check=True)
 
-    command = [
+        command = [
-        "scp",
+            "scp",
-        "-c",
+            "-c",
-        "aes128-ctr",
+            "aes128-ctr",
-        "-rp",
+            "-rp",
-        "-P",
+            "-P",
-        port,
+            port,
-        "-i",
+            "-i",
-        _environ.get("SSH_KEY"),
+            _environ.get("SSH_KEY"),
-        "-o",
+            "-o",
-        "StrictHostKeyChecking=no",
+            "StrictHostKeyChecking=no",
-        str(file_to_upload),
+            str(file_to_upload),
-        "{}:{}".format(host, dest_path),
+            "{}:{}".format(host, dest_path),
-    ]
+        ]
-    print_message("Sending archive with command {}", command)
+        print_message("Sending archive with command {}", command)
-    subprocess.check_call(command)
+        subprocess.check_call(command)
 
 
 def upload_archive(archive, project, make_release, dev_branch=None):

.github/workflows/ci.yml

@@ -26,7 +26,7 @@ jobs:
     - name: Install python modules
       shell: bash
       run: |
-        pip3 install meson pytest requests distro
+        pip3 install meson pytest requests distro paramiko
         pip3 install --no-deps $GITHUB_WORKSPACE
     - name: Setup MSVC compiler
       uses: bus1/cabuild/action/msdevshell@v1
@@ -118,6 +118,10 @@ jobs:
         pip3 install --user --no-deps .
       env:
         REP: ${{github.repository}}
+    - name: Install paramiko
+      if: ${{matrix.image_variant != 'bionic' }}
+      shell: bash
+      run: pip3 install --user paramiko
     - name: secret
       shell: bash
       run: |
@@ -167,6 +171,7 @@ jobs:
         git clone https://github.com/${REP}
         cd ./${REP##*/}
         git checkout --force ${GITHUB_SHA}
+        pip3 install --user paramiko
         pip3 install --user --no-deps .
       env:
         REP: ${{github.repository}}
@@ -227,7 +232,7 @@ jobs:
         brew install pkg-config ninja automake autoconf
     - name: Install python modules
       run: |
-        pip3 install meson pytest requests distro
+        pip3 install meson pytest requests distro paramiko
         pip3 install --no-deps $GITHUB_WORKSPACE
     - name: secret
       shell: bash

setup.py

@@ -36,7 +36,8 @@ setup(
     include_package_data=True,
     install_requires=[
         'meson',
-        'distro'
+        'distro',
+        'paramiko'
     ],
     entry_points={
         'console_scripts': [