Merge pull request #524 from kiwix/dropbox

.github/scripts/common.py

@@ -168,20 +168,38 @@ def upload(file_to_upload, host, dest_path):
         print_message("No {} to upload!", file_to_upload)
         return
 
+    if ":" in host:
+        host, port = host.split(":", 1)
+    else:
+        port = "22"
+
+    # sending SFTP mkdir command to the sftp interactive mode and not batch (-b) mode
+    # as the latter would exit on any mkdir error while it is most likely
+    # the first parts of the destination is already present and thus can't be created
+    sftp_commands = "\n".join(
+        [
+            f"mkdir {part}"
+            for part in list(reversed(Path(dest_path).parents)) + [dest_path]
+        ]
+    )
     command = [
-        "ssh",
+        "sftp",
         "-i",
         _environ.get("SSH_KEY"),
+        "-P",
+        port,
         "-o",
         "StrictHostKeyChecking=no",
         host,
-        "mkdir -p {}".format(dest_path),
     ]
     print_message("Creating dest path {}", dest_path)
-    subprocess.check_call(command)
+    subprocess.run(command, input=sftp_commands.encode("utf-8"), check=True)
 
     command = [
         "scp",
+        "-r",
+        "-P",
+        port,
         "-i",
         _environ.get("SSH_KEY"),
         "-o",
@@ -195,10 +213,10 @@ def upload(file_to_upload, host, dest_path):
 
 def upload_archive(archive, project, make_release):
     if project.startswith("kiwix-") or project in ['libkiwix']:
-        host = "ci@download.kiwix.org"
+        host = "ci@master.download.kiwix.org:30022"
         dest_path = "/data/download/"
     else:
-        host = "ci@download.openzim.org"
+        host = "ci@download.openzim.org:30022"
         dest_path = "/data/openzim/"
 
     if make_release:

.github/scripts/compile_all_deps.py

@@ -32,5 +32,5 @@ else:
 for target in TARGETS:
     run_kiwix_build(target, platform=PLATFORM_TARGET, build_deps_only=True)
     archive_file = make_deps_archive(target=target)
-    upload(archive_file, "ci@tmp.kiwix.org", "/data/tmp/ci")
+    upload(archive_file, "ci@tmp.kiwix.org:30022", "/data/tmp/ci")
     os.remove(str(archive_file))

.github/scripts/ensure_base_deps.py

@@ -69,5 +69,5 @@ except URLError:
     else:
         run_kiwix_build("alldependencies", platform=PLATFORM_TARGET)
         archive_file = make_deps_archive(name=base_dep_archive_name, full=True)
-        upload(archive_file, "ci@tmp.kiwix.org", "/data/tmp/ci")
+        upload(archive_file, "ci@tmp.kiwix.org:30022", "/data/tmp/ci")
         os.remove(str(archive_file))

.github/scripts/upload_failure_logs.sh

@@ -9,7 +9,7 @@ tar -czf ${ARCHIVE_NAME} $HOME/BUILD_* $HOME/SOURCE $HOME/LOGS $HOME/TOOLCHAINS
 
 echo "Uploading archive $ARCHIVE_NAME"
 
-scp -p -i ${SSH_KEY} \
+scp -P 30022 -p -i ${SSH_KEY} \
   -o PasswordAuthentication=no \
   -o StrictHostKeyChecking=no \
   $ARCHIVE_NAME \

appveyor/create_cache.sh

@@ -5,7 +5,7 @@ ARCHIVE_NAME="deps_windows_windows.zip"
 SSH_KEY=C:\\projects\\kiwix-build\\appveyor\\nightlybot_id_key
 
 7z a -tzip $ARCHIVE_NAME $MINGW64_EXTRA_DIR
-scp -vrp -i ${SSH_KEY} -o StrictHostKeyChecking=no \
+scp -P 30022 -vrp -i ${SSH_KEY} -o StrictHostKeyChecking=no \
   ${ARCHIVE_NAME} \
   ci@tmp.kiwix.org:/data/tmp/ci
 

appveyor/deploy.sh

@@ -7,15 +7,15 @@ SSH_KEY=C:\\projects\\kiwix-build\\appveyor\\nightlybot_id_key
 
 if [[ "$APPVEYOR_SCHEDULED_BUILD" = "True" ]]
 then
-  scp -vrp -i ${SSH_KEY} -o StrictHostKeyChecking=no \
+  scp -P 30022 -vrp -i ${SSH_KEY} -o StrictHostKeyChecking=no \
     ${NIGHTLY_KIWIX_ARCHIVES_DIR} \
-    ci@download.kiwix.org:/data/download/nightly
+    ci@master.download.kiwix.org:/data/download/nightly
 fi
 
 if [[ "$APPVEYOR_REPO_TAG" = "true" ]]
 then
   RELEASE_ARCHIVES=$(find $RELEASE_KIWIX_ARCHIVES_DIR -type f)
-  scp -vrp -i ${SSH_KEY} -o StrictHostKeyChecking=no \
+  scp -P 30022 -vrp -i ${SSH_KEY} -o StrictHostKeyChecking=no \
       ${RELEASE_ARCHIVES} \
-      ci@download.kiwix.org:/data/download/release/kiwix-desktop
+      ci@master.download.kiwix.org:/data/download/release/kiwix-desktop
 fi