Da

✏️ Editing: ftp.py

import posixpath
import ssl
import string
import types
from collections import OrderedDict
from ftplib import FTP, error_perm, FTP_TLS, error_temp
from io import BytesIO

from parallels.core import messages, InvalidCredentials, MigrationNoContextError
from parallels.core.logging import get_logger
from parallels.core.registry import Registry
from parallels.core.utils.common import open_no_inherit, safe_format, is_string
from parallels.core.utils.common.string_utils import safe_str_encode, safe_utf8_decode
from parallels.core.utils.entity import Entity
from parallels.core.utils.file_utils import get_relative_path

logger = get_logger(__name__)

class Ftp(object):
    MAX_RETRY = 3
    SSL_VERSIONS = OrderedDict([
        ('AUTO SELECT PROTOCOL', None),
        ('TLSv1.2', ssl.PROTOCOL_TLSv1_2),
        ('TLSv1.1', ssl.PROTOCOL_TLSv1_1),
        ('TLSv1', ssl.PROTOCOL_TLSv1)
    ])

def __init__(self, host, username, password, encoding='utf-8', ignore_passive_mode_server_ip=False, ftps_only=True):
        """
        When ignore_passive_mode_server_ip is set, IP address returned by
        FTP server when opening data channel in passive mode is ignored. That option could be necessary
        because some misconfigured FTP servers return internal IPs, which could not be used
        to connect to the server. When the option is set, hostname specified for control connection
        to the FTP server will be used for data connections.

:type host: str
        :type username: str
        :type password: str
        :type encoding: str
        :type ignore_passive_mode_server_ip: bool
        :type ftps_only: bool
        """
        self._host = host
        self._username = username
        self._password = password
        self._encoding = encoding
        self._ignore_passive_mode_server_ip = ignore_passive_mode_server_ip
        self._ftps_only = ftps_only
        self._ftp_timeout = 30

self._ftp = None

self._is_ftps = None
        self._is_mlsd_supported = None
        self._is_utf8_supported = None
        self._feat_command_output = None
        self._server_version = None
        self._cached_ssl_protocol = None

self.connect()

@property
    def base_path(self):
        return '/'

@property
    def host(self):
        return self._host

@staticmethod
    def join_path(*p):
        return posixpath.join(*p)

def connect(self):
        """Setup connection with remote FTP server

:return:
        """
        if self._is_ftps is None:  # detect FTP/FTPS
            try:
                self._is_ftps = True
                # try to connect with TLS first, and if it failed - use plain insecure FTP
                self._connect_ftps()
                # test connection by executing command:
                # some FTP servers refuse to work in TLS mode with Python FTP library
                # only when executing commands (when separate data channel is open),
                # e.g. https://bugs.python.org/issue19500
                self._ftp.nlst('/')
            except Exception as e_ftps:
                with logger.debug_level_context(
                        messages.DEBUG_FTP_CONNECTION_FAILED_FTPS,
                        host=self._host, username=self._username
                ):
                    logger.exception()

if self._ftp is not None:
                    self.close()

if self._ftps_only:
                    self._check_is_invalid_credentials(e_ftps)
                    raise
                else:

self._is_ftps = False
                    logger.fdebug(messages.DEBUG_TRY_PLAIN_FTP)
                    try:
                        self._connect_ftp()
                    except Exception as e_ftp:
                        with logger.debug_level_context(
                                messages.DEBUG_FTP_CONNECTION_FAILED_FTP,
                                host=self._host, username=self._username
                        ):
                            logger.exception()
                        self._check_is_invalid_credentials(e_ftps)
                        self._check_is_invalid_credentials(e_ftp)
                        raise
        else:
            if self._is_ftps:  # use FTPS, detected in previous runs
                self._connect_ftps()
            else:  # use FTP, detected in previous runs
                self._connect_ftp()

if self._encoding == 'utf-8' and self.is_utf8_supported():
            try:
                self._ftp.voidcmd('OPTS UTF8 ON')
            except Exception as e:
                logger.debug(messages.LOG_EXCEPTION, exc_info=True)
                logger.fwarn(messages.FTP_FAILED_TO_ENABLE_UTF8, reason=e)

def close(self):
        """Close connection

:return:
        """
        # noinspection PyBroadException
        try:
            self._ftp.close()
        except Exception:
            # There is nothing bad if error occurred when closing connection; ignore any error here
            logger.exception()

@property
    def server_version(self):
        """Returns reply of FTP server on 'connect' command.
        FTP server version is expected here.
        If connection was never established, return None

:rtype: str | None
        """
        return self._server_version

def mkdir(self, path):
        """Create directory by given path via FTP

:type path: str
        """
        logger.fdebug(messages.DEBUG_FTP_MKDIR, path=path)
        self._run_ftp_operation(lambda: self._ftp.mkd(path), safe_format(messages.DEBUG_FTP_MKDIR_FAILED, path=path))

def upload_file_content(self, content, target_path):
        logger.fdebug(messages.DEBUG_FTP_UPLOAD_FILE_CONTENT, target_path=target_path)
        self._run_ftp_operation(
            lambda: self._ftp.storbinary('STOR %s' % target_path, BytesIO(safe_str_encode(content))),
            safe_format(messages.DEBUG_FTP_UPLOAD_FILE_CONTENT_FAILED, target_path=target_path)
        )

def upload_local_file(self, source_path, target_path):
        """Upload given file from server where Plesk Migrator is running to remote server via FTP

:type source_path: str
        :type target_path: str
        """
        logger.fdebug(messages.DEBUG_FTP_UPLOAD_FILE, source_path=source_path, target_path=target_path)

def transfer_file():
            with open_no_inherit(source_path, 'rb') as fp:
                self._ftp.storbinary('STOR %s' % target_path, fp)

self._run_ftp_operation(
            transfer_file,
            safe_format(messages.DEBUG_FTP_UPLOAD_FILE_FAILED, source_path=source_path, target_path=target_path)
        )

def upload_local_directory(self, source_path, target_path):
        """Upload given directory via FTP from given server to remote server via FTP

:type source_path: str
        :type target_path: str
        """
        source_server = Registry.get_instance().get_context().migrator_server
        with source_server.runner() as runner:
            # iterate over directories and create it first
            for source_dir_path in runner.iter_dirs_list_nested(source_path):
                # Pass 'False' as the third argument to get_relative_path because FTP is always using '/' separator
                relative_path = get_relative_path(
                    source_dir_path, source_path, False, source_server.is_windows(), source_server.is_windows()
                )
                self.mkdir(self.join_path(target_path, relative_path))
            # upload files
            for source_file_path in runner.iter_files_list_nested(source_path):
                # Pass 'False' as the third argument to get_relative_path because FTP is always using '/' separator
                relative_path = get_relative_path(
                    source_file_path, source_path, False, source_server.is_windows(), source_server.is_windows()
                )
                self.upload_local_file(source_file_path, self.join_path(target_path, relative_path))

def download_file(self, source_path, target_server, target_path):
        """

:type source_path: str
        :type target_server: parallels.core.connections.migrator_server.MigratorServer
        :type target_path: str
        :rtype: None
        """
        logger.fdebug(messages.DEBUG_FTP_DOWNLOAD_FILE, source_path=source_path, target_path=target_path)

def transfer_file():
            try:
                def _write_target_file_content(content):
                    with target_server.runner() as target_server_runner:
                        target_server_runner.append_file_content(target_path, content)

# create empty file on target
                with target_server.runner() as runner:
                    runner.create_file(target_path)
                # retrieve content via FTP and write in into just created file
                self._ftp.retrbinary('RETR %s' % source_path, _write_target_file_content)
            except error_perm as e:
                logger.fdebug(
                    messages.DEBUG_FTP_DOWNLOAD_FILE_FAILED,
                    source_path=source_path, target_path=target_path, message=e
                )

self._run_ftp_operation(
            transfer_file,
            safe_format(messages.DEBUG_FTP_DOWNLOAD_FILE_FAILED, source_path=source_path, target_path=target_path)
        )

def download_file_contents(self, source_path):
        """
        :type source_path: str
        :rtype: str
        """
        logger.fdebug(messages.DEBUG_FTP_DOWNLOAD_FILE_CONTENTS, source_path=source_path)

def download_file_contents():
            file_contents = BytesIO()

try:
                def _write_target_file_content(content):
                    file_contents.write(content)
                self._ftp.retrbinary('RETR %s' % source_path, _write_target_file_content)
            except error_perm as e:
                logger.fdebug(
                    messages.DEBUG_FTP_DOWNLOAD_FILE_CONTENTS_FAILED,
                    source_path=source_path, message=e
                )

return file_contents.getvalue()

return self._run_ftp_operation(
            download_file_contents,
            safe_format(messages.DEBUG_FTP_DOWNLOAD_FILE_CONTENTS_FAILED, source_path=source_path)
        )

def remove_file(self, path):
        logger.fdebug(messages.DEBUG_FTP_REMOVE_FILE, path=path)
        self._run_ftp_operation(
            lambda: self._ftp.delete(path), safe_format(messages.DEBUG_FTP_REMOVE_FILE_FAILED, path=path)
        )

def remove_directory(self, path):
        abs_path = self.join_path('/', path)

for item in self.list_file_and_directory_paths(path):
            if item.is_dir:
                self.remove_directory(self.join_path(abs_path, item.name))
            else:
                self.remove_file(self.join_path(abs_path, item.name))

logger.fdebug(messages.DEBUG_FTP_REMOVE_DIRECTORY, path=path)
        self._run_ftp_operation(
            lambda: self._ftp.rmd(path), safe_format(messages.DEBUG_FTP_REMOVE_DIRECTORY_FAILED, path=path)
        )

def list_file_and_directory_paths(self, path):
        """
        :type path: str
        :rtype: list[parallels.core.utils.ftp.ItemInfo]
        """
        logger.fdebug(messages.DEBUG_FTP_LIST, path=path)

abs_path = self.join_path('/', path)

if not self.is_mlsd_supported():
            def ftp_dir():
                lines = []

def consume_lines(line):
                    lines.append(line)

self._ftp.dir(abs_path, consume_lines)

return lines

dir_lines = self._run_ftp_operation(
                ftp_dir, safe_format(messages.DEBUG_FTP_LIST_FAILED, path=path)
            )

return self._parse_list_output(dir_lines)
        else:
            def mlsd_dir():
                lines = []

def consume_lines(line):
                    lines.append(line)

self._ftp.retrlines('MLSD %s' % abs_path, consume_lines)

return lines

mlsd_lines = self._run_ftp_operation(
                mlsd_dir, safe_format(messages.DEBUG_FTP_LIST_FAILED, path=path)
            )

return self._parse_mlsd_output(mlsd_lines)

def list_directories(self, path):
        """
        :type path: str
        :rtype: list[str]
        """
        return [
            item.name for item in self.list_file_and_directory_paths(path)
            if item.is_dir
        ]

@property
    def is_ftps(self):
        """Whether we use FTPS or plain FTP to communicate with the server

:rtype: bool
        """
        return self._is_ftps

@property
    def ignore_passive_mode_server_ip(self):
        """Whether to ignore IP address returned by FTP server when opening data channel in passive mode

When is set to true, IP address returned by FTP server when opening data channel
        in passive mode is ignored. That option could be necessary because some misconfigured
        FTP servers return internal IPs, which could not be used to connect to the server.
        When the option is set, hostname specified for control connection to the FTP server
        will be used for data connections.

:rtype: bool
        """
        return self._ignore_passive_mode_server_ip

def _check_is_invalid_credentials(self, exception):
        """Check if specified exception is related to incorrect FTP credentials, if so - rethrow InvalidFTPCredentials

:type exception: Exception
        :raises: parallels.core.utils.ftp.InvalidFTPCredentials | parallels.core.utils.ftp.ProbablyVsftpdBadSsl
        :rtype: None
        """
        if isinstance(exception, error_perm) and hasattr(exception, 'args') and len(exception.args) > 0:
            resp = exception.args[0]
            resp_code = resp[:3]
            # 530 code not always means invalid credentials.
            # Server can reply '530 Please login with USER and PASS.' if some command passed before login.
            # Currently consider the order of commands is correct. The only special case with vsftpd server.
            # It returns '530 Please login with USER and PASS.' on login command
            # if connect is via FTPS and SSL is not configured.
            # If there will be another special cases investigate filtering by full server response.
            # For example, filter by '530 Login incorrect.'.
            if resp_code == '530':  # login/password is incorrect
                self._check_is_vsftpd_bad_ssl(resp)
                raise InvalidFTPCredentials(messages.FTP_INVALID_CREDENTIALS)

def _check_is_vsftpd_bad_ssl(self, response_message):
        """vsftpd FTP server returns '530 Please login with USER and PASS.' on login command
        if connect is via FTPS and SSL is not configured.
        Throw special exception if this case is detected.

:type response_message: str
        :raises: parallels.core.utils.ftp.ProbablyVsftpdBadSsl
        """
        if (
                self.is_ftps
                # self._server_version example: '220 (vsFTPd 3.0.2)'
                and is_string(self._server_version) and 'vsftpd' in self._server_version.lower()
                and '530 Please login with USER and PASS.' in response_message.strip()
        ):
            raise ProbablyVsftpdBadSsl(safe_format(messages.PROBABLY_VSFTPD_BAD_SSL, response_message=response_message))

def _run_ftp_operation(self, ftp_operation_function, error_message):
        error = None
        result = None
        retry = 0
        while retry < self.MAX_RETRY:
            error = None
            try:
                result = ftp_operation_function()
                if retry > 0:
                    logger.fdebug(messages.DEBUG_FTP_OPERATION_SUCCESS, retry=retry)
                break
            except error_perm as e:
                logger.exception()
                raise Exception(safe_format(error_message, message=e))
            except Exception as e:
                logger.exception()
                logger.fdebug(messages.DEBUG_FTP_OPERATION_FAILED, message=e, retry=retry+1)
                error = e
                self.close()
                bad_ssl = isinstance(e, error_temp) and is_450_error(e)
                if bad_ssl:
                    protocol_index = list(self.SSL_VERSIONS.values()).index(self._cached_ssl_protocol)
                    protocol_name, protocol_code = list(self.SSL_VERSIONS.items())[protocol_index]
                    logger.fdebug("Failed to connect by FTP over SSL using {protocol}", protocol=protocol_name)
                    protocol_index += 1
                    if protocol_index >= len(self.SSL_VERSIONS):
                        raise Exception(safe_format(error_message, message=e))
                    protocol_name, protocol_code = list(self.SSL_VERSIONS.items())[protocol_index]
                    self._cached_ssl_protocol = protocol_code
                    logger.fdebug("Try to connect by FTP over SSL using {protocol}", protocol=protocol_name)
                self.connect()
                if bad_ssl:
                    continue
            retry += 1
        if error:
            raise Exception(safe_format(error_message, message=error))
        else:
            return result

def is_utf8_supported(self):
        if self._is_utf8_supported is None:
            self._is_utf8_supported = 'utf8' in self.get_feat_command_output()

return self._is_utf8_supported

def is_mlsd_supported(self):
        if self._is_mlsd_supported is None:
            self._is_mlsd_supported = 'mlsd' in self.get_feat_command_output()
            if self._is_mlsd_supported:
                logger.fdebug(messages.DEBUG_FTP_MLSD_ENABLED)

return self._is_mlsd_supported

def get_feat_command_output(self):
        if self._feat_command_output is None:
            try:
                self._feat_command_output = self._ftp.sendcmd('FEAT').lower()
            except Exception:
                logger.exception()
                self._feat_command_output = ''
        return self._feat_command_output

def get_encoding(self):
        """
        :rtype: str
        """
        return self._encoding

def _connect_ftps(self):
        logger.fdebug(messages.DEBUG_FTPS_CONNECTION, host=self._host, username=self._username)
        if self._cached_ssl_protocol is None:
            context = ssl._create_unverified_context()
        else:
            context = ssl._create_unverified_context(protocol=self._cached_ssl_protocol)
        self._ftp = PatchedFtps(
            timeout=self._ftp_timeout,
            context=context,
            ignore_passive_mode_server_ip=self._ignore_passive_mode_server_ip
        )
        self._server_version = self._ftp.connect(self._host)
        logger.fdebug(messages.DEBUG_FTPS_CONNECTION_SUCCESS, host=self._host)

self._login()
        # secure the data connection
        self._ftp.prot_p()

def _connect_ftp(self):
        logger.fdebug(messages.DEBUG_FTP_CONNECTION, host=self._host)
        self._ftp = PatchedFtp(
            timeout=self._ftp_timeout,
            ignore_passive_mode_server_ip=self._ignore_passive_mode_server_ip
        )
        self._server_version = self._ftp.connect(self._host)
        logger.fdebug(messages.DEBUG_FTP_CONNECTION_SUCCESS, host=self._host)

self._login()

def _login(self):
        logger.fdebug(messages.DEBUG_FTP_LOGIN, host=self._host, username=self._username)
        self._ftp.login(self._username, self._password)
        logger.fdebug(messages.DEBUG_FTP_LOGIN_SUCCESS, host=self._host, username=self._username)

@staticmethod
    def _parse_list_output(lines):
        """Parse output of LIST command, return list of ItemInfo objects.

The function parses both Windows and Unix formats, format is detected automatically.
        The function is fault tolerant - in case it failed to parse any line,
        it is simply skipped, no exception is raise.

:type lines: list[str]
        :rtype: list[parallels.core.utils.ftp.ItemInfo]
        """
        items = []

for line in lines:
            if line.strip() == '':
                # ignore empty lines
                continue

if line.startswith('total'):
                # ignore lines like "total 123" stating items count in the directory
                continue

if line[0] in string.digits:
                # Windows format
                line_parts = line.split(None, 3)

if len(line_parts) != 4:
                    # invalid line - skip it
                    continue

name = line_parts[3]

if line_parts[2].lower() == '<dir>':
                    item_type = ItemInfo.TYPE_DIR
                else:
                    item_type = ItemInfo.TYPE_FILE
            else:
                # Unix format

# Two formats are possible: 8-item and 9-item. First detect format by checking the 4-th item,
                # which in case of 9-item format should digits, while in case of 8-digit format should contain
                # letters (month name).
                #
                # Example of 9-item format:
                # drwxr-x--- 5 wpabaturin 99 4096 Apr 26 09:36 public_html
                #
                # Example of 8-item format:
                # -rw-r--r-- 1 staff 3015 May 5 16:08 .bash_aliases
                #
                # See http://ftputil.sschwarzer.net/trac/ticket/12 for details on these formats.
                line_parts = line.split(None, 8)

if len(line_parts) < 7:
                    # invalid line - skip it
                    continue

if not all(c in string.digits for c in line_parts[4]):
                    line_parts = line.split(None, 7)

name = line_parts[len(line_parts) - 1]

file_type = line_parts[0].lower()[:1]

if file_type == 'd':
                    item_type = ItemInfo.TYPE_DIR
                elif file_type == '-':
                    item_type = ItemInfo.TYPE_FILE
                else:
                    # we are not interested in sockets, character devices, symbolic links, etc
                    continue

if name in ('.', '..'):
                continue

items.append(ItemInfo(safe_utf8_decode(name), item_type))

return items

@staticmethod
    def _parse_mlsd_output(lines):
        """Parse output of MSDL command, return list of ItemInfo objects.

:type lines: list[str]
        :rtype: list[parallels.core.utils.ftp.ItemInfo]
        """
        items = []

for line in lines:
            if line.strip() == '':
                # ignore empty lines; MSDL output should not contain such lines, just for better fault tolerance
                continue

parts = line.split(None, 1)

if len(parts) != 2:
                # ignore invalid lines; MSDL output should not contain such lines, just for better fault tolerance
                continue

facts = {}

facts_str, filename = parts
            facts_str_list = facts_str.split(';')

for fact_str_parts in facts_str_list:
                fact_str_parts = fact_str_parts.split('=', 1)

if len(fact_str_parts) != 2:
                    continue

name, value = fact_str_parts

facts[name.lower()] = value

if facts.get('type') == 'dir':
                item_type = ItemInfo.TYPE_DIR
            elif facts.get('type') == 'file':
                item_type = ItemInfo.TYPE_FILE
            else:
                continue

items.append(ItemInfo(safe_utf8_decode(filename), item_type))

return items

class ItemInfo(Entity):
    TYPE_FILE = 'file'
    TYPE_DIR = 'dir'

def __init__(self, name, item_type):
        self._name = name
        self._item_type = item_type

@property
    def name(self):
        return self._name

@property
    def item_type(self):
        return self._item_type

@property
    def is_dir(self):
        return self.item_type == ItemInfo.TYPE_DIR

class PatchedFtp(FTP):
    """Class to workaround ftplib.FTP issues.

Addressed issues:
    1) Added an option to allow ignoring IP address returned by FTP server when opening data channel in passive mode.
    That option could be necessary because some misconfigured FTP servers return internal IPs,
    which could not be used to connect to the server. When the option is set,
    hostname specified for control connection to the FTP server will be used for data connections.
    """
    def __init__(self, timeout, ignore_passive_mode_server_ip=False):
        self._ignore_passive_mode_server_ip = ignore_passive_mode_server_ip
        FTP.__init__(self, timeout=timeout)

def makepasv(self):
        host, port = FTP.makepasv(self)
        if self._ignore_passive_mode_server_ip:
            return self.host, port
        else:
            return host, port

class PatchedFtps(FTP_TLS):
    """Class to workaround ftplib.FTP_TLS issues

2) Fixed compatibility between FTP_TLS class and IIS FTP

FTP_TLS class tries to call "unwrap" method of SSL socket once the socket for data connection
    is not needed anymore. But IIS seems to close connection before we call "unwrap" for SSL shutdown.
    It makes "unwrap" function to fail which leads to failure of every command which uses FTP data channels.

Actually, it seems that it is not required to call "unwrap" - connection will be closed anyway. So we replace
    unwrap function with empty one for each data channel socket.

See http://bugs.python.org/issue10808 for additional details.
    """
    def __init__(self, timeout, context, ignore_passive_mode_server_ip=False):
        self._ignore_passive_mode_server_ip = ignore_passive_mode_server_ip
        FTP_TLS.__init__(self, timeout=timeout, context=context)

def makepasv(self):
        host, port = FTP_TLS.makepasv(self)
        if self._ignore_passive_mode_server_ip:
            return self.host, port
        else:
            return host, port

def transfercmd(self, *args, **kwargs):
        sock = FTP_TLS.transfercmd(self, *args, **kwargs)
        # vsftpd of some versions (e.g. 3.0.3) is not compatible with the w/a -
        # it starts to give "426 Failure reading network stream" error,
        # so apply the w/a for all servers except for vsftpd
        if self.welcome is None or ('vsftpd' not in self.welcome.lower() and 'proftpd' not in self.welcome.lower()):
            sock.unwrap = types.MethodType(lambda _: None, sock)
        return sock

class InvalidFTPCredentials(InvalidCredentials):
    pass

class ProbablyVsftpdBadSsl(MigrationNoContextError):
    """Special exception class for case where
    vsftpd FTP server returns '530 Please login with USER and PASS.' on login command
    if connect is via FTPS and SSL is not configured.
    """
    pass

def is_450_error(e):
    return '450 transfer aborted' in repr(e).lower()

📁 Sid Gifari File Manager