Da

✏️ Editing: steps_profiler.py

from parallels.core import messages
import itertools
from contextlib import contextmanager
from textwrap import wrap
import time
import datetime
import os

from parallels.core.utils.json_utils import write_json
from parallels.core.utils.common import find_first, is_empty, open_no_inherit

_default_steps_profiler = None
_default_steps_report = None

def get_default_steps_profiler():
    """Get default global instance of StepsProfiler"""
    global _default_steps_profiler
    if _default_steps_profiler is None:
        _default_steps_profiler = StepsProfiler()
    return _default_steps_profiler

def get_default_steps_report():
    """Get default global instance of StepsProfilerReport"""
    global _default_steps_report
    if _default_steps_report is None:
        _default_steps_report = StepsProfilerReport(
            profiler=get_default_steps_profiler()
        )
    return _default_steps_report

class StepTime(object):
    """Object representing execution time structure of single migration step"""

def __init__(self, step_id, step_name, time, substeps):
        """Object constructor

Arguments:
        - step_id - string identifier of a step, for example 'restore-hosting',
        or 'copy-mail-content'
        - step_name - user-friendly name of a step
        - time - total step execution time in seconds as floating-point number
        - substeps - list of substeps (instances of StepTime) of this step, for
        example 'transfer-accounts' steps will have 'fetch-source',
        'import-accounts', 'restore-hosting', etc as substeps
        """
        self.step_id = step_id
        self.step_name = step_name
        self.time = time
        self.substeps = substeps

def __repr__(self):
        return "StepTime(%r, %r, %r, %r)" % (
            self.step_id, self.step_name,
            self.time, self.substeps
        )

def as_dict(self):
        return {
            'step_id': self.step_id,
            'step_name': self.step_name,
            'time': self.time,
            'substeps': [s.as_dict() for s in self.substeps]
        }

class CommandTime(object):
    """Object representing execution time of some remote or local command"""

def __init__(self, command, host, time):
        """Object constructor

Arguments:
        - command - string representation of a command (for example, '/bin/sh mkdir -p /tmp')
        - host - title of a host where the command was executed (example: 'source Plesk server')
        - time - command execution time in seconds as floating-point number
        """
        self.command = command
        self.host = host
        self.time = time

def __repr__(self):
        return "CommandTime(%r, %r, %r)" % (
            self.command, self.host, self.time
        )

class APICallTime(object):
    """Object representing execution time of some API call"""

def __init__(self, host, api, method, time):
        """Object constructor

Arguments:
        - host - title of a host where the command was executed (example: 'source Plesk server')
        - api - name of API as text string
        - method - method of API called
        - time - API call execution time in seconds as floating-point number
        """
        self.host = host
        self.api = api
        self.method = method
        self.time = time

def __repr__(self):
        return 'APICallTime(%r, %r, %r, %r)' % (
            self.host, self.api, self.method, self.time
        )

class WaitingTime(object):
    """Object representing waiting (sleep)"""

def __init__(self, description, time):
        """Object constructor

Arguments:
        - description - description of waiting reason
        - time - API call execution time in seconds as floating-point number
        """
        self.description = description
        self.time = time

def __repr__(self):
        return 'WaitingTime(%r, %r)' % (
            self.description, self.time
        )

class TimeGap(object):
    def __init__(self, tag, title, start_time, end_time):
        self.tag = tag
        self.title = title
        self.start_time = start_time
        self.end_time = end_time

def as_dict(self):
        return {
            'tag': self.tag,
            'title': self.title,
            'start_time': self.start_time,
            'end_time': self.end_time
        }

class StepsProfiler(object):
    """Collects execution time information for migration steps
    
    The profiler considers steps hierarchy
    """

def __init__(self):
        self._current_step = StepTime(
            step_id='root', step_name='Root', time=None, substeps=[]
        )
        self._command_calls = []
        self._api_calls = []
        self._waitings = []
        self._time_gaps = []

@contextmanager
    def measure_time_gap(self, tag, step_id):
        start_time = time.time()
        try:
            yield
        finally:
            end_time = time.time()
            self._time_gaps.append(TimeGap(tag, step_id, start_time, end_time))

@contextmanager
    def measure_time(self, step_id, step_name=None):
        """Measure time of some particular step
        
        Usage example:
        with steps_profiles.measure_time(
            'restore-hosting', 'Restore hosting settings'
        ):
            # run restore hosting step
        """
        parent_step = self._current_step
        self._current_step = find_first(
            parent_step.substeps, lambda s: s.step_id == step_id
        )
        if self._current_step is None:
            self._current_step = StepTime(
                step_id=step_id, 
                step_name=step_name, 
                time=0.0, 
                substeps=[]
            )
            parent_step.substeps.append(self._current_step)
        start_time = time.time()
        try:
            yield
        finally:
            end_time = time.time()
            self._current_step.time += (end_time - start_time)
            self._current_step = parent_step

@contextmanager
    def measure_command_call(self, command, host):
        if not StepsProfilerEnvironment().profile_command_calls:
            yield
        else:
            start_time = time.time()
            try:
                yield
            finally:
                end_time = time.time()
                execution_time = end_time - start_time
                self._command_calls.append(CommandTime(
                    command, host, execution_time
                ))

@contextmanager
    def measure_api_call(self, host, api, method):
        if not StepsProfilerEnvironment().profile_api_calls:
            yield
        else:
            start_time = time.time()
            try:
                yield
            finally:
                end_time = time.time()
                execution_time = end_time - start_time
                self._api_calls.append(APICallTime(
                    host, api, method, execution_time
                ))

@contextmanager
    def measure_waiting(self, description):
        if not StepsProfilerEnvironment().profile_waitings:
            yield
        else:
            start_time = time.time()
            try:
                yield
            finally:
                end_time = time.time()
                execution_time = end_time - start_time
                self._waitings.append(WaitingTime(
                    description, execution_time
                ))

def get_root_step(self):
        """Get measured times as instance of root StepTime object"""
        return self._current_step

def get_command_calls(self):
        return self._command_calls

def get_api_calls(self):
        return self._api_calls

def get_waitings(self):
        return self._waitings

def get_time_gaps(self):
        return self._time_gaps

class StepsProfilerReport(object):
    """Print reports of execution times for migration steps"""

def __init__(self, profiler):
        """Arguments:
        - profiler - instance of StepsProfiler to get data from
        """
        self._profiler = profiler
        self._plain_text_report_filename = None
        self._raw_report_filename = None
        self._time_gaps_data_filename = None
        self._migrated_subscriptions_count = None
        self._command_calls_report_filename = None
        self._waitings_report_filename = None
        self._api_calls_report_filename = None

def set_plain_text_report_filename(self, report_filename):
        """Set name of a file to store plain text steps report to"""
        self._plain_text_report_filename = report_filename

def set_time_gaps_data_filename(self, report_filename):
        self._time_gaps_data_filename = report_filename

def set_raw_data_filename(self, report_filename):
        """Set name of a file to store raw steps data to"""
        self._raw_report_filename = report_filename

def set_migrated_subscriptions_count(self, count):
        """Set number of subscriptions that were migrated during execution

This information is used to put per-subscription times into the report
        """
        self._migrated_subscriptions_count = count

def set_command_calls_report_filename(self, report_filename):
        """Set name of a file to store HTML report of command calls"""
        self._command_calls_report_filename = report_filename

def set_waitings_report_filename(self, report_filename):
        """Set name of a file to store HTML report of waitings"""
        self._waitings_report_filename = report_filename

def set_api_calls_report_filename(self, report_filename):
        """Set name of a file to store HTML report of API calls"""
        self._api_calls_report_filename = report_filename

def save_reports(self):
        """Save profile reports to files"""
        self._save_plain_text_report()
        self._save_time_gaps_data()
        self._save_raw_data()
        self._save_command_calls_html_report()
        self._save_api_calls_html_report()
        self._save_waitings_html_report()
        
    def _save_raw_data(self):
        if self._raw_report_filename is None:
            # No raw data filename is specified, so we can't write the data.
            # Consider we just don't want to write it.
            return

write_json(
            self._raw_report_filename, 
            self._profiler.get_root_step().as_dict(),
            pretty_print=True
        )

def _save_plain_text_report(self):
        if self._plain_text_report_filename is None:
            # No plain text report filename is specified, so we can't write
            # report. Consider we just don't want to write it.
            return

with open_no_inherit(self._plain_text_report_filename, 'w') as fp:
            fp.write(messages.PROFILING_RESULTS + "\n")
            for step in self._profiler.get_root_step().substeps:
                self._save_plain_text_step(fp, step)

if all([
                StepsProfilerEnvironment().profile_command_calls,
                StepsProfilerEnvironment().profile_api_calls,
                StepsProfilerEnvironment().profile_waitings
            ]):
                total_time_outside = sum([
                    i.time for i in itertools.chain(
                        self._profiler.get_api_calls(),
                        self._profiler.get_command_calls(),
                        self._profiler.get_waitings()
                    )
                ])
                total_time = sum([
                    step.time
                    for step in self._profiler.get_root_step().substeps
                ])
                
                fp.write("\n\n\n")
                fp.write(
                    messages.PROFILE_TOTAL_TIME % (
                        self._format_seconds(total_time)
                    ) + "\n"
                )
                fp.write(
                    messages.TOTAL_TIME_OUTSIDE_MIGRATOR_API_CALLS % (
                        self._format_seconds(total_time_outside)
                    ) + "\n"
                )
                fp.write(
                    messages.PROFILE_TIME_INSIDE_MIGRATOR % (
                        self._format_seconds(total_time - total_time_outside)
                    ) + "\n"
                )

def _save_time_gaps_data(self):
        if self._time_gaps_data_filename is not None:
            write_json(
                self._time_gaps_data_filename,
                [t.as_dict() for t in self._profiler.get_time_gaps()],
                pretty_print=True
            )

def _save_command_calls_html_report(self):
        if self._command_calls_report_filename is None:
            # No raw data filename is specified, so we can't write the report.
            # Consider we just don't want to write it.
            return

if not StepsProfilerEnvironment().profile_command_calls:
            return

with open_no_inherit(self._command_calls_report_filename, 'w') as fp:
            self._write_html_table(
                fp, ['host', 'command', 'time'], 
                sorted(
                    self._profiler.get_command_calls(),
                    key=lambda c: c.time,
                    reverse=True
                )
            )
            self._write_total_time(fp, self._profiler.get_command_calls())

def _save_api_calls_html_report(self):
        if self._api_calls_report_filename is None:
            # No raw data filename is specified, so we can't write the report.
            # Consider we just don't want to write it.
            return

if not StepsProfilerEnvironment().profile_api_calls:
            return

with open_no_inherit(self._api_calls_report_filename, 'w') as fp:
            self._write_html_table(
                fp, ['api', 'host', 'method', 'time'], 
                sorted(
                    self._profiler.get_api_calls(),
                    key=lambda c: c.time, 
                    reverse=True
                )
            )
            self._write_total_time(fp, self._profiler.get_api_calls())

def _save_waitings_html_report(self):
        if self._waitings_report_filename is None:
            # No raw data filename is specified, so we can't write the report.
            # Consider we just don't want to write it.
            return

if not StepsProfilerEnvironment().profile_waitings:
            return

with open_no_inherit(self._waitings_report_filename, 'w') as fp:
            self._write_html_table(
                fp, ['description', 'time'], 
                sorted(
                    self._profiler.get_waitings(),
                    key=lambda c: c.time, 
                    reverse=True
                )
            )
            self._write_total_time(fp, self._profiler.get_waitings())

def _save_plain_text_step(self, fp, step, indent=0, parent_time=None):
        """Save single migration step in plain-text to a file

Arguments:
        - fp - file handle to write information to
        - step - instance of StepTime
        - indent - indentation of step (numeric)
        - parent_time - time of a parent step, to could percentage of time
        spent on each substep
        """

def print_line(title, time, percentage, indent):
            if (
                self._migrated_subscriptions_count is not None
                and self._migrated_subscriptions_count != 0
            ):
                per_subscription = ', %s' % (
                    messages.PROFILE_PER_SUBSCRIPTION % (
                        self._format_seconds(
                            time / self._migrated_subscriptions_count
                        )
                    ),
                )
            else:
                per_subscription = ''

fp.write("%s%s: %s (%s) total%s\n" % (
                '\t' * indent, 
                title, self._format_seconds(time), 
                (
                    "%.0f%%" % (percentage * 100.0) 
                    if percentage is not None 
                    else 'N/A%'
                ),
                per_subscription
            ))

if step.step_name is not None:
            step_title = self._uppercase_first_letter(step.step_name)
        else:
            step_title = step.step_id

if parent_time is None:
            # if that is a root step - consider 100%
            step_percantage = 1.0
        else:
            if parent_time == 0.0:
                # if parent time took no time (exactly 0.0!)
                step_percantage = None
            else:
                step_percantage = step.time / parent_time

print_line(
            title=step_title, 
            time=step.time, indent=indent, 
            percentage=step_percantage
        )

if len(step.substeps) > 0:
            for substep in step.substeps:
                self._save_plain_text_step(fp, substep, indent+1, step.time)

substeps_time = sum(substep.time for substep in step.substeps)
            other_time = step.time - substeps_time

print_line(
                title=messages.PROFILE_OTHER_TIME,
                time=other_time, 
                percentage=other_time / step.time if step.time != 0.0 else None,
                indent=indent+1
            )

def _write_total_time(self, fp, items):
        total = sum([i.time for i in items])
        fp.write('<b>%s</b>' % (messages.PROFILE_TOTAL_TIME % self._format_seconds(total)))

@staticmethod
    def _write_html_table(fp, headers, rows):
        fp.write(u'<html><body><table border="1">')
        fp.write(u'<tr>\n')
        for header in headers:
            fp.write(u'<th>%s</th>\n' % header)
        fp.write(u'</tr>\n')

for row in rows:
            fp.write(u'<tr>\n')
            for header in headers:
                fp.write(u'<td>%s</td>\n' % u"\n" .join(wrap(unicode(getattr(row, header)))))
            fp.write(u'</tr>\n')
        fp.write(u'</table></body></html>')

@staticmethod
    def _uppercase_first_letter(string):
        if len(string) == 0:
            return string

return string[0].upper() + string[1:]

@staticmethod
    def _format_seconds(seconds):
        return str(datetime.timedelta(seconds=round(seconds)))

class StepsProfilerEnvironment(object):
    @property
    def profile_api_calls(self):
        return os.getenv('MIGRATOR_PROFILE_API_CALLS') == '1'

@property
    def profile_command_calls(self):
        return os.getenv('MIGRATOR_PROFILE_COMMAND_CALLS') == '1'

@property
    def profile_waitings(self):
        return os.getenv('MIGRATOR_PROFILE_WAITINGS') == '1'

def sleep(seconds, description):
    profiler = get_default_steps_profiler()
    with profiler.measure_waiting(description):
        time.sleep(seconds)

def configure_report_locations(profiler, server, command_name):
    """Configure locations of profile data files and reports

:type profiler: parallels.core.utils.steps_profiler.StepsProfilerReport
    :type server: parallels.core.connections.migrator_server.MigratorServer
    :type command_name: str | unicode | None
    """
    if not is_empty(command_name):
        profile_suffix = '%s.' % (command_name,)
    else:
        profile_suffix = ''

profile_suffix += datetime.datetime.now().strftime("%Y.%m.%d.%H.%M.%S")

directory = server.get_session_file_path('profile')
    with server.runner() as runner:
        runner.mkdir(directory)

profiler.set_plain_text_report_filename(
        os.path.join(directory, 'report.%s.txt' % profile_suffix)
    )
    profiler.set_command_calls_report_filename(
        os.path.join(directory, 'command-calls.%s.html' % profile_suffix)
    )
    profiler.set_api_calls_report_filename(
        os.path.join(directory, 'api-calls.%s.html' % profile_suffix)
    )
    profiler.set_waitings_report_filename(
        os.path.join(directory, 'waitings.%s.html' % profile_suffix)
    )
    profiler.set_raw_data_filename(
        os.path.join(directory, 'action-tree.%s.json' % profile_suffix)
    )
    profiler.set_time_gaps_data_filename(
        os.path.join(directory, 'time-gaps.%s.json' % profile_suffix)
    )

📁 Sid Gifari File Manager