MAINHACK

Viewing / Editing File: acronis.py

File is not writable. Editing disabled.
"""Interface to Acronis file restoration API"""

import functools
import json
import logging
import operator
import os
import platform
import re
import resource
import socket
import time
import urllib.parse as urlparse  # path differs from python 2.7!
from contextlib import closing
from itertools import chain
from multiprocessing import Pool
from pathlib import Path
from subprocess import PIPE, Popen, run
from typing import Dict, Iterable, List, Optional
from xml.etree import ElementTree

import requests

from restore_infected import __version__
from restore_infected.backup_backends_lib import (
    BackendNotAuthorizedError,
    backend_auth_required,
    backup_client_required,
)
from .. import helpers, package_manager
from ..backup_backends_lib import asyncable, extra
from ..helpers import from_env

REPEAT_NUM = 30
REPEAT_WAIT = 5  # seconds
BACKUP_ATTEMPTS = 2

TOKEN_FILE = '/var/restore_infected/acronis_api_token.json'

auth_required = backend_auth_required(
    TOKEN_FILE, "Initialize Acronis first!")
client_required = backup_client_required(
    '/usr/lib/Acronis/BackupAndRecovery/mms',
    'Acronis client should be installed first!')


def is_suitable():
    return True


class FileDeliverError(Exception):
    pass


class BackupFailed(Exception):
    pass


class TokenExistsError(Exception):
    pass


class ClientAlreadyInstalledError(Exception):
    pass


class RequestFailed(Exception):
    pass


def check_response(response):
    """
    Exit with error status unless response code is 200
    :param response: obj -> response object
    """
    try:
        response.raise_for_status()
    except requests.HTTPError as e:
        if e.response.status_code == 401:
            raise BackendNotAuthorizedError("Token is invalid") from e
        raise e


class AcronisConnector:
    api_path = 'api/1'
    entrypoint = 'https://cloud.acronis.com'
    refresh_interval = 86400  # one day

    # URL templates
    accounts_tpl = '{}/{}/accounts?login={}'
    token_api_tpl = '{}/api/idp/v1/token'
    login_tpl = '{}/{}/login'
    logout_tpl = '{}/{}/logout'
    bredirect_url_tpl = '{}/{}/groups/{}/backupconsole'
    backup_url_api_tpl = '{}api/ams'

    token_path = TOKEN_FILE  # for backward compatibility

    def __init__(self, hostname=None):
        self.hostname = hostname if hostname else socket.gethostname()
        self._session = requests.Session()
        self._session.headers.update(
            {"User-Agent": "cloudlinux-backup-utils v{}".format(__version__)}
        )
        self._logged_in = False
        self.server_url = None

    @classmethod
    def connect(cls, hostname=None):
        _class = cls(hostname)
        _class.login()
        return _class

    def get(self, url, **kw):
        return self._session.get('{}/{}'.format(self.base_url, url), **kw)

    def post(self, url, **kw):
        return self._session.post('{}/{}'.format(self.base_url, url), **kw)

    def put(self, url, **kw):
        return self._session.put('{}/{}'.format(self.base_url, url), **kw)

    def login(self):
        """
        Creates a session to work with restoration API
        """
        self._login_to_api()
        self._get_url()
        self._logged_in = True

    def logout(self):
        """
        Destroys the session
        """
        url = self.logout_tpl.format(self.server_url, self.api_path)
        check_response(self._session.post(url))
        self._logged_in = False

    @classmethod
    def create_token(cls, username, password, force=False):
        """
        Gets and saves authentication token using provided credentials
        :param username: str -> username
        :param password: str -> password
        :param force: bool -> overwrite token file
        """
        if os.path.exists(TOKEN_FILE) and not force:
            raise TokenExistsError("Token file already exists. If you want to "
                                   "override it, delete '{}' manually and run "
                                   "init again.".format(TOKEN_FILE))
        server_url = cls.get_server_url(username)
        url = cls.token_api_tpl.format(server_url)
        params = {
            'grant_type': 'password',
            'username': username,
            'password': password,
            'client_id': 'backupAgent'}

        r = requests.post(url, data=params)
        check_response(r)

        data = r.json()
        data['timestamp'] = int(time.time())
        data['username'] = username
        for key in 'token_type', 'access_token':
            data.pop(key, None)
        cls._save_token(data)

    def refresh_token(self):
        """
        Refreshes old token with old token refresh key
        """
        self._refresh_token()

    @staticmethod
    def get_saved_token():
        """Reads token from file"""
        try:
            with open(TOKEN_FILE) as f:
                return json.load(f)
        except (OSError, IOError, ValueError, TypeError) as e:
            raise Exception("Could not read token: {}".format(str(e)))

    def _login_to_api(self):
        """Log into common API"""
        if self._logged_in:
            return

        token_data = self.get_saved_token()
        if token_data is None or 'username' not in token_data:
            raise Exception('Missing or incomplete credentials. Exit')

        if not self.server_url:
            self.server_url = self.get_server_url(token_data['username'])

        if time.time() - token_data['timestamp'] > self.refresh_interval:
            self._refresh_token(token_data)

        url = self.login_tpl.format(self.server_url, self.api_path)
        params = {'jwt': token_data.get('id_token')}
        r = self._session.get(url, params=params)
        check_response(r)
        self._user_data = r.json()

    def _get_url(self):
        """After common API login get redirected to backup API"""
        if hasattr(self, 'url'):
            return
        if not hasattr(self, '_user_data'):
            raise Exception("No user data. Call basic_login first")

        url = self.bredirect_url_tpl.format(
            self.server_url, self.api_path,
            self._user_data.get('group', {}).get('id'))

        r = self._session.get(url)
        check_response(r)
        redirect_url = r.json().get('redirect_url')
        r = self._session.get(redirect_url)
        self.base_url = self.backup_url_api_tpl.format(r.url)
        self._subscription_url = self.base_url.replace('ams', 'subscriptions')

    @classmethod
    def get_server_url(cls, username):
        """
        To start working we have to get our working server URL
        from the main entrypoint
        :param username: str -> username to get working server
        :return: str -> working server URL
        """
        server_url = prev_server_url = cls.entrypoint
        for _ in range(10):
            url = cls.accounts_tpl.format(server_url, cls.api_path, username)
            r = requests.get(url)
            check_response(r)
            server_url = r.json().get("server_url")
            if server_url == prev_server_url:
                break
            prev_server_url = server_url
        return server_url

    def _refresh_token(self, token=None):
        """
        Gets new token and passes it for saving
        :param token: dict -> old token data as dict
        :return: dict -> refreshed token
        """
        if token is None:  # no token passed
            token = self.get_saved_token()
        if token is None:  # getting token from file failure
            raise Exception('Could not get saved token to refresh')
        if not self.server_url:
            self.server_url = self.get_server_url(token['username'])
        url = self.token_api_tpl.format(self.server_url)
        params = {
            'grant_type': 'refresh_token',
            'refresh_token': token['refresh_token'],
            'client_id': 'backupAgent'}
        r = self._session.post(url, data=params)
        check_response(r)
        data = r.json()

        for key in 'refresh_token', 'id_token':
            token[key] = data.get(key)
        token['timestamp'] = int(time.time())
        self._save_token(token)
        return token

    @classmethod
    def _save_token(cls, token):
        """
        Saves token to file
        :param token: dict -> token data to be saved
        :return: bool -> True if succeeded and False otherwise
        """
        helpers.mkdir(os.path.dirname(TOKEN_FILE))
        try:
            with open(TOKEN_FILE, 'w') as f:
                json.dump(token, f)
        except (OSError, IOError, ValueError, TypeError) as e:
            raise Exception("Could not save token to file: {}".format(e))
        return True

    def get_login_url(self):
        """
        Get login url to sign up without username/password prompt
        :return: str -> login url
        """
        token_data = self._refresh_token()
        id_token = token_data.get('id_token')
        if not id_token:
            raise Exception("Error obtaining login token")
        username = token_data['username']
        if not self.server_url:
            self.server_url = self.get_server_url(username)
        return "{}?jwt={}".format(self.server_url, id_token)

    def subscription(self, sid=None, params=None):
        if params is None:
            params = {}
        url = '/'.join(filter(None, (self._subscription_url, sid)))
        headers = {'Content-Type': 'application/json'}
        r = self._session.post(url, headers=headers, json=params)
        check_response(r)
        return r.json()


@functools.total_ordering
class FileData:
    def __init__(self, item_id, filename, size, mtime):
        self.id = item_id
        self.filename = filename
        self.size = size
        self.mtime = mtime

    def __str__(self):
        return "{}; {}; {}; {}".format(
            self.id, self.filename, self.size, self.mtime.isoformat(),
        )

    def __repr__(self):
        return (
            "<{__class__.__name__}"
            " (filename={filename!s}, mtime={mtime!r})>".format(
                __class__=self.__class__, **self.__dict__
            )
        )

    def __lt__(self, other):
        return self.mtime < other.mtime

    def __eq__(self, other):
        return self.mtime == other.mtime

    def __hash__(self):
        return hash((self.id, self.filename, self.size, self.mtime))


class AcronisRestoreItem(FileData):
    def __init__(self, item, restore_prefix, destination_folder):
        super().__init__(
            item_id=item.id,
            filename=item.filename,
            size=item.size,
            mtime=item.mtime)
        self.restore_prefix = restore_prefix
        self.destination_folder = destination_folder

    @property
    def restore_path(self):
        # /tmp/tmpdir234234/home/user/public_html/blog/index.php
        return os.path.join(
            self.destination_folder,
            os.path.relpath(self.filename, self.restore_prefix)
        )

    @classmethod
    def get_chunks(
        cls, items, n: int, temp_dir: str
    ) -> Iterable[List["AcronisRestoreItem"]]:
        common_prefix = common_path(items)
        offset = 0
        chunk = items[offset: offset + n]
        while chunk:
            restore_prefix = common_path(chunk)
            destination_folder = os.path.join(
                temp_dir, os.path.relpath(restore_prefix, common_prefix)
            )
            yield [
                cls(item, restore_prefix, destination_folder)
                for item in chunk
            ]
            offset += n
            chunk = items[offset: offset + n]


def common_path(items):
    if len(items) == 1:
        return os.path.dirname(items[0].filename)

    return os.path.commonpath([item.filename for item in items])


class AcronisBackup:
    _contents_tpl = (
        "archives/<patharchiveId>/backups/<pathbackupId>/items?"
        "type={}&machineId={}&backupId={}&id={}"
    )
    _restore_session_tpl = "archives/downloads?machineId={}"
    _download_item_tpl = (
        "archives/downloads/{}/<pathfileName>?"
        "fileName={}&format={}&machineId={}&start_download=1"
    )
    _draft_request_tpl = "recovery/plan_drafts?machineId={}"
    _target_loc_tpl = "recovery/plan_drafts/{}/target_location?machineId={}"
    _create_recovery_plan_tpl = "recovery/plan_drafts/{}"
    _run_recovery_plan_tpl = "recovery/plan_operations/run?machineId={}"
    default_download_dir = '/tmp'
    jheaders = {"Content-Type": "application/json"}
    restore_fmt = "PLAIN"
    chunk_size = 50

    def __init__(self, conn, host_id, backup_id, created):
        self._conn = conn
        self.host_id = host_id
        self.backup_id = backup_id
        self.created = created
        logging.getLogger(__name__).addHandler(logging.NullHandler())

    def __str__(self):
        return "{}".format(self.created.isoformat())

    def __repr__(self):
        return "<{__class__.__name__} (created={created!r})>".format(
            __class__=self.__class__, **self.__dict__
        )

    def __eq__(self, other):
        return self.created == other.created

    def _get_draft_request(self, items):
        payload = {
            'backupId': self.backup_id,
            'operationId': 'rec-draft-2-4',
            'machineId': self.host_id,
            'resourcesIds': [item.id for item in items]}
        url = self._draft_request_tpl.format(self.host_id)
        response = self._conn.post(url, headers=self.jheaders, json=payload)
        check_response(response)
        return response.json()

    def _set_target_location(self, draft, destination_folder: str):
        payload = {
            'machineId': self.host_id,
            'type': 'local_folder',
            'path': destination_folder}
        url = self._target_loc_tpl.format(draft.get('DraftID'), self.host_id)
        response = self._conn.put(url, headers=self.jheaders, json=payload)
        check_response(response)

    def _create_recovery_plan(self, draft):
        payload = {'operationId': 'rec-createplan-1-1'}
        url = self._create_recovery_plan_tpl.format(draft.get('DraftID'))
        r = self._conn.post(url, headers=self.jheaders, json=payload)
        check_response(r)
        return r.json()

    def _run_recovery_plan(self, plan):
        payload = {'machineId': self.host_id,
                   'operationId': 'noWait',
                   'planId': plan.get('PlanID')}
        url = self._run_recovery_plan_tpl.format(self.host_id)
        r = self._conn.post(url, headers=self.jheaders, json=payload)
        check_response(r)

    def _get_restore_session(self, file_path):
        """
        Gets session for downloading a selected item
        :return: str -> session ID
        """
        url = self._restore_session_tpl.format(self.host_id)
        json_params = {
            'format': self.restore_fmt,
            'machineId': self.host_id,
            'backupId': self.backup_id,
            'backupUri': self.backup_id,
            'items': [file_path]}
        r = self._conn.post(url, json=json_params)
        check_response(r)
        return r.json().get('SessionID')

    def _download_backup_item(self, session_id, item: AcronisRestoreItem):
        """
        Downloads chosen item from acronis backup to current host
        :param session_id: str -> download session token
        """
        file_name = os.path.basename(item.filename)
        url = self._download_item_tpl.format(
            session_id, file_name, self.restore_fmt, self.host_id
        )
        r = self._conn.get(url)
        check_response(r)
        os.makedirs(
            os.path.dirname(item.restore_path), mode=0o700, exist_ok=True
        )
        with open(item.restore_path, "wb") as f:
            f.write(r.content)

    def _get_path_item(self, host_id, backup_id, wanted_path, path_id=None):
        """
        Recursively gets path items
        :param wanted_path: str -> file path to restore
        :param path_id: str -> acronis path ID or None
        :return: dict -> found backup item or None
        """
        wanted_path_parents = Path(wanted_path).parents

        try:
            result = self._get_path_item_inner(host_id, backup_id, path_id)
            if not result:
                return  # There are no data. So return
        except RequestFailed:
            return

        for item in result:
            item_path = _strip_device_name(item.get('fullPath'))
            if item_path == wanted_path:  # We've found item we search for
                return item

            if Path(item_path) == Path('/'):  # If current path is a root path
                # it means that we selected some volume. We will try to find
                # needed file in that volume, if failed -- we have to look in
                # another volume
                return_value = self._get_path_item(
                    host_id, backup_id, wanted_path, item.get('id')
                )
                if return_value:
                    # We found file in the current volume
                    return return_value
                else:
                    # Look in the another volume
                    continue

            elif Path(item_path) in wanted_path_parents:
                # we're on the right way
                return self._get_path_item(
                    host_id, backup_id, wanted_path, item.get('id')
                )

    def __hash__(self):  # quick hack for lru_cache
        return hash((type(self), self.host_id, self.backup_id))

    @functools.lru_cache(maxsize=128)
    def _get_path_item_inner(self, host_id, backup_id, path_id=None):
        """
        Gets path items
        :param host_id: str -> acronis host ID
        :param backup_id: str -> acronis backup ID
        :param path_id: str -> acronis path ID or None
        :return: list -> found backup items
        :raise: RequestFailed
        """
        if path_id is None:
            path_id = ''
        backup_type = 'files'
        url = self._contents_tpl.format(
            backup_type, host_id, urlparse.quote_plus(backup_id),
            urlparse.quote_plus(path_id))

        params_for_json = {
            'id': path_id,
            'backupId': backup_id,
            'machineId': host_id,
            'type': backup_type}
        try:
            r = self._conn.post(url, json=params_for_json)
        except requests.exceptions.ConnectionError as e:
            logging.error('Fetching backup data failure: {}'.format(e))
            # we don't want it to get cached
            raise RequestFailed('Fetching backup data failure: {}'.format(e))
        if r.status_code != 200:
            raise RequestFailed(
                'Fetching backup data failure: status code {}'.format(
                    r.status_code,
                ),
            )

        result = r.json()
        if result.get('total') == 0 or len(result.get('data', [])) == 0:
            return []

        return [{
            'id': item.get('id'),
            'fullPath': item.get('fullPath'),
            'size': item.get('size'),
            'lastChange': item.get('lastChange'),
        } for item in result.get('data', [])]

    def _get_backup_contents(self, paths):
        """
        Frontend for recursive backup contents getter
        :param paths: list -> list of file paths to restore
        """
        runner = functools.partial(
            self._get_path_item, self.host_id, self.backup_id)
        total = len(paths)
        pool = Pool(5 if total > 5 else total)
        results = pool.map(runner, paths)
        self.contents = []
        for idx, value in enumerate(results):
            if value is None:
                raise FileNotFoundError(paths[idx])
            self.contents.append(
                FileData(
                    item_id=value.get('id'),
                    filename=value.get('fullPath'),
                    size=value.get('size'),
                    mtime=helpers.DateTime(value.get('lastChange', '')),
                )
            )

    def file_data(self, path):
        result = self._get_path_item(self.host_id, self.backup_id, path)
        if result is None:
            raise FileNotFoundError(path)
        return FileData(
            item_id=result.get('id'),
            filename=_strip_device_name(result.get('fullPath')),
            size=result.get('size'),
            mtime=helpers.DateTime(result.get('lastChange', '')),
        )

    @staticmethod
    def wait_for_arrival(items, timeout):
        yet_to_arrive = set(items)
        start = time.monotonic()
        while time.monotonic() - start < timeout:
            yet_to_arrive = {
                item
                for item in yet_to_arrive
                if not (
                    os.path.exists(item.restore_path)
                    and item.size == os.stat(item.restore_path).st_size
                )
            }
            if not yet_to_arrive:
                break
            time.sleep(1)
        return yet_to_arrive

    def restore(
        self,
        items: List[FileData],
        destination_folder: Optional[str],
        timeout=600,
        use_download=False,
    ) -> Dict[str, str]:
        if not items:
            return {}
        destination_folder = destination_folder or self.default_download_dir
        if use_download:
            # FIXME: figure out how to download the items in bulk as well
            restore_prefix = common_path(items)
            restore_items = [
                AcronisRestoreItem(item, restore_prefix, destination_folder)
                for item in items
            ]
            for item in restore_items:
                session_id = self._get_restore_session(item.filename)
                self._download_backup_item(session_id, item)
        else:
            chunk_list = list(
                AcronisRestoreItem.get_chunks(
                    items, self.chunk_size, destination_folder
                )
            )
            for chunk in chunk_list:
                recovery_draft = self._get_draft_request(chunk)
                self._set_target_location(
                    recovery_draft, chunk[0].destination_folder
                )
                recovery_plan = self._create_recovery_plan(recovery_draft)
                self._run_recovery_plan(recovery_plan)

            restore_items = list(chain.from_iterable(chunk_list))

        did_not_arrive = self.wait_for_arrival(restore_items, timeout)
        if did_not_arrive:
            raise FileDeliverError(
                "Some files ({}) have not been "
                "delivered in a given time slot ({})"
                .format(did_not_arrive, timeout)
            )
        for item in restore_items:
            atime = helpers.DateTime.now().timestamp()
            mtime = item.mtime.timestamp()
            os.utime(item.restore_path, (atime, mtime))

        return {item.restore_path: item.filename for item in restore_items}

    def close(self):
        pass


class AcronisAPI:
    _archives_tpl = ('locations/<pathlocationId>/archives?'
                     'locationId={}&machineId={}')
    _backups_tpl = ('archives/<patharchiveId>/backups?'
                    'archiveId={}&machineId={}')
    _region_pattern = re.compile(
        r'(?P<region>\w+)(?:-cloud\.acronis|\.cloudlinuxbackup)\.com'
    )
    _backup_and_recovery_config_path = '/etc/Acronis/BackupAndRecovery.config'

    backup_log_path = '/var/restore_infected/acronis_backup.log'

    default_start_time = {'hour': 5, 'minute': 0, 'second': 0}

    mysql_freeze_options = {
        'prePostCommands': {
            'preCommands': {
                'command': '',
                'commandArguments': '',
                'continueOnCommandError': False,
                'waitCommandComplete': True,
                'workingDirectory': '',
            },
            'postCommands': {
                'command': '',
                'commandArguments': '',
                'continueOnCommandError': False,
                'waitCommandComplete': True,
                'workingDirectory': '',
            },
            'useDefaultCommands': True,
            'usePreCommands': False,
            'usePostCommands': False,
        },
        'prePostDataCommands': {
            'preCommands': {
                'command':
                    '/usr/libexec/cloudlinux-backup/pre-mysql-freeze-script',
                'commandArguments': '',
                'continueOnCommandError': False,
                'waitCommandComplete': True,
                'workingDirectory': '',
            },
            'postCommands': {
                'command':
                    '/usr/libexec/cloudlinux-backup/post-mysql-thaw-script',
                'commandArguments': '',
                'continueOnCommandError': False,
                'waitCommandComplete': True,
                'workingDirectory': '',
            },
            'useDefaultCommands': False,
            'usePreCommands': True,
            'usePostCommands': True,
        }
    }

    scheme = {
        'schedule': {
            'daysOfWeek': [
                'sunday',
                'monday',
                'tuesday',
                'wednesday',
                'thursday',
                'friday',
                'saturday',
            ],
            'startAt': default_start_time,
            'type': 'weekly'
        },
        'type': 'alwaysIncremental'
    }

    def __init__(self, hostname=None, subscribe=False):
        self.hostname = hostname if hostname else socket.gethostname()
        self._conn = AcronisConnector.connect(self.hostname)
        if subscribe:
            self._sid = self._conn.subscription().get('id')
            if self._sid is None:
                raise Exception("Could not get subscription ID")
        logging.getLogger(__name__).addHandler(logging.NullHandler())

    @classmethod
    def make_initial_backup(cls, plan_name='CloudLinux Backup'):
        _class = cls(subscribe=True)
        _class._get_hostinfo()
        plan = _class._select_plan()  # Pick the first plan for this host
        if plan is not None:
            return _class._run_backup_plan(plan['id'])
        draft_id = _class._set_backup_plan_draft()
        params = {'planName': plan_name}
        _class._set_backup_plan_params(draft_id, params)
        _class._set_backup_plan_options(draft_id, cls.mysql_freeze_options)
        _class._set_backup_plan_scheme(draft_id, cls.scheme)
        plan_id = _class._create_backup_plan(draft_id)
        _class._run_backup_plan(plan_id)

    @classmethod
    def pull_backups(cls, until=None):
        _class = cls()
        _class._get_hostinfo()
        _class._get_host_archives()
        _class._get_archive_backups(until)
        return _class.backup_list

    @classmethod
    def migration_fix_backup_plan(cls):
        a = cls(subscribe=True)
        a._get_hostinfo()

        obsolete_time = {'hour': 18, 'minute': 0, 'second': 0}

        for plan in a._get_cloudlinux_plans():
            draft_id = a._create_draft_from_plan(plan)
            a._set_backup_plan_options(draft_id, cls.mysql_freeze_options)

            if cls._get_plan_time(plan) == obsolete_time:
                a._set_backup_plan_scheme(draft_id, cls.scheme)

            a._save_draft(draft_id)

    def _get_hostid(self, config_path=None):
        if hasattr(self, 'hostid'):
            return self.hostid

        if not config_path:
            config_path = self._backup_and_recovery_config_path

        et = ElementTree.parse(config_path)
        root = et.getroot()

        for settings in root:
            if settings.tag == 'key' and settings.get('name') == 'Settings':
                break
        else:
            raise Exception("Can't find 'Settings' key entry in config file "
                            "'%s'" % config_path)

        for machine_manager in settings:
            if machine_manager.tag == 'key' and \
                    machine_manager.get('name') == 'MachineManager':
                break
        else:
            raise Exception("Can't find 'MachineManager' key entry in "
                            "'Settings'")

        for value in machine_manager:
            if value.tag == 'value' and \
                    value.get('name') == 'MMSCurrentMachineID':
                break
        else:
            raise Exception("Can't find 'MMSCurrentMachineID' value entry in "
                            "'MachineManager'")

        self.hostid = value.text.strip().strip('"')
        return self.hostid

    def _refresh_hostinfo(self):
        """
        Refreshes the existing hostinfo or saves a new one if not present.
        Returns True if successful, False otherwise.
        """
        hostid = self._get_hostid()
        r = self._conn.get('resources')
        check_response(r)
        resources = r.json()
        for res in resources.get('data', []):
            if res.get('type') == 'machine' and res.get('hostId') == hostid:
                self.hostinfo = res
                return True

        return False

    def _get_hostinfo(self):
        """
        Gets backup host info from acronis.
        Exits with error message unless host found.
        """
        if hasattr(self, 'hostinfo'):
            return self.hostinfo
        if not _repeat(self._refresh_hostinfo):
            raise Exception("No backup for host {}".format(self.hostname))

    def get_backup_progress(self):
        """
        Returns the progress of a running backup in percents.
        """
        if self.is_backup_running():
            progress = self.hostinfo.get('progress')
            if progress:
                return int(progress)

    def get_backup_status(self):
        """
        Returns last backup status.
            * ok - backup succeeded
            * error - backup failed
            * unknown - there were no backups
            * warning - backup succeeded, but there are alerts
        """
        self._refresh_hostinfo()

        return self.hostinfo['status']

    def is_backup_running(self):
        """
        Returns True if there is a backup currently running or False otherwise.
        """
        if self._refresh_hostinfo():
            if self.hostinfo['state']:
                return self.hostinfo['state'][0] == 'backup'

            return False

        raise Exception("No backup for host {}".format(self.hostname))

    def get_alerts(self, as_json=True):
        """
        Get alerts for the current machine
        :param as_json: set to false to get alerts in human readable form
        :return list: list of alerts for the machine this backup belongs to
        """
        r = self._conn.get('alerts')
        check_response(r)
        alerts_json = r.json()
        alerts = [alert
                  for alert in alerts_json.get('data', [])
                  if alert.get('meta', []).get('machineName') == self.hostname]
        if as_json:
            return alerts

        alerts_as_str = ''
        for alert in alerts:
            data = ': '.join(
                filter(None, [alert.get('title'), alert.get('description')]))
            if data:
                if alerts_as_str:
                    alerts_as_str += '\n'
                alerts_as_str += data

        return alerts_as_str

    def _get_host_archives(self, location='cloud'):
        """
        Gets host archives. Required archive names are expected to start with
        hostname
        :param location: str -> location where archive is saved
        """
        self.archives = []
        if not hasattr(self, 'hostinfo'):
            raise Exception('No host data. Has "_get_hostinfo" been run?')
        host_id = self.hostinfo.get('hostId')
        url = self._archives_tpl.format(location, host_id)
        r = self._conn.get(url)
        check_response(r)
        archives = r.json()
        for archive in archives.get('data', []):
            if any([archive.get('archiveName', '').startswith(self.hostname),
                    archive.get('archiveMachineId') == host_id]):
                self.archives.append(archive)

    def _get_archive_backups(self, until=None):
        """
        Gets list of backups (not contents!) from the archive
        :param until: str -> timestamp
        """
        host_id = self.hostinfo.get('hostId')
        archive_ids = [i['id'] for i in self.archives]
        if until is not None and isinstance(until, str):
            until = helpers.DateTime(until)
        self.backup_list = []
        for archive_id in archive_ids:
            url = (self._backups_tpl.format(
                urlparse.quote_plus(archive_id), host_id))
            r = self._conn.get(url)
            if not r.ok:
                continue
            for backup in r.json().get('data', []):
                created = helpers.DateTime(backup.get('creationTime'))
                if until is not None and created < until:
                    continue
                self.backup_list.append(
                    AcronisBackup(
                        conn=self._conn,
                        host_id=host_id,
                        backup_id=backup.get('id'),
                        created=created))
        self.backup_list.sort(
            key=operator.attrgetter('created'), reverse=True)

    def _apply_default_plan(self):
        payload = {
            'resourcesIds': [self.hostinfo.get('id')]
        }
        r = self._conn.post('resource_operations/apply_default_backup_plan',
                            headers={'Content-Type': 'application/json'},
                            json=payload)
        return r.status_code

    def _get_applicable_plans(self):
        payload = {
            'resourcesIds': [self.hostinfo.get('id')]
        }
        r = self._conn.post(
            'backup/plan_draft_operations/get_applicable_plans',
            headers={'Content-Type': 'application/json'},
            json=payload,
        )
        check_response(r)
        return r.json().get('Plans', [])

    def _get_plans(self):
        if hasattr(self, '_backup_plans'):
            return self._backup_plans
        r = self._conn.get('backup/plans',
                           headers={'Content-Type': 'application/json'})
        check_response(r)
        self._backup_plans = r.json()['data']
        return self._backup_plans

    def _select_plan(self):
        if not hasattr(self, '_backup_plans'):
            self._get_plans()
        for plan in self._backup_plans:
            for source in plan['sources']['data']:
                hostid = self.hostinfo.get('hostId')
                source_id = self.hostinfo.get('id')
                if source.get('hostID') == hostid or \
                        hostid in source.get('machineId', '') or \
                        source_id == source.get('id', ''):
                    return plan

    def _set_backup_plan_draft(self):
        url = 'backup/plan_drafts'
        payload = {
            'subscriptionId': self._sid,
            'operationId': 'bc-draft-{}-0'.format(self._sid),
            'language': 'en',
            'resourcesIds': [self.hostinfo.get('id')],
            'defaultStartTime': self.default_start_time}
        return self._post_pop_request(url, payload)['DraftID']

    def _set_backup_plan_params(self, draft_id, payload):
        url = 'backup/plan_drafts/{}/parameters'.format(draft_id)
        headers = {'Content-Type': 'application/json'}
        r = self._conn.put(url, headers=headers, json=payload)
        check_response(r)

    def _set_backup_plan_options(self, draft_id, payload):
        url = 'backup/plan_drafts/{}/options'.format(draft_id)
        headers = {'Content-Type': 'application/json'}
        r = self._conn.put(url, headers=headers, json=payload)
        check_response(r)

    def _set_backup_plan_scheme(self, draft_id, payload):
        url = 'backup/plan_drafts/{}/scheme'.format(draft_id)
        headers = {'Content-Type': 'application/json'}
        r = self._conn.put(url, headers=headers, json=payload)
        check_response(r)

    def _post_pop_request(self, url, payload):
        headers = {'Content-Type': 'application/json'}
        r = self._conn.post(url, headers=headers, json=payload)
        check_response(r)

        params = {'action': 'pop', 'timeout': 10}
        data = _repeat(
            self._conn.subscription, **{'sid': self._sid, 'params': params})
        return data[0]['data']['result']

    def _create_backup_plan(self, draft_id):
        url = 'backup/plan_drafts_operations/create_plan'
        payload = {
            'subscriptionId': self._sid,
            'operationId': 'bc-createplan-{}-1'.format(self._sid),
            'draftId': draft_id}
        return self._post_pop_request(url, payload)['PlanID']

    @staticmethod
    def _get_plan_time(plan):
        return plan["scheme"].get("schedule", {}).get("startAt")

    @staticmethod
    def _get_plan_options(plan):
        return plan.get('options').get('backupOptions')

    @classmethod
    def _is_cloudlinux_plan(cls, plan):
        return 'cloudlinux' in str(cls._get_plan_options(plan))

    def _get_cloudlinux_plans(self):
        cl_plans = []
        for plan in self._get_plans():
            if self._is_cloudlinux_plan(plan):
                cl_plans.append(plan)
        return cl_plans

    def _create_draft_from_plan(self, plan):
        url = 'backup/plan_drafts'
        payload = {
            'language': 'en',
            'subscriptionId': self._sid,
            'operationId': 'bc-draft-{}-1'.format(self._sid),
            'planId': plan.get('id')}
        return self._post_pop_request(url, payload)['DraftID']

    def _save_draft(self, draft_id):
        url = 'backup/plan_drafts_operations/save_plan'
        payload = {
            'subscriptionId': self._sid,
            'operationId': 'bc-saveplan-{}-1'.format(self._sid),
            'draftId': draft_id}
        return self._post_pop_request(url, payload)

    def _run_backup_plan(self, plan_id):
        payload = {
            'planId': plan_id,
            'resourcesIds': [self.hostinfo.get('id')]}
        r = self._conn.post('backup/plan_operations/run',
                            headers={'Content-Type': 'application/json'},
                            json=payload)
        check_response(r)

    def get_info(self):
        """
        Get info from Acronis
        :return: dict
        """
        storage_info = self._conn.get('online_storage_information')
        check_response(storage_info)

        storage_info_data = storage_info.json()
        usage = storage_info_data.get("usage", {}).get("storageSize")

        if usage is None:
            for offering_item in storage_info_data.get("offering_items", []):
                if offering_item.get("usage_name") == "total_storage":
                    usage = offering_item.get("value")
                    break

        self._get_hostinfo()
        plan = self._select_plan()
        if plan and plan["enabled"]:
            scheme = plan["scheme"]
            if scheme["type"] == "custom":
                schedule = scheme.get("scheduleItems", [None])[0]
            else:
                schedule = scheme.get("schedule")
        else:
            schedule = None

        region_match = self._region_pattern.search(self._conn.base_url)
        region = region_match.groupdict()['region'] if region_match else None

        info_data = {
            'usage': usage,  # space used by backups
            'schedule': schedule,
            'region': region,
        }
        token_data = self._conn.get_saved_token()
        for key in 'username', 'timestamp':
            info_data[key] = token_data.get(key)
        return info_data


class AcronisClientInstaller:
    ACRONIS_BIN_LINK = '{endpoint}/bc/api/ams/links/agents/' \
                       'redirect?language=multi&system=linux&' \
                       'architecture={arch}&productType=enterprise'

    ACRONIS_BIN = 'acronis.bin'

    REG_BIN_OBSOLETE = '/usr/lib/Acronis/BackupAndRecovery/AmsRegisterHelper'
    REG_CMD_OBSOLETE = '{bin} register {endpoint} {login} {password}'

    REG_BIN = '/usr/lib/Acronis/RegisterAgentTool/RegisterAgent'
    REG_CMD = '{bin} -o register -t cloud -a {endpoint} ' \
              '-u {login} -p {password}'

    ACRONIS_DEPS = ('gcc', 'make', 'rpm')
    ACRONIS_INSTALL_BIN_CMD = './{bin} ' \
                              '--auto ' \
                              '--rain={endpoint} ' \
                              '--login={login} ' \
                              '--password={password}' \
                              '{optional}'
    MAKE_EXECUTABLE_CMD = 'chmod +x {}'
    INSTALL_LOG = '/var/restore_infected/acronis_installation_{time}.log'

    DTS_REPO_URL = 'https://people.centos.org/tru/devtools-2/devtools-2.repo'
    DTS_DEPS = ('devtoolset-2-gcc', 'devtoolset-2-binutils')
    DTS_BIN_PATH = '/opt/rh/devtoolset-2/root/usr/bin'

    YUM_REPO_DIR = '/etc/yum.repos.d'

    DKMS_CONFIG = '/etc/dkms/framework.conf'
    DKMS_PATH_VAR = 'export PATH={path}:$PATH' + os.linesep

    UNINSTALL_BIN = '/usr/lib/Acronis/BackupAndRecovery/uninstall/uninstall'
    UNINSTALL_CMD = (UNINSTALL_BIN, '--auto')

    def __init__(self, server_url, log_to_file=True):
        self.server_url = server_url
        self.logger = logging.getLogger(self.__class__.__name__)
        if log_to_file:
            log_file = self.INSTALL_LOG.format(time=time.time())
            self.logger.setLevel(logging.INFO)
            self.logger.addHandler(logging.FileHandler(log_file))

        self.arch = '64'
        if not platform.machine().endswith('64'):
            self.arch = '32'

        self.pm = package_manager.detect()

    def _exec(self, command):
        self.logger.debug('exec: `%s`', command)
        process = Popen(command.split(' '), stdout=PIPE, stderr=PIPE)
        out, err = process.communicate()
        exit_code = process.returncode
        self.logger.debug('Return code: %d\n\tSTDOUT:\n%s\n\tSTDERR:\n%s',
                          exit_code,
                          out.decode('utf-8').strip(),
                          err.decode('utf-8').strip())

        return exit_code, out.decode('utf-8')

    def _download_binary(self):
        self.logger.info('\tSaving from %s for x%s to acronis.bin',
                         self.server_url, self.arch)
        response = requests.get(
            self.ACRONIS_BIN_LINK.format(endpoint=self.server_url,
                                         arch=self.arch), allow_redirects=True,
            stream=True)

        response.raise_for_status()

        with open(self.ACRONIS_BIN, 'wb') as handle:
            for block in response.iter_content(1024):
                handle.write(block)

    @staticmethod
    def _check_rlimit_stack():
        """
        Acronis agent has wrappers for its binaries, changing stack size
        to 2048. Some kernels have bug preventing us from setting this size
        lower than 4096 (https://bugzilla.redhat.com/show_bug.cgi?id=1463241).
        This is a simple check to detect such kernels (normally /bin/true
        always successful and returns nothing)
        """
        try:
            p = Popen(['/bin/true'], preexec_fn=_pre_exec_check)
            p.wait()
        except OSError as e:
            if getattr(e, 'strerror', '') == 'Argument list too long':
                return False
            raise  # reraise if we face another error message
        return True

    def _install_binary(self, username, password, tmp_dir=None):
        self.logger.info('\tMaking %s executable', self.ACRONIS_BIN)
        self._exec(self.MAKE_EXECUTABLE_CMD.format(self.ACRONIS_BIN))

        self.logger.info('\tInstalling binary')
        res = self._exec(
            self.ACRONIS_INSTALL_BIN_CMD.format(
                bin=self.ACRONIS_BIN,
                endpoint=self.server_url,
                login=username,
                password=password,
                optional=' --tmp-dir=' + tmp_dir if tmp_dir else ''
            ))

        return res

    @classmethod
    def is_agent_installed(cls):
        return os.path.exists(cls.UNINSTALL_BIN)

    def install_client(self, username, password, force=False, tmp_dir=None):
        self.logger.info('Checking if agent is installable')
        if not self._check_rlimit_stack():
            raise Exception(
                "The backup agent cannot be installed on this system "
                "because of a bug in the current running kernel "
                "(see https://bugzilla.redhat.com/show_bug.cgi?id=1463241). "
                "Update kernel and try again. "
                "Use KernelCare to apply kernel updates without a reboot -- "
                "https://kernelcare.com/"
            )
        self.logger.info('Searching for installed agents')
        installed = self.is_agent_installed()

        if installed and not force:
            raise ClientAlreadyInstalledError(
                'Looks like Acronis backup is already installed. '
                'Use --force to override')

        deps = self.ACRONIS_DEPS
        kernel_dev_args = ()

        if self._is_hybrid_kernel():
            self.logger.info('Hybrid Kernel detected: installing devtoolset')
            deps, kernel_dev_args = self._enable_devtoolset_repo()
            self._enable_devtoolset_dkms()

        self.logger.info('Installing dependencies')
        try:
            self.pm.install(*deps)
        except package_manager.PackageManagerError as e:
            raise Exception(
                'Error installing dependencies: {}'.format(e.stderr.strip())
            )

        self.logger.info('Kernel package installed: %s',
                         self.pm.kernel_name_full)
        self.logger.info('Checking if %s is installed.',
                         self.pm.kernel_dev_name_full)
        if not self.pm.is_kernel_dev_installed():
            self.logger.info('Installing %s', self.pm.kernel_dev_name_full)
            try:
                self.pm.install_kernel_dev(*kernel_dev_args)
            except package_manager.PackageManagerError:
                raise Exception('{} package could not be found. '
                                'Please install it manually or try with '
                                'a different kernel.'
                                .format(self.pm.kernel_dev_name_full))

        self.logger.info('%s is installed.', self.pm.kernel_dev_name_full)

        self.logger.info('Downloading Acronis binary')
        self._download_binary()

        self.logger.info('Installing binary')
        ret, out = self._install_binary(username, password, tmp_dir)

        self.logger.info(out)

        if ret != 0:
            raise Exception('Error while installing binary: {}'.format(out))

    def register(self, username, password):
        self.logger.info(
            'Registering on {endpoint}'.format(endpoint=self.server_url)
        )

        if os.path.exists(self.REG_BIN):
            reg_bin, reg_cmd = self.REG_BIN, self.REG_CMD
        elif os.path.exists(self.REG_BIN_OBSOLETE):
            reg_bin, reg_cmd = self.REG_BIN_OBSOLETE, self.REG_CMD_OBSOLETE
        else:
            raise Exception('No any register binary found')

        rc, out = self._exec(reg_cmd.format(
            bin=reg_bin,
            endpoint=self.server_url,
            login=username,
            password=password,
        ))
        if rc != 0:
            raise Exception('Error while registering: {}'.format(out))
        self.logger.info('Registered successfully!')

    def _is_hybrid_kernel(self):
        return '.el6h.' in self.pm.kernel_ver

    def _enable_devtoolset_repo(self):
        yum_repo_file = Path(self.YUM_REPO_DIR) / Path(self.DTS_REPO_URL).name

        with yum_repo_file.open('wb+') as f:
            with closing(requests.get(self.DTS_REPO_URL, stream=True)) as r:
                r.raise_for_status()
                for content in r.iter_content(chunk_size=1024):
                    f.write(content)

        return (
            self.ACRONIS_DEPS + self.DTS_DEPS,
            ('--enablerepo', 'cloudlinux-hybrid-testing'),
        )

    def _enable_devtoolset_dkms(self):
        dkms_config = Path(self.DKMS_CONFIG)
        config = []

        if dkms_config.exists():
            with dkms_config.open() as r:
                config = [
                    line
                    for line in r.readlines()
                    if not (
                        line.startswith("export PATH") and "devtoolset" in line
                    )
                ]

        config.append(self.DKMS_PATH_VAR.format(path=self.DTS_BIN_PATH))

        dkms_config.parent.mkdir(parents=True, exist_ok=True)
        with dkms_config.open('w+') as w:
            w.writelines(config)

    @classmethod
    def uninstall(cls):
        cls.is_agent_installed() and run(cls.UNINSTALL_CMD)


def _strip_device_name(path, patt=re.compile(r'^[A-Za-z][A-Za-z0-9]+?:')):
    """
    Here we've to remove disk prefix (like 'vda5:') and optionally prepend
    forward slash if missing
    :param path: str -> path to handle
    :param patt: regex obj -> pattern to match against
    :return: str -> modified path
    """
    path = patt.sub('', path)
    if '/' not in path:
        path = "/{}".format(path)
    return path


def _pre_exec_check():
    resource.setrlimit(resource.RLIMIT_STACK, (4096, 4096))


def _repeat(fn, *callee_args, **callee_kw):
    rv = None
    for i in range(REPEAT_NUM):
        rv = fn(*callee_args, **callee_kw)
        if rv:
            return rv
        logging.warning(
            "Callee %s returned falsy result. Attempt %d", fn.__name__, i)
        time.sleep(REPEAT_WAIT)
    return rv


@from_env(username="ACCOUNT_NAME", password="PASSWORD")
def init(
    username, password, provision=False, force=False, tmp_dir=None,
):
    AcronisConnector.create_token(username, password, force)

    server_url = AcronisConnector.get_server_url(username)
    installer = AcronisClientInstaller(server_url)

    if provision:
        installer.install_client(username, password, force, tmp_dir)
    elif installer.is_agent_installed():
        installer.register(username, password)


@asyncable
def is_agent_installed():
    return AcronisClientInstaller.is_agent_installed()


def _wait_backup_running(acr):
    alerts = []

    for a in range(1, BACKUP_ATTEMPTS + 1):
        logging.warning("Starting backup, attempt %s", a)

        AcronisAPI.make_initial_backup()

        # wait until start
        for i in range(1, REPEAT_NUM + 1):
            if acr.is_backup_running():
                return
            logging.warning("Backup is not started after scheduling, "
                            "attempt %s, waiting %s seconds", i, REPEAT_WAIT)
            time.sleep(REPEAT_WAIT)

        alerts = AcronisAPI().get_alerts(as_json=False)
        if alerts:
            break

    raise BackupFailed(alerts)


@asyncable
@auth_required
@client_required
@extra
def make_initial_backup_strict():
    """
    Starts initial backup and waits until the backup will be started and
    completed and logs the progress to the file specified in
    AcronisAPI.backup_log_path.
    :raise BackupFailed: if backup not started of failed
    """
    acr = AcronisAPI(subscribe=True)
    _wait_backup_running(acr)

    # log backup progress
    with open(acr.backup_log_path, 'w') as f:
        while acr.is_backup_running():
            progress = acr.get_backup_progress()
            if progress:
                f.write('{}\n'.format(progress))
                f.flush()
            time.sleep(5)

    if acr.get_backup_status() == 'error':
        alerts = AcronisAPI().get_alerts(as_json=False)
        raise BackupFailed(alerts)


@asyncable
@auth_required
def make_initial_backup(trace=False):
    """
    DEPRECATED
    Starts initial backup.
    If trace is True, the function waits until the backup
      is completed and logs the progress to the file specified in
      AcronisAPI.backup_log_path. Returns True if no errors occurred
      and False otherwise.
    If trace is False always returns None.
    """
    AcronisAPI.make_initial_backup()

    if not trace:
        return None

    acr = AcronisAPI(subscribe=True)
    with open(acr.backup_log_path, 'w') as f:
        while acr.is_backup_running():
            f.write('{}\n'.format(acr.get_backup_progress()))
            f.flush()
            time.sleep(5)

        return acr.get_backup_status() != 'error'


@asyncable
@auth_required
@client_required
def is_backup_running():
    return AcronisAPI(subscribe=True).is_backup_running()


@asyncable
@auth_required
@client_required
def get_backup_progress():
    return AcronisAPI(subscribe=True).get_backup_progress()


@auth_required
@client_required
def backups(until=None, **__):
    return AcronisAPI.pull_backups(until=until)


@asyncable
@auth_required
@extra
def login_url():
    return AcronisConnector().get_login_url()


@auth_required
@client_required
def info():
    return AcronisAPI().get_info()


@auth_required
@extra
def refresh_token():
    AcronisConnector().refresh_token()


@auth_required
@extra
def migration_fix_backup_plan():
    AcronisAPI.migration_fix_backup_plan()


@asyncable
@auth_required
def get_alerts(as_json=True):
    return AcronisAPI().get_alerts(as_json)


@extra
def is_installed():
    if AcronisClientInstaller.is_agent_installed():
        return 'Acronis agent is installed'
    else:
        raise helpers.ActionError('Acronis agent is NOT installed!')


@extra
def uninstall():
    AcronisClientInstaller.uninstall()
Cancel / Back
[ MAINHACK ]

Viewing / Editing File: acronis.py

Server Info

System Features

User Info
