__license__   = 'GPL v3'
__copyright__ = '2008, Kovid Goyal <kovid at kovidgoyal.net>'

import io

# Imports {{{
import os
import queue
import re
import sys
import time
import traceback
import weakref
from threading import Event, Thread

from qt.core import QAction, QActionGroup, QCoreApplication, QDialog, QDialogButtonBox, QEventLoop, QIcon, QMenu, QObject, QTimer, QVBoxLayout, pyqtSignal

from calibre import as_unicode, force_unicode, preferred_encoding, prints, sanitize_file_name
from calibre.constants import DEBUG
from calibre.customize.ui import available_input_formats, available_output_formats, device_plugins, disabled_device_plugins, initialize_plugin
from calibre.devices.errors import (
    BlacklistedDevice,
    FreeSpaceError,
    InitialConnectionError,
    OpenActionNeeded,
    OpenFailed,
    OpenFeedback,
    UserFeedback,
    WrongDestinationError,
)
from calibre.devices.folder_device.driver import FOLDER_DEVICE
from calibre.devices.interface import DevicePlugin, currently_connected_device
from calibre.devices.scanner import DeviceScanner
from calibre.ebooks.covers import cprefs, generate_cover, override_prefs, scale_cover
from calibre.ebooks.metadata import authors_to_string
from calibre.gui2 import (
    Dispatcher,
    FunctionDispatcher,
    config,
    dynamic,
    error_dialog,
    gprefs,
    info_dialog,
    question_dialog,
    show_restart_warning,
    warning_dialog,
)
from calibre.gui2.dialogs.choose_format_device import ChooseFormatDeviceDialog
from calibre.gui2.widgets import BusyCursor
from calibre.library.save_to_disk import find_plugboard
from calibre.prints import debug_print
from calibre.ptempfile import PersistentTemporaryFile
from calibre.ptempfile import force_unicode as filename_to_unicode
from calibre.startup import connect_lambda
from calibre.utils.config import device_prefs, tweaks
from calibre.utils.filenames import ascii_filename
from calibre.utils.img import scale_image
from calibre.utils.ipc.job import BaseJob
from calibre.utils.localization import ngettext

# }}}


class DeviceJob(BaseJob):  # {{{

    def __init__(self, func, done, job_manager, args=[], kwargs={},
            description=''):
        BaseJob.__init__(self, description)
        self.func = func
        self.callback_on_done = done
        if not isinstance(self.callback_on_done, (Dispatcher,
            FunctionDispatcher)):
            self.callback_on_done = FunctionDispatcher(self.callback_on_done)

        self.args, self.kwargs = args, kwargs
        self.exception = None
        self.job_manager = job_manager
        self._details = _('No details available.')
        self._aborted = False

    def start_work(self):
        if DEBUG:
            prints('Job:', self.id, self.description, 'started')
        self.start_time = time.time()
        self.job_manager.changed_queue.put(self)

    def job_done(self):
        self.duration = time.time() - self.start_time
        self.percent = 1
        if DEBUG:
            prints('DeviceJob:', self.id, self.description,
                    'done, calling callback')

        try:
            self.callback_on_done(self)
        except Exception:
            pass
        if DEBUG:
            prints('DeviceJob:', self.id, self.description,
                    'callback returned')
        self.job_manager.changed_queue.put(self)

    def report_progress(self, percent, msg=''):
        self.notifications.put((percent, msg))
        self.job_manager.changed_queue.put(self)

    def run(self):
        self.start_work()
        try:
            self.result = self.func(*self.args, **self.kwargs)
            if self._aborted:
                return
        except (Exception, SystemExit) as err:
            if self._aborted:
                return
            self.failed = True
            ex = as_unicode(err)
            self._details = ex + '\n\n' + \
                force_unicode(traceback.format_exc())
            self.exception = err
        finally:
            self.job_done()

    def abort(self, err):
        call_job_done = False
        if self.run_state == self.WAITING:
            self.start_work()
            call_job_done = True
        self._aborted = True
        self.failed = True
        self._details = str(err)
        self.exception = err
        if call_job_done:
            self.job_done()

    @property
    def log_file(self):
        return io.BytesIO(self._details.encode('utf-8'))

    # }}}


def device_name_for_plugboards(device_class):
    if hasattr(device_class, 'DEVICE_PLUGBOARD_NAME'):
        return device_class.DEVICE_PLUGBOARD_NAME
    return device_class.__class__.__name__


def convert_open_popup(opm, skip_key):
    class OPM(OpenFeedback):

        def __init__(self, opm):
            super().__init__('placeholder')
            self.opm = opm
            self.skip_key = skip_key

        def custom_dialog(self, parent):
            from calibre.gui2.dialogs.message_box import MessageBox

            class M(MessageBox):
                def on_cd_finished(s):
                    gprefs.set(self.skip_key, not s.toggle_checkbox.isChecked())
            m = M({
                'info': MessageBox.INFO, 'information': MessageBox.INFO,
                'warn': MessageBox.WARNING, 'warning': MessageBox.WARNING,
                }[self.opm.level], self.opm.title, self.opm.message,
                parent=parent
            )
            tc = m.toggle_checkbox
            tc.setVisible(True)
            tc.setText(_('Show this message again'))
            tc.setChecked(not self.opm.skip_dialog_skip_precheck)
            m.resize_needed.emit()
            m.finished.connect(m.on_cd_finished)
            return m

    return OPM(opm)


class DeviceManager(Thread):  # {{{

    def __init__(self, connected_slot, job_manager, open_feedback_slot,
                 open_feedback_msg, allow_connect_slot,
                 after_callback_feedback_slot, sleep_time=2):
        '''
        :sleep_time: Time to sleep between device probes in secs
        '''
        Thread.__init__(self, name='DeviceManager', daemon=True)
        # [Device driver, Showing in GUI, Ejected]
        self.devices        = list(device_plugins())
        self.disabled_device_plugins = list(disabled_device_plugins())
        self.managed_devices = [x for x in self.devices if
                not x.MANAGES_DEVICE_PRESENCE]
        self.unmanaged_devices = [x for x in self.devices if
                x.MANAGES_DEVICE_PRESENCE]
        self.sleep_time     = sleep_time
        self.connected_slot = connected_slot  # see DeviceMixin.device_connected()
        self.allow_connect_slot = allow_connect_slot
        self.jobs           = queue.Queue(0)
        self.job_steps      = queue.Queue(0)
        self.shutdown_event = Event()
        self.job_manager    = job_manager
        self.reported_errors = set()
        self.shown_open_popups = set()
        self.current_job    = None
        self.scanner        = DeviceScanner()
        self.connected_device = None
        self.connected_device_kind = None
        self.ejected_devices  = set()
        self.mount_connection_requests = queue.Queue(0)
        self.open_feedback_slot = open_feedback_slot
        self.open_feedback_only_once_seen = set()
        self.after_callback_feedback_slot = after_callback_feedback_slot
        self.open_feedback_msg = open_feedback_msg
        self._device_information = None
        self.current_library_uuid = None
        self.call_shutdown_on_disconnect = False
        self.devices_initialized = Event()
        self.dynamic_plugins = {}

    def report_progress(self, *args):
        pass

    @property
    def is_device_connected(self):
        return self.connected_device is not None

    @property
    def is_device_present(self):
        return self.connected_device is not None and self.connected_device not in self.ejected_devices

    @property
    def device(self):
        return self.connected_device

    def do_connect(self, connected_devices, device_kind):
        for dev, detected_device in connected_devices:
            if dev.OPEN_FEEDBACK_MESSAGE is not None:
                self.open_feedback_slot(dev.OPEN_FEEDBACK_MESSAGE)
            opm = dev.get_open_popup_message()
            if opm is not None:
                skip_key = f'do_not_show_device_open_popup_message_{dev.__class__.__name__}'
                if skip_key not in self.shown_open_popups and not gprefs.get(skip_key, False):
                    self.shown_open_popups.add(skip_key)
                    self.open_feedback_msg(dev.get_gui_name(), convert_open_popup(opm, skip_key))
            try:
                dev.reset(detected_device=detected_device,
                    report_progress=self.report_progress)
                dev.open(detected_device, self.current_library_uuid)
            except OpenFeedback as e:
                if dev not in self.ejected_devices:
                    self.open_feedback_msg(dev.get_gui_name(), e)
                    self.ejected_devices.add(dev)
                continue
            except OpenFailed:
                raise
            except Exception:
                tb = traceback.format_exc()
                if DEBUG or tb not in self.reported_errors:
                    self.reported_errors.add(tb)
                    prints('Unable to open device', str(dev))
                    prints(tb)
                continue
            self.after_device_connect(dev, device_kind)
            return True
        return False

    def after_device_connect(self, dev, device_kind):
        allow_connect = True
        try:
            uid = dev.get_device_uid()
        except NotImplementedError:
            uid = None
        asked = gprefs.get('ask_to_manage_device', [])
        if (dev.ASK_TO_ALLOW_CONNECT and uid and uid not in asked):
            if not self.allow_connect_slot(dev.get_gui_name(), dev.icon):
                allow_connect = False
            asked.append(uid)
            gprefs.set('ask_to_manage_device', asked)
        if not allow_connect:
            dev.ignore_connected_device(uid)
            return

        self.connected_device = currently_connected_device._device = dev
        self.connected_device.specialize_global_preferences(device_prefs)
        self.connected_device_kind = device_kind
        self.connected_slot(True, device_kind)

    def connected_device_removed(self):
        while True:
            try:
                job = self.jobs.get_nowait()
                job.abort(Exception(_('Device no longer connected.')))
            except queue.Empty:
                break
        try:
            self.connected_device.post_yank_cleanup()
        except Exception:
            pass
        if self.connected_device in self.ejected_devices:
            self.ejected_devices.remove(self.connected_device)
            call_connected_slot = False
        else:
            call_connected_slot = True
        if self.call_shutdown_on_disconnect:
            # The current device is an instance of a plugin class instantiated
            # to handle this connection, probably as a mounted device. We are
            # now abandoning the instance that we created, so we tell it that it
            # is being shut down.
            self.connected_device.shutdown()
            self.call_shutdown_on_disconnect = False

        device_prefs.set_overrides()
        self.connected_device = currently_connected_device._device = None
        self._device_information = None
        if call_connected_slot:
            self.connected_slot(False, None)

    def detect_device(self):
        if self.is_device_connected and self.connected_device_kind in {'folder', 'folder-as-device'}:
            if not self.connected_device.is_folder_still_available():
                self.connected_device_removed()
            return
        self.scanner.scan()

        if self.is_device_connected:
            if self.connected_device.MANAGES_DEVICE_PRESENCE:
                cd = self.connected_device.detect_managed_devices(self.scanner.devices)
                if cd is None:
                    self.connected_device_removed()
            else:
                connected, detected_device = \
                    self.scanner.is_device_connected(self.connected_device,
                            only_presence=True)
                if not connected:
                    if DEBUG:
                        # Allow the device subsystem to output debugging info about
                        # why it thinks the device is not connected. Used, e.g.
                        # in the can_handle() method of the T1 driver
                        self.scanner.is_device_connected(self.connected_device,
                                only_presence=True, debug=True)
                    self.connected_device_removed()
        else:
            for dev in self.unmanaged_devices:
                try:
                    cd = dev.detect_managed_devices(self.scanner.devices)
                except Exception:
                    prints(f'Error during device detection for {dev}:')
                    traceback.print_exc()
                else:
                    if cd is not None:
                        try:
                            dev.open(cd, self.current_library_uuid)
                        except BlacklistedDevice as e:
                            prints(f'Ignoring blacklisted device: {as_unicode(e)}')
                        except OpenActionNeeded as e:
                            if e.only_once_id not in self.open_feedback_only_once_seen:
                                self.open_feedback_only_once_seen.add(e.only_once_id)
                                self.open_feedback_msg(e.device_name, e)
                        except Exception:
                            prints(f'Error while trying to open {cd} (Driver: {dev})')
                            traceback.print_exc()
                        else:
                            self.after_device_connect(dev, 'unmanaged-device')
                            return
            try:
                possibly_connected_devices = []
                for device in self.managed_devices:
                    if device in self.ejected_devices:
                        continue
                    try:
                        possibly_connected, detected_device = \
                                self.scanner.is_device_connected(device)
                    except InitialConnectionError as e:
                        self.open_feedback_msg(device.get_gui_name(), e)
                        continue
                    if possibly_connected:
                        possibly_connected_devices.append((device, detected_device))
                if possibly_connected_devices:
                    if not self.do_connect(possibly_connected_devices,
                                           device_kind='device'):
                        if DEBUG:
                            prints('Connect to device failed, retrying in 5 seconds...')
                        time.sleep(5)
                        if not self.do_connect(possibly_connected_devices,
                                           device_kind='device'):
                            if DEBUG:
                                prints('Device connect failed again, giving up')
            except OpenFailed as e:
                if e.show_me:
                    traceback.print_exc()

    # Mount devices that don't use USB, such as the folder device
    # This will be called on the GUI thread. Because of this, we must store
    # information that the scanner thread will use to do the real work.
    def mount_device(self, kls, kind, path, model_metadata=None):
        self.mount_connection_requests.put((kls, kind, path, model_metadata))

    # disconnect a device
    def umount_device(self, *args):
        if self.is_device_connected and not self.job_manager.has_device_jobs():
            if self.connected_device_kind in {'unmanaged-device', 'device'}:
                self.connected_device.eject()
                if self.connected_device_kind != 'unmanaged-device':
                    self.ejected_devices.add(self.connected_device)
                self.connected_slot(False, None)
            elif hasattr(self.connected_device, 'unmount_device'):
                # As we are on the wrong thread, this call must *not* do
                # anything besides set a flag that the right thread will see.
                self.connected_device.unmount_device()

    def next_job(self):
        if not self.job_steps.empty():
            try:
                return self.job_steps.get_nowait()
            except queue.Empty:
                pass

        if not self.jobs.empty():
            try:
                return self.jobs.get_nowait()
            except queue.Empty:
                pass

    def run_startup(self, dev):
        name = 'unknown'
        try:
            name = dev.__class__.__name__
            dev.startup()
        except Exception:
            prints(f'Startup method for device {name} threw exception')
            traceback.print_exc()

    def run(self):
        # Do any device-specific startup processing.
        for d in self.devices:
            self.run_startup(d)
            n = d.is_dynamically_controllable()
            if n:
                self.dynamic_plugins[n] = d
        self.devices_initialized.set()

        while not self.shutdown_event.is_set():
            kls = model_metadata = None
            while True:
                try:
                    kls, device_kind, folder_path, model_metadata = self.mount_connection_requests.get_nowait()
                except queue.Empty:
                    break
            if model_metadata is not None:
                try:
                    for candidate in self.devices:
                        if type(candidate) is model_metadata.driver_class:
                            dev = candidate
                            break
                    else:
                        # new device instance so run startup and set flag to
                        # run shutdown on disconnect
                        dev = initialize_plugin(model_metadata.driver_class)
                        self.run_startup(dev)
                        self.call_shutdown_on_disconnect = True
                    self.do_connect([[dev, model_metadata.detected_device(folder_path)],], device_kind=device_kind)
                except Exception:
                    prints(f'Unable to open {device_kind} as device ({folder_path})')
                    traceback.print_exc()
            elif kls is not None:
                try:
                    dev = kls(folder_path)
                    # We just created a new device instance. Call its startup
                    # method and set the flag to call the shutdown method when
                    # it disconnects.
                    self.run_startup(dev)
                    self.call_shutdown_on_disconnect = True
                    self.do_connect([[dev, None],], device_kind=device_kind)
                except Exception:
                    prints(f'Unable to open {device_kind} as device ({folder_path})')
                    traceback.print_exc()
            else:
                self.detect_device()

            do_sleep = True
            while True:
                job = self.next_job()
                if job is not None:
                    do_sleep = False
                    self.current_job = job
                    if self.device is not None:
                        self.device.set_progress_reporter(job.report_progress)
                    self.current_job.run()
                    self.current_job = None
                    feedback = getattr(self.device, 'user_feedback_after_callback', None)
                    if feedback is not None:
                        self.device.user_feedback_after_callback = None
                        self.after_callback_feedback_slot(feedback)
                else:
                    break
            if do_sleep:
                self.shutdown_event.wait(self.sleep_time)

        # We are exiting. Call the shutdown method for each plugin
        for p in self.devices:
            try:
                p.shutdown()
            except Exception:
                pass

    def create_job_step(self, func, done, description, to_job, args=[], kwargs={}):
        job = DeviceJob(func, done, self.job_manager,
                        args=args, kwargs=kwargs, description=description)
        self.job_manager.add_job(job)
        if (done is None or isinstance(done, FunctionDispatcher)) and \
                    (to_job is not None and to_job == self.current_job):
            self.job_steps.put(job)
        else:
            self.jobs.put(job)
        return job

    def create_job(self, func, done, description, args=[], kwargs={}):
        return self.create_job_step(func, done, description, None, args, kwargs)

    def has_card(self):
        try:
            return bool(self.device.card_prefix())
        except Exception:
            return False

    def _debug_detection(self):
        from calibre.devices import debug
        raw = debug(plugins=self.devices,
                disabled_plugins=self.disabled_device_plugins)
        return raw

    def debug_detection(self, done):
        if self.is_device_connected:
            raise ValueError('Device is currently detected in calibre, cannot'
                    ' debug device detection')
        self.create_job(self._debug_detection, done,
                _('Debug device detection'))

    def _get_device_information(self):
        info = self.device.get_device_information(end_session=False)
        if len(info) < 5:
            info = tuple(list(info) + [{}])
        info = [i.replace('\x00', '').replace('\x01', '') if isinstance(i, str) else i
                 for i in info]
        cp = self.device.card_prefix(end_session=False)
        fs = self.device.free_space()
        self._device_information = {'info': info, 'prefixes': cp, 'freespace': fs}
        return info, cp, fs

    def get_device_information(self, done, add_as_step_to_job=None):
        '''Get device information and free space on device'''
        return self.create_job_step(self._get_device_information, done,
                    description=_('Get device information'), to_job=add_as_step_to_job)

    def _set_library_information(self, library_name, library_uuid, field_metadata):
        '''Give the device the current library information'''
        self.device.set_library_info(library_name, library_uuid, field_metadata)

    def set_library_information(self, done, library_name, library_uuid,
                                 field_metadata, add_as_step_to_job=None):
        '''Give the device the current library information'''
        return self.create_job_step(self._set_library_information, done,
                    args=[library_name, library_uuid, field_metadata],
                    description=_('Set library information'), to_job=add_as_step_to_job)

    def slow_driveinfo(self):
        ''' Update the stored device information with the driveinfo if the
        device indicates that getting driveinfo is slow '''
        info = self._device_information['info']
        if (not info[4] and self.device.SLOW_DRIVEINFO):
            info = list(info)
            info[4] = self.device.get_driveinfo()
            self._device_information['info'] = tuple(info)

    def get_current_device_information(self):
        return self._device_information if self.is_device_present else None

    def _books(self):
        '''Get metadata from device'''
        mainlist = self.device.books(oncard=None, end_session=False)
        cardalist = self.device.books(oncard='carda')
        cardblist = self.device.books(oncard='cardb')
        return mainlist, cardalist, cardblist

    def books(self, done, add_as_step_to_job=None):
        '''Return callable that returns the list of books on device as two booklists'''
        return self.create_job_step(self._books, done,
                description=_('Get list of books on device'), to_job=add_as_step_to_job)

    def _prepare_addable_books(self, paths):
        return self.device.prepare_addable_books(paths)

    def prepare_addable_books(self, done, paths, add_as_step_to_job=None):
        return self.create_job_step(self._prepare_addable_books, done, args=[paths],
                description=_('Prepare files for transfer from device'),
                to_job=add_as_step_to_job)

    def _annotations(self, path_map):
        return self.device.get_annotations(path_map)

    def annotations(self, done, path_map, add_as_step_to_job=None):
        '''Return mapping of ids to annotations. Each annotation is of the
        form (type, location_info, content). path_map is a mapping of
        ids to paths on the device.'''
        return self.create_job_step(self._annotations, done, args=[path_map],
                description=_('Get annotations from device'), to_job=add_as_step_to_job)

    def _sync_booklists(self, booklists):
        '''Sync metadata to device'''
        self.device.sync_booklists(booklists, end_session=False)
        return self.device.card_prefix(end_session=False), self.device.free_space()

    def sync_booklists(self, done, booklists, plugboards, add_as_step_to_job=None):
        if hasattr(self.connected_device, 'set_plugboards') and \
                callable(self.connected_device.set_plugboards):
            self.connected_device.set_plugboards(plugboards, find_plugboard)
        return self.create_job_step(self._sync_booklists, done, args=[booklists],
                        description=_('Send metadata to device'), to_job=add_as_step_to_job)

    def upload_collections(self, done, booklist, on_card, add_as_step_to_job=None):
        return self.create_job_step(booklist.rebuild_collections, done,
                               args=[booklist, on_card],
                        description=_('Send collections to device'),
                        to_job=add_as_step_to_job)

    def _upload_books(self, files, names, on_card=None, metadata=None, plugboards=None):
        '''Upload books to device: '''
        from calibre.ebooks.metadata.meta import set_metadata
        if hasattr(self.connected_device, 'set_plugboards') and \
                callable(self.connected_device.set_plugboards):
            self.connected_device.set_plugboards(plugboards, find_plugboard)
        if metadata and files and len(metadata) == len(files):
            for f, mi in zip(files, metadata):
                if isinstance(f, str):
                    ext = f.rpartition('.')[-1].lower()
                    cpb = find_plugboard(
                            device_name_for_plugboards(self.connected_device),
                            ext, plugboards)
                    if ext:
                        try:
                            if DEBUG:
                                prints('Setting metadata in:', mi.title, 'at:',
                                        f, file=sys.__stdout__)
                            with open(f, 'r+b') as stream:
                                if cpb:
                                    newmi = mi.deepcopy_metadata()
                                    newmi.template_to_attribute(mi, cpb)
                                else:
                                    newmi = mi
                                nuke_comments = getattr(self.connected_device,
                                        'NUKE_COMMENTS', None)
                                if nuke_comments is not None:
                                    mi.comments = nuke_comments
                                set_metadata(stream, newmi, stream_type=ext)
                        except Exception:
                            if DEBUG:
                                prints(traceback.format_exc(), file=sys.__stdout__)

        try:
            return self.device.upload_books(files, names, on_card,
                    metadata=metadata, end_session=False)
        finally:
            if metadata:
                for mi in metadata:
                    try:
                        if mi.cover:
                            os.remove(mi.cover)
                    except Exception:
                        pass

    def upload_books(self, done, files, names, on_card=None, titles=None,
                     metadata=None, plugboards=None, add_as_step_to_job=None):
        desc = ngettext('Upload one book to the device', 'Upload {} books to the device', len(names)).format(len(names))
        if titles:
            desc += ': ' + ', '.join(titles)
        return self.create_job_step(self._upload_books, done, to_job=add_as_step_to_job,
                               args=[files, names],
                kwargs={'on_card':on_card,'metadata':metadata,'plugboards':plugboards}, description=desc)

    def add_books_to_metadata(self, locations, metadata, booklists):
        self.device.add_books_to_metadata(locations, metadata, booklists)

    def _delete_books(self, paths):
        '''Remove books from device'''
        self.device.delete_books(paths, end_session=True)

    def delete_books(self, done, paths, add_as_step_to_job=None):
        return self.create_job_step(self._delete_books, done, args=[paths],
                        description=_('Delete books from device'),
                        to_job=add_as_step_to_job)

    def remove_books_from_metadata(self, paths, booklists):
        self.device.remove_books_from_metadata(paths, booklists)

    def _save_books(self, paths, target):
        '''Copy books from device to disk'''
        for path in paths:
            name = sanitize_file_name(os.path.basename(path))
            dest = os.path.join(target, name)
            if os.path.abspath(dest) != os.path.abspath(path):
                with open(dest, 'wb') as f:
                    self.device.get_file(path, f)

    def save_books(self, done, paths, target, add_as_step_to_job=None):
        return self.create_job_step(self._save_books, done, args=[paths, target],
                        description=_('Download books from device'),
                        to_job=add_as_step_to_job)

    def _view_book(self, path, target):
        with open(target, 'wb') as f:
            self.device.get_file(path, f)
        return target

    def view_book(self, done, path, target, add_as_step_to_job=None):
        return self.create_job_step(self._view_book, done, args=[path, target],
                        description=_('View book on device'), to_job=add_as_step_to_job)

    def set_current_library_uuid(self, uuid):
        self.current_library_uuid = uuid

    def set_driveinfo_name(self, location_code, name):
        if self.connected_device:
            self.connected_device.set_driveinfo_name(location_code, name)

    # dynamic plugin interface

    # This is a helper function that handles queueing with the device manager
    def _call_request(self, name, method, *args, **kwargs):
        d = self.dynamic_plugins.get(name, None)
        if d:
            return getattr(d, method)(*args, **kwargs)
        return kwargs.get('default', None)

    # The dynamic plugin methods below must be called on the GUI thread. They
    # will switch to the device thread before calling the plugin.

    def start_plugin(self, name):
        return self._call_request(name, 'start_plugin')

    def stop_plugin(self, name):
        self._call_request(name, 'stop_plugin')

    def get_option(self, name, opt_string, default=None):
        return self._call_request(name, 'get_option', opt_string, default=default)

    def set_option(self, name, opt_string, opt_value):
        self._call_request(name, 'set_option', opt_string, opt_value)

    def is_running(self, name):
        if self._call_request(name, 'is_running'):
            return True
        return False

    def is_enabled(self, name):
        try:
            d = self.dynamic_plugins.get(name, None)
            if d:
                return True
        except Exception:
            pass
        return False

    # }}}


class DeviceAction(QAction):  # {{{

    a_s = pyqtSignal(object)

    def __init__(self, dest, delete, specific, icon_path, text, parent=None):
        QAction.__init__(self, QIcon.ic(icon_path), text, parent)
        self.dest = dest
        self.delete = delete
        self.specific = specific
        self.triggered.connect(self.emit_triggered)

    def emit_triggered(self, *args):
        self.a_s.emit(self)

    def __repr__(self):
        return self.__class__.__name__ + f':{self.dest}:{self.delete}:{self.specific}'
    # }}}


class DeviceMenu(QMenu):  # {{{

    fetch_annotations = pyqtSignal()
    disconnect_mounted_device = pyqtSignal()
    sync = pyqtSignal(object, object, object)

    def __init__(self, parent=None):
        QMenu.__init__(self, parent)
        self.group = QActionGroup(self)
        self._actions = []
        self._memory = []

        self.set_default_menu = QMenu(_('Set default send to device action'))
        self.set_default_menu.setIcon(QIcon.ic('config.png'))

        basic_actions = [
                ('main:', False, False, 'reader.png',
                    _('Send to main memory')),
                ('carda:0', False, False, 'sd.png',
                    _('Send to storage card A')),
                ('cardb:0', False, False, 'sd.png',
                    _('Send to storage card B')),
        ]

        delete_actions = [
                ('main:', True, False, 'reader.png',
                    _('Main memory')),
                ('carda:0', True, False, 'sd.png',
                    _('Storage card A')),
                ('cardb:0', True, False, 'sd.png',
                    _('Storage card B')),
        ]

        specific_actions = [
                ('main:', False, True, 'reader.png',
                    _('Main memory')),
                ('carda:0', False, True, 'sd.png',
                    _('Storage card A')),
                ('cardb:0', False, True, 'sd.png',
                    _('Storage card B')),
        ]

        later_menus = []

        for menu in (self, self.set_default_menu):
            for actions, desc in (
                    (basic_actions, ''),
                    (specific_actions, _('Send specific format to')),
                    (delete_actions, _('Send and delete from library')),
                    ):
                mdest = menu
                if actions is not basic_actions:
                    mdest = QMenu(desc)
                    self._memory.append(mdest)
                    later_menus.append(mdest)
                    if menu is self.set_default_menu:
                        menu.addMenu(mdest)
                        menu.addSeparator()

                for dest, delete, specific, icon, text in actions:
                    action = DeviceAction(dest, delete, specific, icon, text, self)
                    self._memory.append(action)
                    if menu is self.set_default_menu:
                        action.setCheckable(True)
                        action.setText(action.text())
                        self.group.addAction(action)
                    else:
                        action.a_s.connect(self.action_triggered)
                        self._actions.append(action)
                    mdest.addAction(action)
                if actions is basic_actions:
                    menu.addSeparator()

        da = config['default_send_to_device_action']
        done = False
        for action in self.group.actions():
            if repr(action) == da:
                action.setChecked(True)
                done = True
                break
        if not done:
            action = list(self.group.actions())[0]
            action.setChecked(True)
            config['default_send_to_device_action'] = repr(action)

        self.group.triggered.connect(self.change_default_action)
        self.addSeparator()

        self.addMenu(later_menus[0])
        self.addSeparator()

        mitem = self.addAction(QIcon.ic('eject.png'), _('Eject device'))
        mitem.setEnabled(False)
        connect_lambda(mitem.triggered, self, lambda self, x: self.disconnect_mounted_device.emit())
        self.disconnect_mounted_device_action = mitem
        self.addSeparator()

        self.addMenu(self.set_default_menu)
        self.addSeparator()

        self.addMenu(later_menus[1])
        self.addSeparator()

        annot = self.addAction(_('Fetch annotations (experimental)'))
        annot.setEnabled(False)
        connect_lambda(annot.triggered, self, lambda self, x: self.fetch_annotations.emit())
        self.annotation_action = annot
        self.enable_device_actions(False)

    def change_default_action(self, action):
        config['default_send_to_device_action'] = repr(action)
        action.setChecked(True)

    def action_triggered(self, action):
        self.sync.emit(action.dest, action.delete, action.specific)

    def trigger_default(self, *args):
        r = config['default_send_to_device_action']
        for action in self._actions:
            if repr(action) == r:
                self.action_triggered(action)
                break

    def enable_device_actions(self, enable, card_prefix=(None, None),
            device=None):
        for action in self._actions:
            if action.dest in ('main:', 'carda:0', 'cardb:0'):
                if not enable:
                    action.setEnabled(False)
                elif action.dest == 'main:':
                    action.setEnabled(True)
                elif action.dest == 'carda:0':
                    if card_prefix and card_prefix[0] is not None:
                        action.setEnabled(True)
                    else:
                        action.setEnabled(False)
                elif action.dest == 'cardb:0':
                    if card_prefix and card_prefix[1] is not None:
                        action.setEnabled(True)
                    else:
                        action.setEnabled(False)

        annot_enable = enable and getattr(device, 'SUPPORTS_ANNOTATIONS', False)
        self.annotation_action.setEnabled(annot_enable)

    # }}}


class DeviceSignals(QObject):  # {{{
    #: This signal is emitted once, after metadata is downloaded from the
    #: connected device.
    #: The sequence: gui.device_manager.is_device_connected will become True,
    #: and the device_connection_changed signal will be emitted,
    #: then sometime later gui.device_metadata_available will be signaled.
    #: This does not mean that there are no more jobs running. Automatic metadata
    #: management might have kicked off a sync_booklists to write new metadata onto
    #: the device, and that job might still be running when the signal is emitted.
    device_metadata_available = pyqtSignal()

    #: This signal is emitted once when the device is detected and once when
    #: it is disconnected. If the parameter is True, then it is a connection,
    #: otherwise a disconnection. Device information is not available in either
    # case. If you need device information when connecting then use the
    # device_metadata_available signal.
    device_connection_changed = pyqtSignal(object)


device_signals = DeviceSignals()
# }}}


class DeviceMixin:  # {{{

    def __init__(self, *args, **kwargs):
        pass

    def init_device_mixin(self):
        self.device_error_dialog = error_dialog(self, _('Error'),
                _('Error communicating with device'), ' ')
        self.device_error_dialog.setModal(False)
        self.device_manager = DeviceManager(FunctionDispatcher(self.device_detected),
                self.job_manager, Dispatcher(self.status_bar.show_message),
                Dispatcher(self.show_open_feedback),
                FunctionDispatcher(self.allow_connect), Dispatcher(self.after_callback_feedback))
        self.device_manager.start()
        self.device_manager.devices_initialized.wait()
        if tweaks['auto_connect_to_folder']:
            self.connect_to_folder_named(tweaks['auto_connect_to_folder'])

    def allow_connect(self, name, icon):
        return question_dialog(self, _('Manage the %s?')%name,
                _('Detected the <b>%s</b>. Do you want calibre to manage it?')%
                name, show_copy_button=False,
                override_icon=QIcon.ic(icon))

    def after_callback_feedback(self, feedback):
        title, msg, det_msg = feedback
        info_dialog(self, feedback['title'], feedback['msg'], det_msg=feedback['det_msg']).show()

    def debug_detection(self, done):
        self.debug_detection_callback = weakref.ref(done)
        self.device_manager.debug_detection(FunctionDispatcher(self.debug_detection_done))

    def debug_detection_done(self, job):
        d = self.debug_detection_callback()
        if d is not None:
            d(job)

    def show_open_feedback(self, devname, e):
        try:
            self.__of_dev_mem__ = d = e.custom_dialog(self)
        except NotImplementedError:
            self.__of_dev_mem__ = d = info_dialog(self, devname, e.feedback_msg)
        d.show()

    def auto_convert_question(self, msg, autos):
        autos = '\n'.join(map(str, map(force_unicode, autos)))
        return self.ask_a_yes_no_question(
                _('No suitable formats'), msg,
                ans_when_user_unavailable=True,
                det_msg=autos, skip_dialog_name='auto_convert_before_send'
        )

    def set_default_thumbnail(self, height):
        ratio = height / float(cprefs['cover_height'])
        self.default_thumbnail_prefs = prefs = override_prefs(cprefs)
        scale_cover(prefs, ratio)

    def connect_to_folder_named(self, folder, model_metadata=None):
        if os.path.exists(folder) and os.path.isdir(folder):
            if model_metadata is not None:
                self.device_manager.mount_device(kls=None, kind='folder-as-device', path=folder, model_metadata=model_metadata)
            else:
                self.device_manager.mount_device(kls=FOLDER_DEVICE, kind='folder', path=folder)

    def connect_to_folder(self):
        from calibre.gui2.dialogs.connect_to_folder import ConnectToFolder
        d = ConnectToFolder(self)
        if d.exec() == QDialog.DialogCode.Accepted:
            folder_path, model_metadata = d.ans
            if folder_path:
                self.connect_to_folder_named(folder_path, model_metadata)

    # disconnect from folder devices
    def disconnect_mounted_device(self):
        self.device_manager.umount_device()

    def configure_connected_device(self):
        if not self.device_manager.is_device_connected:
            return
        if self.job_manager.has_device_jobs(queued_also=True):
            return error_dialog(self, _('Running jobs'),
                    _('Cannot configure the device while there are running'
                        ' device jobs.'), show=True)
        dev = self.device_manager.connected_device

        cw = dev.config_widget()
        config_dialog = QDialog(self)

        config_dialog.setWindowTitle(_('Configure %s')%dev.get_gui_name())
        config_dialog.setWindowIcon(QIcon.ic('config.png'))
        l = QVBoxLayout(config_dialog)
        config_dialog.setLayout(l)
        bb = QDialogButtonBox(QDialogButtonBox.StandardButton.Ok|QDialogButtonBox.StandardButton.Cancel)
        bb.accepted.connect(config_dialog.accept)
        bb.rejected.connect(config_dialog.reject)
        l.addWidget(cw)
        l.addWidget(bb)
        # We do not save/restore the size of this dialog as different devices
        # have very different size requirements
        config_dialog.resize(config_dialog.sizeHint())

        def validate():
            if cw.validate():
                QDialog.accept(config_dialog)
        config_dialog.accept = validate
        if config_dialog.exec() == QDialog.DialogCode.Accepted:
            dev.save_settings(cw)

            do_restart = show_restart_warning(_('Restart calibre for the changes to %s'
                ' to be applied.')%dev.get_gui_name(), parent=self)
            if do_restart:
                self.quit(restart=True)

    def _sync_action_triggered(self, *args):
        m = getattr(self, '_sync_menu', None)
        if m is not None:
            ids = self.library_view.get_selected_ids(as_set=True)
            db = self.current_db.new_api
            already_on_device = db.all_field_for('ondevice', ids, default_value='')
            books_on_device = {book_id for book_id, val in already_on_device.items() if val}
            if books_on_device:
                if len(books_on_device) == 1:
                    if not question_dialog(self, _('Book already on device'), _(
                        'The book {} is already present on the device. Resending it might cause any'
                            ' annotations/bookmarks on the device for this book to be lost. Are you sure?').format(
                                db.field_for('title', tuple(books_on_device)[0])), skip_dialog_name='confirm-resend-existing-books'
                    ):
                        return
                else:
                    title_sorts = db.all_field_for('sort', books_on_device)
                    titles = sorted(db.all_field_for('title', books_on_device).items(), key=lambda x: title_sorts[x[0]])
                    details = '\n'.join(title for book_id, title in titles)
                    if not question_dialog(self, _('Some books already on device'), _(
                        'Some of the selected books are already on the device. Resending them might cause any annotations/bookmarks on the'
                        ' device for these books to be lost. Click "Show details" to see the books already on the device. Are you sure?'),
                                        skip_dialog_name='confirm-resend-existing-books', det_msg=details
                    ):
                        return
            m.trigger_default()

    def create_device_menu(self):
        self._sync_menu = DeviceMenu(self)
        self.iactions['Send To Device'].qaction.setMenu(self._sync_menu)
        self.iactions['Connect Share'].build_email_entries()
        self._sync_menu.sync.connect(self.dispatch_sync_event)
        self._sync_menu.fetch_annotations.connect(
                self.iactions['Fetch Annotations'].fetch_annotations)
        self._sync_menu.disconnect_mounted_device.connect(self.disconnect_mounted_device)
        self.iactions['Connect Share'].set_state(self.device_connected,
                None)
        if self.device_connected:
            self._sync_menu.disconnect_mounted_device_action.setEnabled(True)
        else:
            self._sync_menu.disconnect_mounted_device_action.setEnabled(False)

    def device_job_exception(self, job):
        '''
        Handle exceptions in threaded device jobs.
        '''
        if isinstance(getattr(job, 'exception', None), UserFeedback):
            ex = job.exception
            func = {UserFeedback.ERROR:error_dialog,
                    UserFeedback.WARNING:warning_dialog,
                    UserFeedback.INFO:info_dialog}[ex.level]
            return func(self, _('Failed'), ex.msg, det_msg=ex.details or '', show=True)

        try:
            if 'Could not read 32 bytes on the control bus.' in \
                    str(job.details):
                error_dialog(self, _('Error talking to device'),
                             _('There was a temporary error talking to the '
                             'device. Please unplug and reconnect the device '
                             'or reboot.')).show()
                return
        except Exception:
            pass
        if getattr(job, 'exception', None).__class__.__name__ == 'MTPInvalidSendPathError':
            try:
                from calibre.gui2.device_drivers.mtp_config import SendError
                return SendError(self, job.exception).exec()
            except Exception:
                traceback.print_exc()
        try:
            prints(job.details, file=sys.stderr)
        except Exception:
            pass
        if not self.device_error_dialog.isVisible():
            self.device_error_dialog.set_details(job.details)
            self.device_error_dialog.show()

    # Device connected {{{

    def set_device_menu_items_state(self, connected):
        self.iactions['Connect Share'].set_state(connected,
                self.device_manager.device)
        if connected:
            self._sync_menu.disconnect_mounted_device_action.setEnabled(True)
            self._sync_menu.enable_device_actions(True,
                    self.device_manager.device.card_prefix(),
                    self.device_manager.device)
            self.eject_action.setEnabled(True)
        else:
            self._sync_menu.disconnect_mounted_device_action.setEnabled(False)
            self._sync_menu.enable_device_actions(False)
            self.eject_action.setEnabled(False)

    def device_detected(self, connected, device_kind):
        '''
        Called when a device is connected to the computer.

        If connected is False then device_kind is None.
        '''
        # This can happen as this function is called in a queued connection and
        # the user could have yanked the device in the meantime
        if connected and not self.device_manager.is_device_connected:
            connected = False
        self.set_device_menu_items_state(connected)
        if connected:
            self.device_connected = device_kind
            self.device_manager.get_device_information(
                    FunctionDispatcher(self.info_read))
            self.set_default_thumbnail(
                    self.device_manager.device.THUMBNAIL_HEIGHT)
            self.status_bar.show_message(_('Device: ')+
                self.device_manager.device.get_gui_name()+
                        _(' detected.'), 3000)
            self.library_view.set_device_connected(self.device_connected)
            self.refresh_ondevice(reset_only=True)
        else:
            self.device_connected = None
            self.status_bar.device_disconnected()
            dviews = (self.memory_view, self.card_a_view, self.card_b_view)
            for v in dviews:
                v.save_state()
            if self.current_view() != self.library_view:
                self.book_details.reset_info()
            self.location_manager.update_devices()
            self.bars_manager.update_bars(reveal_bar=True)
            self.library_view.set_device_connected(self.device_connected)
            # Empty any device view information
            for v in dviews:
                v.set_database([])
            # Use a singleShot timer to ensure that the job event queue has
            # emptied before the ondevice column is removed from the booklist.
            # This deals with race conditions when repainting the booklist
            # causing incorrect evaluation of the connected_device_name
            # formatter function
            QTimer.singleShot(0, self.refresh_ondevice)
        device_signals.device_connection_changed.emit(connected)

    def info_read(self, job):
        '''
        Called once device information has been read.
        '''
        if job.failed:
            return self.device_job_exception(job)
        info, cp, fs = job.result
        self.location_manager.update_devices(cp, fs,
                self.device_manager.device.icon)
        self.bars_manager.update_bars(reveal_bar=True)
        self.status_bar.device_connected(info[0])
        db = self.current_db
        self.device_manager.set_library_information(None, os.path.basename(db.library_path),
                                    db.library_id, db.field_metadata,
                                    add_as_step_to_job=job)
        self.device_manager.books(FunctionDispatcher(self.metadata_downloaded),
                                  add_as_step_to_job=job)

    def metadata_downloaded(self, job):
        '''
        Called once metadata has been read for all books on the device.
        '''
        if job.failed:
            self.device_job_exception(job)
            return
        self.device_manager.slow_driveinfo()

        # set_books_in_library might schedule a sync_booklists job
        debug_print('DeviceJob: metadata_downloaded: Starting set_books_in_library')
        self.set_books_in_library(job.result, reset=True, add_as_step_to_job=job)

        debug_print('DeviceJob: metadata_downloaded: updating views')
        mainlist, cardalist, cardblist = job.result
        self.memory_view.set_database(mainlist)
        self.memory_view.set_editable(self.device_manager.device.CAN_SET_METADATA,
                                      self.device_manager.device.BACKLOADING_ERROR_MESSAGE
                                      is None)
        self.card_a_view.set_database(cardalist)
        self.card_a_view.set_editable(self.device_manager.device.CAN_SET_METADATA,
                                      self.device_manager.device.BACKLOADING_ERROR_MESSAGE
                                      is None)
        self.card_b_view.set_database(cardblist)
        self.card_b_view.set_editable(self.device_manager.device.CAN_SET_METADATA,
                                      self.device_manager.device.BACKLOADING_ERROR_MESSAGE
                                      is None)
        debug_print('DeviceJob: metadata_downloaded: syncing')
        self.sync_news()
        self.sync_catalogs()

        debug_print('DeviceJob: metadata_downloaded: refreshing ondevice')
        self.refresh_ondevice()

        debug_print('DeviceJob: metadata_downloaded: sending metadata_available signal')
        device_signals.device_metadata_available.emit()

    def refresh_ondevice(self, reset_only=False):
        '''
        Force the library view to refresh, taking into consideration new
        device books information
        '''
        with self.library_view.preserve_state():
            self.book_on_device(None, reset=True)
            if reset_only:
                return
            self.library_view.model().refresh_ondevice()

    # }}}

    def remove_paths(self, paths):
        return self.device_manager.delete_books(
                FunctionDispatcher(self.books_deleted), paths)

    def books_deleted(self, job):
        '''
        Called once deletion is done on the device
        '''
        cv, row = self.current_view(), -1
        if cv is not self.library_view:
            row = cv.currentIndex().row()
        for view in (self.memory_view, self.card_a_view, self.card_b_view):
            view.model().deletion_done(job, job.failed)
        if job.failed:
            self.device_job_exception(job)
            return

        dm = self.iactions['Remove Books'].delete_memory
        if job in dm:
            paths, model = dm.pop(job)
            self.device_manager.remove_books_from_metadata(paths,
                    self.booklists())
            model.paths_deleted(paths)
        # Force recomputation the library's ondevice info. We need to call
        # set_books_in_library even though books were not added because
        # the deleted book might have been an exact match. Upload the booklists
        # if set_books_in_library did not.
        if not self.set_books_in_library(self.booklists(), reset=True,
                                 add_as_step_to_job=job, do_device_sync=False):

            self.upload_booklists(job)
        # We need to reset the ondevice flags in the library. Use a big hammer,
        # so we don't need to worry about whether some succeeded or not.
        self.refresh_ondevice()

        if row > -1:
            cv.set_current_row(row)
        try:
            if not self.current_view().currentIndex().isValid():
                self.current_view().set_current_row()
            self.current_view().refresh_book_details()
        except Exception:
            traceback.print_exc()

    def dispatch_sync_event(self, dest, delete, specific):
        rows = self.library_view.selectionModel().selectedRows()
        if not rows or len(rows) == 0:
            error_dialog(self, _('No books'), _('No books')+' '+
                    _('selected to send')).exec()
            return

        fmt = None
        if specific:
            if (not self.device_connected or not self.device_manager or
                    self.device_manager.device is None):
                error_dialog(self, _('No device'),
                        _('No device connected'), show=True)
                return
            formats = []
            aval_out_formats = available_output_formats()
            format_count = {}
            for row in rows:
                fmts = self.library_view.model().db.formats(row.row())
                if fmts:
                    for f in fmts.split(','):
                        f = f.lower()
                        if f in format_count:
                            format_count[f] += 1
                        else:
                            format_count[f] = 1
            for f in self.device_manager.device.settings().format_map:
                if f in format_count.keys():
                    formats.append((f, _('%(num)i of %(total)i books') % dict(
                        num=format_count[f], total=len(rows)),
                        True if f in aval_out_formats else False))
                elif f in aval_out_formats:
                    formats.append((f, _('0 of %i books') % len(rows), True))
            d = ChooseFormatDeviceDialog(self, _('Choose format to send to device'), formats)
            if d.exec() != QDialog.DialogCode.Accepted:
                return
            if d.format():
                fmt = d.format().lower()
        dest, sub_dest = dest.partition(':')[0::2]
        if dest in ('main', 'carda', 'cardb'):
            if not self.device_connected or not self.device_manager:
                error_dialog(self, _('No device'),
                        _('Cannot send: No device is connected')).exec()
                return
            if dest == 'carda' and not self.device_manager.has_card():
                error_dialog(self, _('No card'),
                        _('Cannot send: Device has no storage card')).exec()
                return
            if dest == 'cardb' and not self.device_manager.has_card():
                error_dialog(self, _('No card'),
                        _('Cannot send: Device has no storage card')).exec()
                return
            if dest == 'main':
                on_card = None
            else:
                on_card = dest
            self.sync_to_device(on_card, delete, fmt)
        elif dest == 'mail':
            sub_dest_parts = sub_dest.split(';')
            while len(sub_dest_parts) < 3:
                sub_dest_parts.append('')
            to = sub_dest_parts[0]
            fmts = sub_dest_parts[1]
            subject = ';'.join(sub_dest_parts[2:])
            fmts = [x.strip().lower() for x in fmts.split(',')]
            self.send_by_mail(to, fmts, delete, subject=subject)
        elif dest == 'choosemail':
            from calibre.gui2.email import select_recipients
            data = select_recipients(self)
            if data:
                self.send_multiple_by_mail(data, delete)

    def cover_to_thumbnail(self, data):
        if self.device_manager.device and \
                hasattr(self.device_manager.device, 'THUMBNAIL_WIDTH'):
            try:
                return scale_image(data,
                                 self.device_manager.device.THUMBNAIL_WIDTH,
                                 self.device_manager.device.THUMBNAIL_HEIGHT,
                                 preserve_aspect_ratio=False)
            except Exception:
                pass
            return
        ht = self.device_manager.device.THUMBNAIL_HEIGHT \
                if self.device_manager else DevicePlugin.THUMBNAIL_HEIGHT
        try:
            return scale_image(data, ht, ht,
                    compression_quality=self.device_manager.device.THUMBNAIL_COMPRESSION_QUALITY)
        except Exception:
            pass

    def sync_catalogs(self, send_ids=None, do_auto_convert=True):
        if self.device_connected:
            settings = self.device_manager.device.settings()
            ids = list(dynamic.get('catalogs_to_be_synced', set())) if send_ids is None else send_ids
            ids = [id for id in ids if self.library_view.model().db.has_id(id)]
            with BusyCursor():
                files, _auto_ids = self.library_view.model().get_preferred_formats_from_ids(
                                ids, settings.format_map,
                                exclude_auto=do_auto_convert)
            auto = []
            if do_auto_convert and _auto_ids:
                for id in _auto_ids:
                    dbfmts = self.library_view.model().db.formats(id, index_is_id=True)
                    formats = [] if dbfmts is None else \
                        [f.lower() for f in dbfmts.split(',')]
                    if set(formats).intersection(available_input_formats()) \
                            and set(settings.format_map).intersection(available_output_formats()):
                        auto.append(id)
            if auto:
                format = None
                for fmt in settings.format_map:
                    if fmt in list(set(settings.format_map).intersection(set(available_output_formats()))):
                        format = fmt
                        break
                if format is not None:
                    autos = [self.library_view.model().db.title(id, index_is_id=True) for id in auto]
                    if self.auto_convert_question(
                        _('Auto convert the following books before uploading to '
                            'the device?'), autos):
                        self.iactions['Convert Books'].auto_convert_catalogs(auto, format)
            files = [f for f in files if f is not None]
            if not files:
                dynamic.set('catalogs_to_be_synced', set())
                return
            metadata = self.library_view.model().metadata_for(ids)
            names = []
            for book_id, mi in zip(ids, metadata):
                prefix = ascii_filename(mi.title)
                if not isinstance(prefix, str):
                    prefix = prefix.decode(preferred_encoding, 'replace')
                prefix = ascii_filename(prefix)
                names.append(f'{prefix}_{book_id}{os.path.splitext(files[-1])[1]}')
                self.update_thumbnail(mi)
            dynamic.set('catalogs_to_be_synced', set())
            if files:
                remove = []
                space = {self.location_manager.free[0]: None,
                    self.location_manager.free[1]: 'carda',
                    self.location_manager.free[2]: 'cardb'}
                on_card = space.get(sorted(space.keys(), reverse=True)[0], None)
                self.upload_books(files, names, metadata,
                        on_card=on_card,
                        memory=[files, remove])
                self.status_bar.show_message(_('Sending catalogs to device.'), 5000)

    @property
    def news_to_be_synced(self):
        'Set of ids to be sent to device'
        ans = []
        try:
            ans = self.library_view.model().db.new_api.pref('news_to_be_synced',
                    [])
        except Exception:
            import traceback
            traceback.print_exc()
        return set(ans)

    @news_to_be_synced.setter
    def news_to_be_synced(self, ids):
        try:
            self.library_view.model().db.new_api.set_pref('news_to_be_synced',
                    list(ids))
        except Exception:
            import traceback
            traceback.print_exc()

    def sync_news(self, send_ids=None, do_auto_convert=True):
        if self.device_connected:
            del_on_upload = config['delete_news_from_library_on_upload']
            settings = self.device_manager.device.settings()
            ids = list(self.news_to_be_synced) if send_ids is None else send_ids
            ids = [book_id for book_id in ids if self.library_view.model().db.has_id(book_id)]
            with BusyCursor():
                files, _auto_ids = self.library_view.model().get_preferred_formats_from_ids(
                                ids, settings.format_map,
                                exclude_auto=do_auto_convert)
            auto = []
            if do_auto_convert and _auto_ids:
                for book_id in _auto_ids:
                    dbfmts = self.library_view.model().db.formats(book_id, index_is_id=True)
                    formats = [] if dbfmts is None else \
                        [f.lower() for f in dbfmts.split(',')]
                    if set(formats).intersection(available_input_formats()) \
                            and set(settings.format_map).intersection(available_output_formats()):
                        auto.append(book_id)
            if auto:
                format = None
                for fmt in settings.format_map:
                    if fmt in list(set(settings.format_map).intersection(set(available_output_formats()))):
                        format = fmt
                        break
                if format is not None:
                    autos = [self.library_view.model().db.title(book_id, index_is_id=True) for book_id in auto]
                    if self.auto_convert_question(
                        _('Auto convert the following books before uploading to '
                            'the device?'), autos):
                        self.iactions['Convert Books'].auto_convert_news(auto, format)
            files = [f for f in files if f is not None]
            if not files:
                self.news_to_be_synced = set()
                return
            metadata = self.library_view.model().metadata_for(ids)
            names = []
            for book_id, mi in zip(ids, metadata):
                prefix = ascii_filename(mi.title)
                if not isinstance(prefix, str):
                    prefix = prefix.decode(preferred_encoding, 'replace')
                prefix = ascii_filename(prefix)
                names.append(f'{prefix}_{book_id}{os.path.splitext(files[-1])[1]}')
                self.update_thumbnail(mi)
            self.news_to_be_synced = set()
            if config['upload_news_to_device'] and files:
                remove = ids if del_on_upload else []
                space = {self.location_manager.free[0]: None,
                    self.location_manager.free[1]: 'carda',
                    self.location_manager.free[2]: 'cardb'}
                on_card = space.get(sorted(space.keys(), reverse=True)[0], None)
                try:
                    total_size = sum(os.stat(f).st_size for f in files)
                except Exception:
                    try:
                        import traceback
                        traceback.print_exc()
                    except Exception:
                        pass
                    total_size = self.location_manager.free[0]
                loc = tweaks['send_news_to_device_location']
                loc_index = {'carda': 1, 'cardb': 2}.get(loc, 0)
                if self.location_manager.free[loc_index] > total_size + (1024**2):
                    # Send news to main memory if enough space available
                    # as some devices like the Nook Color cannot handle
                    # periodicals on SD cards properly
                    on_card = loc if loc in ('carda', 'cardb') else None
                self.upload_books(files, names, metadata,
                        on_card=on_card,
                        memory=[files, remove])
                self.status_bar.show_message(_('Sending news to device.'), 5000)

    def sync_to_device(self, on_card, delete_from_library,
            specific_format=None, send_ids=None, do_auto_convert=True):
        ids = [self.library_view.model().id(r)
               for r in self.library_view.selectionModel().selectedRows()] \
                                if send_ids is None else send_ids
        if not self.device_manager or not ids or len(ids) == 0 or \
                not self.device_manager.is_device_connected:
            return

        settings = self.device_manager.device.settings()

        with BusyCursor():
            _files, _auto_ids = self.library_view.model().get_preferred_formats_from_ids(ids,
                                    settings.format_map,
                                    specific_format=specific_format,
                                    exclude_auto=do_auto_convert)
        if do_auto_convert:
            ok_ids = list(set(ids).difference(_auto_ids))
            ids = [i for i in ids if i in ok_ids]
        else:
            _auto_ids = []

        metadata = self.library_view.model().metadata_for(ids)
        ids = iter(ids)
        for mi in metadata:
            self.update_thumbnail(mi)
        imetadata = iter(metadata)

        bad, good, gf, names, remove_ids = [], [], [], [], []
        for f in _files:
            mi = next(imetadata)
            id = next(ids)
            if f is None:
                bad.append(mi.title)
            else:
                remove_ids.append(id)
                good.append(mi)
                gf.append(f)
                t = mi.title
                if not t:
                    t = _('Unknown')
                a = mi.format_authors()
                if not a:
                    a = _('Unknown')
                prefix = ascii_filename(t+' - '+a)
                if not isinstance(prefix, str):
                    prefix = prefix.decode(preferred_encoding, 'replace')
                prefix = ascii_filename(prefix)
                names.append(f'{prefix}_{id}{os.path.splitext(f)[1]}')
        remove = remove_ids if delete_from_library else []
        self.upload_books(gf, names, good, on_card, memory=(_files, remove))
        self.status_bar.show_message(_('Sending books to device.'), 5000)

        auto = []
        if _auto_ids != []:
            for id in _auto_ids:
                if specific_format is None:
                    formats = self.library_view.model().db.formats(id, index_is_id=True)
                    formats = formats.split(',') if formats is not None else []
                    formats = [f.lower().strip() for f in formats]
                    if (list(set(formats).intersection(available_input_formats())) != [] and
                        list(set(settings.format_map).intersection(available_output_formats())) != []):
                        auto.append(id)
                    else:
                        bad.append(self.library_view.model().db.title(id, index_is_id=True))
                elif specific_format in list(set(settings.format_map).intersection(set(available_output_formats()))):
                    auto.append(id)
                else:
                    bad.append(self.library_view.model().db.title(id, index_is_id=True))

        if auto != []:
            format = specific_format if specific_format in \
                            list(set(settings.format_map).intersection(set(available_output_formats()))) \
                            else None
            if not format:
                for fmt in settings.format_map:
                    if fmt in list(set(settings.format_map).intersection(set(available_output_formats()))):
                        format = fmt
                        break
            if not format:
                bad += auto
            else:
                autos = [self.library_view.model().db.title(id, index_is_id=True) for id in auto]
                if self.auto_convert_question(
                    _('Auto convert the following books before uploading to '
                        'the device?'), autos):
                    self.iactions['Convert Books'].auto_convert(auto, on_card, format)

        if bad:
            bad = '\n'.join(f'{i}' for i in bad)
            d = warning_dialog(self, _('No suitable formats'),
                    _('Could not upload the following books to the device, '
                'as no suitable formats were found. Convert the book(s) to a '
                'format supported by your device first.'
                ), bad)
            d.exec()

    def upload_dirtied_booklists(self):
        '''
        Upload metadata to device.
        '''
        plugboards = self.library_view.model().db.new_api.pref('plugboards', {})
        self.device_manager.sync_booklists(Dispatcher(lambda x: x),
                                           self.booklists(), plugboards)

    def upload_booklists(self, add_as_step_to_job=None):
        '''
        Upload metadata to device.
        '''
        plugboards = self.library_view.model().db.new_api.pref('plugboards', {})
        self.device_manager.sync_booklists(FunctionDispatcher(self.metadata_synced),
                                           self.booklists(), plugboards,
                                           add_as_step_to_job=add_as_step_to_job)

    def metadata_synced(self, job):
        '''
        Called once metadata has been uploaded.
        '''
        if job.failed:
            self.device_job_exception(job)
            return
        cp, fs = job.result
        self.location_manager.update_devices(cp, fs,
                self.device_manager.device.icon)
        # reset the views so that up-to-date info is shown. These need to be
        # here because some drivers update collections in sync_booklists
        cv, row = self.current_view(), -1
        if cv is not self.library_view:
            row = cv.currentIndex().row()
        self.memory_view.reset()
        self.card_a_view.reset()
        self.card_b_view.reset()
        if row > -1:
            cv.set_current_row(row)

    def _upload_collections(self, job):
        if job.failed:
            self.device_job_exception(job)

    def upload_collections(self, booklist, view=None, oncard=None):
        return self.device_manager.upload_collections(self._upload_collections,
                                                       booklist, oncard)

    def upload_books(self, files, names, metadata, on_card=None, memory=None):
        '''
        Upload books to device.
        :param files: List of either paths to files or file like objects
        '''
        titles = [i.title for i in metadata]
        plugboards = self.library_view.model().db.new_api.pref('plugboards', {})
        job = self.device_manager.upload_books(
                FunctionDispatcher(self.books_uploaded),
                files, names, on_card=on_card,
                metadata=metadata, titles=titles, plugboards=plugboards
              )
        self.upload_memory[job] = (metadata, on_card, memory, files)

    def books_uploaded(self, job):
        '''
        Called once books have been uploaded.
        '''
        metadata, on_card, memory, files = self.upload_memory.pop(job)

        if job.exception is not None:
            if isinstance(job.exception, FreeSpaceError):
                where = 'in main memory.' if 'memory' in str(job.exception) \
                        else 'on the storage card.'
                titles = '\n'.join(['<li>'+mi.title+'</li>'
                                    for mi in metadata])
                d = error_dialog(self, _('No space on device'),
                                 _('<p>Cannot upload books to device there '
                                 'is no more free space available ')+where+
                                 f'</p>\n<ul>{titles}</ul>')
                d.exec()
            elif isinstance(job.exception, WrongDestinationError):
                error_dialog(self, _('Incorrect destination'),
                        str(job.exception), show=True)
            else:
                self.device_job_exception(job)
            return

        try:
            self.device_manager.add_books_to_metadata(job.result,
                    metadata, self.booklists())
        except Exception:
            traceback.print_exc()
            raise

        books_to_be_deleted = []
        if memory and memory[1]:
            books_to_be_deleted = memory[1]
            self.library_view.model().delete_books_by_id(books_to_be_deleted)

        # There are some cases where sending a book to the device overwrites a
        # book already there with a different book. This happens frequently in
        # news. When this happens, the book match indication will be wrong
        # because the UUID changed. Force both the device and the library view
        # to refresh the flags. Set_books_in_library could upload the booklists.
        # If it does not, then do it here.
        if not self.set_books_in_library(self.booklists(), reset=True,
                                     add_as_step_to_job=job, do_device_sync=False):
            self.upload_booklists(job)
        self.refresh_ondevice()

        view = self.card_a_view if on_card == 'carda' else \
            self.card_b_view if on_card == 'cardb' else self.memory_view
        view.model().resort(reset=False)
        view.model().research()
        if files:
            for f in files:
                # Remove temporary files
                try:
                    rem = not getattr(
                            self.device_manager.device,
                            'KEEP_TEMP_FILES_AFTER_UPLOAD', False)
                    if rem and 'caltmpfmt.' in f:
                        os.remove(f)
                except Exception:
                    pass

    def update_metadata_on_device(self):
        self.set_books_in_library(self.booklists(), reset=True, force_send=True)
        self.refresh_ondevice()

    def set_current_library_information(self, library_name, library_uuid, field_metadata):
        self.device_manager.set_current_library_uuid(library_uuid)
        if self.device_manager.is_device_connected:
            self.device_manager.set_library_information(None, library_name,
                                            library_uuid, field_metadata)

    def book_on_device(self, id, reset=False):
        '''
        Return an indication of whether the given book represented by its db id
        is on the currently connected device. It returns a 5 element list. The
        first three elements represent memory locations main, carda, and cardb,
        and are true if the book is identifiably in that memory. The fourth
        is a count of how many instances of the book were found across all
        the memory locations. The fifth is a set of paths to the
        matching books on the device.
        '''
        loc = [None, None, None, 0, set()]

        if reset:
            self.book_db_id_cache = None
            self.book_db_id_counts = None
            self.book_db_uuid_path_map = None
            return

        if not self.device_manager.is_device_connected or \
                        not hasattr(self, 'db_book_uuid_cache'):
            return loc

        if self.book_db_id_cache is None:
            self.book_db_id_cache = []
            self.book_db_id_counts = {}
            self.book_db_uuid_path_map = {}
            for i, l in enumerate(self.booklists()):
                self.book_db_id_cache.append(set())
                for book in l:
                    db_id = getattr(book, 'application_id', None)
                    if db_id is not None:
                        # increment the count of books on the device with this
                        # db_id.
                        self.book_db_id_cache[i].add(db_id)
                        if db_id not in self.book_db_uuid_path_map:
                            self.book_db_uuid_path_map[db_id] = set()
                        if getattr(book, 'lpath', False):
                            self.book_db_uuid_path_map[db_id].add(book.lpath)
                        c = self.book_db_id_counts.get(db_id, 0)
                        self.book_db_id_counts[db_id] = c + 1

        for i, l in enumerate(self.booklists()):
            if id in self.book_db_id_cache[i]:
                loc[i] = True
                loc[3] = self.book_db_id_counts.get(id, 0)
                loc[4] |= self.book_db_uuid_path_map[id]
        return loc

    def update_thumbnail(self, book):
        if book.cover and os.access(book.cover, os.R_OK):
            with open(book.cover, 'rb') as f:
                book.thumbnail = self.cover_to_thumbnail(f.read())
        else:
            cprefs = self.default_thumbnail_prefs
            book.thumbnail = (cprefs['cover_width'], cprefs['cover_height'], generate_cover(book, prefs=cprefs))

    def set_books_in_library(self, booklists, reset=False, add_as_step_to_job=None,
                             force_send=False, do_device_sync=True):
        '''
        Set the ondevice indications in the device database.
        This method should be called before book_on_device is called, because
        it sets the application_id for matched books. Book_on_device uses that
        to both speed up matching and to count matches.
        '''

        if not self.device_manager.is_device_connected:
            return False

        # It might be possible to get here without having initialized the
        # library view. In this case, simply give up
        try:
            db = self.library_view.model().db
        except Exception:
            return False

        # Define the cleaning function
        string_pat = re.compile(r'(?u)\W|[_]')

        def clean_string(x):
            try:
                # Convert to lowercase if x is not None or empty
                x = x.lower() if x else ''
            except Exception:
                x = ''
            # Perform regex substitution
            return string_pat.sub('', x)

        update_metadata = (
           device_prefs['manage_device_metadata'] == 'on_connect' or force_send)

        get_covers = False
        desired_thumbnail_height = 0
        if update_metadata and self.device_manager.is_device_connected:
            if self.device_manager.device.WANTS_UPDATED_THUMBNAILS:
                get_covers = True
                desired_thumbnail_height = self.device_manager.device.THUMBNAIL_HEIGHT

        # Force a reset if the caches are not initialized
        if reset or not hasattr(self, 'db_book_title_cache'):
            # Build a cache (map) of the library, so the search isn't On**2
            db_book_title_cache = {}
            db_book_uuid_cache = {}

            for id_ in db.data.iterallids():
                title = clean_string(db.title(id_, index_is_id=True))
                if title not in db_book_title_cache:
                    db_book_title_cache[title] = \
                                {'authors':{}, 'author_sort':{}, 'db_ids':{}}
                # If there are multiple books in the library with the same title
                # and author, then remember the last one. That is OK, because as
                # we can't tell the difference between the books, one is as good
                # as another.
                authors = clean_string(db.authors(id_, index_is_id=True))
                if authors:
                    db_book_title_cache[title]['authors'][authors] = id_
                if db.author_sort(id_, index_is_id=True):
                    aus = clean_string(db.author_sort(id_, index_is_id=True))
                    db_book_title_cache[title]['author_sort'][aus] = id_
                db_book_title_cache[title]['db_ids'][id_] = id_
                db_book_uuid_cache[db.uuid(id_, index_is_id=True)] = id_
            self.db_book_title_cache = db_book_title_cache
            self.db_book_uuid_cache = db_book_uuid_cache

        book_ids_to_refresh = set()
        book_formats_to_send = []
        books_with_future_dates = []
        first_call_to_synchronize_with_db = [True]

        def update_book(id_, book):
            if not update_metadata:
                return
            mi = db.get_metadata(id_, index_is_id=True, get_cover=get_covers)
            book.smart_update(mi, replace_metadata=True)
            if get_covers and desired_thumbnail_height != 0:
                self.update_thumbnail(book)

        def updateq(id_, book):
            try:
                if not update_metadata:
                    return False

                if do_device_sync and self.device_manager.device is not None:
                    set_of_ids, (fmt_name, date_bad) = \
                            self.device_manager.device.synchronize_with_db(db, id_, book,
                                           first_call_to_synchronize_with_db[0])
                    first_call_to_synchronize_with_db[0] = False
                    if date_bad:
                        books_with_future_dates.append(book.title)
                    elif fmt_name is not None:
                        book_formats_to_send.append((id_, fmt_name))
                    if set_of_ids is not None:
                        book_ids_to_refresh.update(set_of_ids)
                        return True

                return (db.metadata_last_modified(id_, index_is_id=True) !=
                        getattr(book, 'last_modified', None) or
                        (isinstance(getattr(book, 'thumbnail', None), (list, tuple)) and
                         max(book.thumbnail[0], book.thumbnail[1]) != desired_thumbnail_height
                        )
                       )
            except Exception:
                return True

        def get_by_author(book, d, author):
            book_id = d['authors'].get(author)
            if book_id is not None:
                book.in_library = 'AUTHOR'
                book.application_id = book_id
                update_book(book_id, book)
                return True
            book_id = d['author_sort'].get(author)
            if book_id is not None:
                book.in_library = 'AUTH_SORT'
                book.application_id = book_id
                update_book(book_id, book)
                return True
            return False

        # Now iterate through all the books on the device, setting the
        # in_library field. If the UUID matches a book in the library, then
        # do not consider that book for other matching. In all cases set
        # the application_id to the db_id of the matching book. This value
        # will be used by books_on_device to indicate matches. While we are
        # going by, update the metadata for a book if automatic management is on

        total_book_count = 0
        for booklist in booklists:
            for book in booklist:
                if book:
                    total_book_count += 1
        debug_print('DeviceJob: set_books_in_library: books to process=', total_book_count)

        start_time = time.time()

        with BusyCursor():
            current_book_count = 0
            for booklist in booklists:
                for book in booklist:
                    if current_book_count % 100 == 0:
                        self.status_bar.show_message(
                                _('Analyzing books on the device: %d%% finished')%(
                                    int((float(current_book_count)/total_book_count)*100.0)), show_notification=False)

                    # I am assuming that this sort-of multi-threading won't break
                    # anything. Reasons: excluding UI events prevents the user
                    # from explicitly changing anything, and (in theory) no
                    # changes are happening because of timers and the like.
                    # Why every tenth book? WAG balancing performance in the
                    # loop with preventing App Not Responding errors
                    if current_book_count % 10 == 0:
                        QCoreApplication.processEvents(
                            flags=QEventLoop.ProcessEventsFlag.ExcludeUserInputEvents|QEventLoop.ProcessEventsFlag.ExcludeSocketNotifiers)
                    current_book_count += 1
                    book.in_library = None
                    if getattr(book, 'uuid', None) in self.db_book_uuid_cache:
                        id_ = db_book_uuid_cache[book.uuid]
                        if updateq(id_, book):
                            update_book(id_, book)
                        book.in_library = 'UUID'
                        # ensure that the correct application_id is set
                        book.application_id = id_
                        continue
                    # No UUID exact match. Try metadata matching.
                    book_title = clean_string(book.title)
                    d = self.db_book_title_cache.get(book_title, None)
                    if d is not None:
                        # At this point we know that the title matches. The book
                        # will match if any of the db_id, author, or author_sort
                        # also match.
                        if getattr(book, 'application_id', None) in d['db_ids']:
                            id_ = getattr(book, 'application_id', None)
                            update_book(id_, book)
                            book.in_library = 'APP_ID'
                            # app_id already matches a db_id. No need to set it.
                            continue
                        # Sonys know their db_id independent of the application_id
                        # in the metadata cache. Check that as well.
                        if getattr(book, 'db_id', None) in d['db_ids']:
                            update_book(book.db_id, book)
                            book.in_library = 'DB_ID'
                            book.application_id = book.db_id
                            continue
                        # We now know that the application_id is not right. Set it
                        # to None to prevent book_on_device from accidentally
                        # matching on it. It will be set to a correct value below if
                        # the book is matched with one in the library
                        book.application_id = None
                        if book.authors:
                            # Compare against both author and author sort, because
                            # either can appear as the author
                            all_book_authors = clean_string(authors_to_string(book.authors))
                            if not get_by_author(book, d, all_book_authors):
                                for author in book.authors:
                                    if get_by_author(book, d, clean_string(author)):
                                        break
                    else:
                        # Book definitely not matched. Clear its application ID
                        book.application_id = None
                    # Set author_sort if it isn't already
                    asort = getattr(book, 'author_sort', None)
                    if not asort and book.authors:
                        book.author_sort = self.library_view.model().db.\
                                    author_sort_from_authors(book.authors)

            if update_metadata:
                if self.device_manager.is_device_connected:
                    plugboards = self.library_view.model().db.new_api.pref('plugboards', {})
                    self.device_manager.sync_booklists(
                                FunctionDispatcher(self.metadata_synced), booklists,
                                plugboards, add_as_step_to_job)

            if book_ids_to_refresh:
                try:
                    prints('DeviceJob: set_books_in_library refreshing GUI for ',
                           len(book_ids_to_refresh), 'books')
                    self.library_view.model().refresh_ids(book_ids_to_refresh,
                                      current_row=self.library_view.currentIndex().row())
                except Exception:
                    # This shouldn't ever happen, but just in case ...
                    traceback.print_exc()

            # Sync books if necessary
            try:
                files, names, metadata = [], [], []
                for id_, fmt_name in book_formats_to_send:
                    if DEBUG:
                        prints('DeviceJob: Syncing book. id:', id_, 'name from device', fmt_name)
                    ext = os.path.splitext(fmt_name)[1][1:]
                    fmt_info = db.new_api.format_metadata(id_, ext)
                    if fmt_info:
                        try:
                            pt = PersistentTemporaryFile(suffix='caltmpfmt.'+ext)
                            db.new_api.copy_format_to(id_, ext, pt)
                            pt.close()
                            files.append(filename_to_unicode(os.path.abspath(pt.name)))
                            names.append(fmt_name)
                            mi = db.new_api.get_metadata(id_, get_cover=True)
                            self.update_thumbnail(mi)
                            metadata.append(mi)
                        except Exception:
                            prints('Problem creating temporary file for', fmt_name)
                            traceback.print_exc()
                    elif DEBUG:
                        prints("DeviceJob: book doesn't have that format")
                if files:
                    self.upload_books(files, names, metadata)
            except Exception:
                # Shouldn't ever happen, but just in case
                traceback.print_exc()

            # Inform user about future-dated books
            try:
                if books_with_future_dates:
                    d = error_dialog(self, _('Book format sync problem'),
                                 _('Some book formats in your library cannot be '
                                   'synced because they have dates in the future'),
                                 det_msg='\n'.join(books_with_future_dates),
                                 show=False,
                                 show_copy_button=True)
                    d.show()
            except Exception:
                traceback.print_exc()

        debug_print('DeviceJob: set_books_in_library finished: time=', time.time() - start_time)
        # The status line is reset when the job finishes
        return update_metadata
    # }}}
