# -*- coding: utf-8 -*-

# PyNX - Python tools for Nano-structures Crystallography
#   (c) 2016-present : ESRF-European Synchrotron Radiation Facility
#       authors:
#         Vincent Favre-Nicolin, favre@esrf.fr

import sys
import os
import timeit
import argparse

from ...utils import h5py
import numpy as np
import re
import time
import locale
import fabio
from silx.io.specfile import SpecFile
from .runner import PtychoRunner, PtychoRunnerScan, PtychoRunnerException, default_params as params0
from .parser import ActionPtychoMotors

helptext_epilog = """
Examples:
    
* ``pynx-ptycho-id01 --h5file /data/visitor/mi1421/id01/sample/sample_0001/sample_0001.h5 \
--scan 2 --ptychomotors pix piz x -y --probe focus,400e-6x400e-6,1.57 --detector eiger2M \
--detectordistance 5 --liveplot --roi 0 380 1900 2160 \
--algorithm 'analysis,ML**100,position=1,ML**200,DM**300,nbprobe=3,probe=1'``

(old version using spec data):

* ``pynx-ptycho-id01 --specfile siemens.spec --scan 57 --detector_distance 1.3 --ptycho_motors pix piz -x y \
--probe focus,60e-6x200e-6,0.09 --algorithm 'analysis,ML**100,DM**200,nbprobe=3,probe=1'\
--mask maxipix --verbose 10 --save all --saveplot --liveplot``
"""

params_beamline = {
    'specfile': None,
    'detector': None,
    'imgcounter': 'auto',
    'imgname': None,
    'crazy_eiger': False,
    'kmapfile': None,
    'monitor': None,
    'ptychomotors': None,
    'instrument': 'ESRF id01'
}

default_params = params0.copy()
for k, v in params_beamline.items():
    default_params[k] = v


class PtychoRunnerScanID01(PtychoRunnerScan):

    def load_scan(self):
        if self.params['specfile'] is not None:
            # SPEC
            self.s = SpecFile(self.params['specfile'])['%d.1' % self.scan]
            self.h = self.s.scan_header_dict
            self.print('Read scan:', self.h['S'])

            if self.params['imgcounter'] == 'auto':
                if 'ei2minr' in self.s.labels:
                    self.params['imgcounter'] = 'ei2minr'
                elif 'mpx4inr' in self.s.labels:
                    self.params['imgcounter'] = 'mpx4inr'
                self.print("Using image counter: %s" % (self.params['imgcounter']))

            m = self.params['ptychomotors']
            xmot, ymot = m[0:2]

            if self.s.labels.count(xmot) == 0 or self.s.labels.count(ymot) == 0:
                raise PtychoRunnerException(
                    'Ptycho motors (%s, %s) not found in scan #%d of specfile:%s' % (
                        xmot, ymot, self.scan, self.params['specfile']))

            self.x, self.y = self.s.data_column_by_name(xmot), self.s.data_column_by_name(ymot)
            if len(m) == 4:
                x, y = self.x, self.y
                self.x, self.y = eval(m[2]), eval(m[3])
            if len(self.x) < 4:
                raise PtychoRunnerException("Less than 4 scan positions, is this a ptycho scan ?")
            # Spec values are in microns, convert to meters
            self.x *= 1e-6
            self.y *= 1e-6

            if self.params['kmapfile'] is None and self.params['imgcounter'] != 'auto':
                # Ignore image numbers if we read a kmap file, assume the images are stored sequentially in a single file
                # If imgcounter == 'auto', assume images are numbered from zero if mpx4inr or ei2minr were not found
                imgn = self.s.data_column_by_name(self.params['imgcounter']).astype(int)
            else:
                imgn = np.arange(len(self.x), dtype=int)

            if self.params['monitor'] is not None:
                mon = self.s.data_column_by_name(self.params['monitor'])
                self.raw_data_monitor = mon
                mon0 = np.median(mon)
                mon /= mon0
                self.validframes = np.where(mon > 0.1)
                if len(self.validframes) != len(mon):
                    self.print('WARNING: The following frames have a monitor value < 0.1'
                               ' the median value and will be ignored (no beam ?)')
                    self.print(np.where(mon <= (mon0 * 0.1)))
                self.x = np.take(self.x, self.validframes)
                self.y = np.take(self.y, self.validframes)
                imgn = np.take(imgn, self.validframes)
                self.raw_data_monitor = np.take(self.raw_data_monitor, self.validframes)
            else:
                mon = None

            if 'xyrange' in self.params:
                xmin, xmax, ymin, ymax = self.params['xyrange']
                print("Restricting scan positions to %f < x < %f and %f < y < %f" % (xmin, xmax, ymin, ymax))
                idx = np.where((self.x >= xmin) * (self.x <= xmax) * (self.y >= ymin) * (self.y <= ymax))[0]
                if len(idx) < 10:
                    raise PtychoRunnerException("Only %d points remaining after applying xyrange"
                                                "constraint. original range: %5e<x<%5e %5e<y<%5e"
                                                % (len(idx), self.x.min(), self.x.max(), self.y.min(), self.y.max()))
                else:
                    print("   ... %d/%d remaining positions" % (len(idx), len(x)))
                imgn = imgn.take(idx)
                self.x = self.x.take(idx)
                self.y = self.y.take(idx)

            if self.params['moduloframe'] is not None:
                n1, n2 = self.params['moduloframe']
                idx = np.where(imgn % n1 == n2)[0]
                imgn = imgn.take(idx)
                self.x = self.x.take(idx)
                self.y = self.y.take(idx)

            if self.params['maxframe'] is not None:
                N = self.params['maxframe']
                if len(imgn) > N:
                    self.print("MAXFRAME: only using first %d frames" % (N))
                    imgn = imgn[:N]
                    self.x = self.x[:N]
                    self.y = self.y[:N]

            # Check we really have frames available (save is enabled)
            if int(round(imgn.max() - imgn.min()) + 1) < len(imgn):
                if imgn.min() == imgn.max():
                    raise PtychoRunnerException(
                        "Frame range (%d-%d) does not match expected number of frames (%d). Were frames SAVED ?"
                        % (imgn.min(), imgn.max(), len(imgn)))
                else:
                    raise PtychoRunnerException("Frame range (%d-%d) does not match expected number of frames (%d)." % (
                        imgn.min(), imgn.max(), len(imgn)))

            self.imgn = imgn

        elif self.params['data'] is not None:
            # BLISS hdf5 
            if os.path.isfile(self.params['data']) is False:
                raise PtychoRunnerException("h5 file does not exist: %s" % (self.params['data']))
            h5f = h5py.File(self.params['data'], 'r')

            if self.params['ptychomotors'] is not None:
                m = self.params['ptychomotors']
                xmot, ymot = m[0:2]
                if f'{self.scan}.1/measurement/{xmot}' not in h5f or \
                        f'{self.scan}.1/measurement/{ymot}' not in h5f:
                    raise PtychoRunnerException(
                        f'Ptycho motors ({xmot}, {ymot}) not found in '
                        f'{self.params["data"]}:{self.scan}.1/measurement/')
                x = h5f[f'{self.scan}.1/measurement/{xmot}'][()]
                y = h5f[f'{self.scan}.1/measurement/{ymot}'][()]

                if len(m) == 4:
                    x, y = eval(m[2]), eval(m[3])
            else:
                # Currently this'll never be used as the lack of ptychomotors will raise an exception
                print("ptychomotors=... not supplied, see if a lookupscan with pix/piy/piz was used... "
                      "You may need to correct the motors orientation with e.g. xy=...")
                title = h5f[f'{self.scan}.1/title'][()].decode("utf-8")
                pattern = r"\w{3},\w{3}"
                xmot, ymot = re.findall(pattern, title)[0].split(',')

                x = h5f[f'{self.scan}.1/measurement/{xmot}'][()] * 1e-6
                y = h5f[f'{self.scan}.1/measurement/{ymot}'][()] * 1e-6
                # x, y = h5f['/entry_1/sample_1/geometry_1/translation'][0:2]

            if len(x) < 4:
                raise PtychoRunnerException("Less than 4 scan positions, is this a ptycho scan ?")

            imgn = np.arange(len(x), dtype=int)

            if self.params['monitor'] is not None:
                mon = h5f[f"{self.scan}.1/measurement/{self.params['monitor']}"][()]
                self.raw_data_monitor = mon.copy()

                mon0 = np.median(mon)
                mon /= mon0
                validframes = np.where(mon > 0.2)[0]
                if len(validframes) != len(mon):
                    print('WARNING: The following frames have a monitor value < 0.2'
                          ' x the median value and will be ignored (no beam ?)')
                    print(np.where(mon <= 0.2))
                x = np.take(x, validframes)
                y = np.take(y, validframes)
                imgn = np.take(imgn, validframes)
                self.raw_data_monitor = np.take(self.raw_data_monitor, self.validframes)

            if self.params['xyrange'] is not None:
                xmin, xmax, ymin, ymax = self.params['xyrange']
                print("Restricting scan positions to %f < x < %f and %f < y < %f" % (xmin, xmax, ymin, ymax))
                idx = np.where((x >= xmin) * (x <= xmax) * (y >= ymin) * (y <= ymax))[0]
                if len(idx) < 10:
                    raise PtychoRunnerException("Only %d points remaining after applying the xyrange "
                                                "constraint. original range: %5e<x<%5e %5e<y<%5e"
                                                % (len(idx), x.min(), x.max(), y.min(), y.max()))
                else:
                    print("   ... %d/%d remaining positions" % (len(idx), len(x)))
                imgn = imgn.take(idx)
                x = x.take(idx)
                y = y.take(idx)

            if self.params['moduloframe'] is not None:
                n1, n2 = self.params['moduloframe']
                idx = np.where(imgn % n1 == n2)[0]
                imgn = imgn.take(idx)
                x = x.take(idx)
                y = y.take(idx)

            if self.params['maxframe'] is not None:
                N = self.params['maxframe']
                if len(imgn) > N:
                    print("MAXFRAME: only using first %d frames" % (N))
                    imgn = imgn[:N]
                    x = x[:N]
                    y = y[:N]

            self.x, self.y, self.imgn = x, y, imgn

    def load_data(self):
        if self.params['specfile'] is not None:
            # SPEC
            if self.params['pixelsize'] is None:
                self.params['pixelsize'] = 55e-6
            self.s = SpecFile(self.params['specfile'])['%d.1' % (self.scan)]
            self.h = self.s.scan_header_dict

            date_string = self.h['D']  # 'Wed Mar 23 14:41:56 2016'
            date_string = date_string[date_string.find(' ') + 1:]
            pattern = '%b %d %H:%M:%S %Y'
            try:
                lc = locale._setlocale(locale.LC_ALL)
                locale._setlocale(locale.LC_ALL, 'C')
                epoch = int(time.mktime(time.strptime(date_string, pattern)))
                locale._setlocale(locale.LC_ALL, lc)
                self.print("SPEC date: %s -> %s " % (date_string, time.strftime("%Y-%m-%dT%H:%M:%S%z",
                                                                                time.localtime(epoch))), epoch)
            except ValueError:
                self.print("Could not extract time from spec header,"
                           " unrecognized format: %s, expected:" % (date_string) + pattern)

            if self.params['nrj'] is None:
                self.print("Reading nrj", self.h['UMONO'])
                self.params['nrj'] = float(self.h['UMONO'].split('mononrj=')[1].split('ke')[0])

            if self.params['detectordistance'] is None and 'UDETCALIB' in self.h:
                if 'det_distance_CC=' in self.h['UDETCALIB']:
                    # UDETCALIB cen_pix_x=18.347,cen_pix_y=278.971,pixperdeg=445.001,det_distance_CC=1402.175,
                    #   det_distance_COM=1401.096,timestamp=20170926...
                    self.params['detectordistance'] = float(self.h['UDETCALIB'].split('stance_CC=')[1].split(',')[0])
                    self.print("Reading detector distance from spec data: det_distance_CC=%6.3fm"
                               % self.params['detectordistance'])

            if self.params['detectordistance'] is None:
                raise PtychoRunnerException(
                    'Missing argument: no detectordistance given or from scpec header (UDETCALIB)')

            # Load all frames
            imgn = self.imgn
            vimg = None
            d0 = 0
            t0 = timeit.default_timer()
            if self.params['kmapfile'] is not None:
                sys.stdout.write("Reading frames from KMAP file (this WILL take a while)...")
                sys.stdout.flush()
                kfile = fabio.open(self.params['kmapfile'])
                if kfile.getNbFrames() < len(imgn):
                    raise PtychoRunnerException("KMAP: only %d frames instead of %d in data file (%s) ! "
                                                "Did you save all frames in a single file ?" %
                                                (kfile.getNbFrames(), len(imgn), self.params['kmapfile']))
                ii = 0
                for i in imgn:
                    if (i - imgn[0]) % 20 == 0:
                        sys.stdout.write('%d ' % (i - imgn[0]))
                        sys.stdout.flush()
                    frame = kfile.getframe(i).data
                    if vimg is None:
                        vimg = np.empty((len(imgn), frame.shape[0], frame.shape[1]), dtype=frame.dtype)
                    vimg[ii] = frame
                    d0 += frame
                    ii += 1
            else:
                if self.params['imgname'] is None:
                    if 'ULIMA_eiger2M' in self.h:
                        imgname = self.h['ULIMA_eiger2M'].strip().split('_eiger2M_')[0] + '_eiger2M_%05d.edf.gz'
                        self.print("Using Eiger 2M detector images: %s" % (imgname))
                    else:
                        imgname = self.h['ULIMA_mpx4'].strip().split('_mpx4_')[0] + '_mpx4_%05d.edf.gz'
                        self.print("Using Maxipix mpx4 detector images: %s" % (imgname))
                else:
                    imgname = self.params['imgname']
                if self.mpi_master:
                    sys.stdout.write('Reading frames: ')
                ii = 0
                for i in imgn:
                    if (i - imgn[0]) % 20 == 0 and self.mpi_master:
                        sys.stdout.write('%d ' % (i - imgn[0]))
                        sys.stdout.flush()
                    frame = fabio.open(imgname % i).data
                    if vimg is None:
                        vimg = np.empty((len(imgn), frame.shape[0], frame.shape[1]), dtype=frame.dtype)
                    vimg[ii] = frame
                    d0 += frame
                    ii += 1
            self.print("\n")
            dt = timeit.default_timer() - t0
            print('Time to read all frames: %4.1fs [%5.2f Mpixel/s]' % (dt, d0.size * len(vimg) / 1e6 / dt))

            self.raw_data = vimg
            self.load_data_post_process()

        elif self.params['data'] is not None:
            # BLISS
            imgn = self.imgn
            if imgn is None:
                raise PtychoRunnerException("load_data(): imgn is None. Did you call load_scan() before ?")
            if os.path.isfile(self.params['data']) is False:
                raise PtychoRunnerException("hdf5 file does not exist: %s" % (self.params['data']))
            h5file = h5py.File(self.params['data'], 'r')
            try:
                self.params['instrument'] = h5file[f'/{self.scan}.1/instrument/title'][()]
            except:
                self.params['instrument'] = "ID01"
            if self.params['nrj'] is None:
                self.params['nrj'] = \
                    (12.39842 * 1e-10) / h5file[f'/{self.scan}.1/instrument/monochromator/WaveLength'][()]
                # / 1.60218e-16

            det = self.params['detector']
            if det is None:
                # Identify detector
                smpx1x4 = f"/{self.scan}.1/measurement/mpx1x4"
                seiger2M = f"/{self.scan}.1/measurement/eiger2M"
                if smpx1x4 in h5file:
                    det = "mpx1x4"
                    if seiger2M in h5file:
                        raise PtychoRunnerException("You need to select either detector=mpx1x4 or eiger2M "
                                                    "- both are present")
                elif seiger2M in h5file:
                    det = "eiger2M"
                else:
                    raise PtychoRunnerException("You need to supply detector=... ; neither mpx1x4 or eiger2M was found")
                self.params['detector'] = det

            if self.params['detectordistance'] is None:
                if f"/{self.scan}.1/instrument/{det}/distance" not in h5file:
                    raise PtychoRunnerException("No detector distance in data file. "
                                                "Please supply detectordistance=xx in meters")
                self.params['detectordistance'] = \
                    h5file[f"/{self.scan}.1/instrument/{det}/distance"][()]
                self.print("Detector distance from nexus file: %6.3fm" % self.params['detectordistance'])

            if self.params['pixelsize'] is None:
                if f"/{self.scan}.1/instrument/{det}/distance" not in h5file:
                    raise PtychoRunnerException("No x_pixel_size in data file. "
                                                "Please supply pixelsize=xx in meters")
                self.params['pixelsize'] = \
                    h5file[f"/{self.scan}.1/instrument/{det}/x_pixel_size"][()]
                self.print("Pixel size from nexus file: %6.2fµm" % (self.params['pixelsize'] * 1e6))

            # Load all frames
            sys.stdout.write("Reading %d frames from BLISS hdf5 file (detector=%s): 0" % (len(imgn), det))
            sys.stdout.flush()
            vrange = np.arange(0, len(imgn), 1)
            t0 = timeit.default_timer()
            vimg = None
            for i in range(len(vrange)):
                i1 = vrange[i]
                if i1 == len(vrange) - 1:
                    i2 = len(imgn)
                else:
                    i2 = vrange[i + 1]
                # print(imgn[i1:i2].tolist())
                tmp = h5file[f"/{self.scan}.1/measurement/{det}"][imgn[i1:i2].tolist()]
                if vimg is None:
                    vimg = np.empty((len(imgn), tmp.shape[-2], tmp.shape[-1]), dtype=tmp.dtype)
                vimg[i1:i2] = tmp
                if i2 % 20 == 0:
                    sys.stdout.write('.%d' % i2)
                    sys.stdout.flush()
            print()
            d0 = vimg.sum(axis=0)
            dt = timeit.default_timer() - t0
            print('Time to read all frames: %4.1fs [%5.2f Mpixel/s]' % (dt, d0.size * len(vimg) / 1e6 / dt))

            if f"/{self.scan}.1/instrument/{det}/pixel_mask" in h5file:
                if self.params['loadmask'] is None:
                    mask = h5file[f"/{self.scan}.1/instrument/{det}/pixel_mask"][()]
                    if np.ndim(mask) >= 2:
                        self.raw_mask = (mask != 0).astype(np.int8)
                        print(f"Loaded mask from bliss nexus data: /{self.scan}.1/instrument/{det}/pixel_mask")
                else:
                    print('Ignoring NeXus pixel_mask since mask was supplied from the command-line')

            self.raw_data = vimg
            self.load_data_post_process()

    def load_data_post_process(self):
        if self.params['crazy_eiger']:
            # Some module lines may behave incorrectly
            # Let's demonstrate how ugly a kludge we can apply...
            if self.raw_data.shape[-1] != 1030:
                raise PtychoRunnerException("'crazy_eiger' option has been given but the frame horizontal "
                                            "size is not 1030 pixels - aborting")
            print('Looking for crazy half-lines in cursed ID01 Eiger detector')
            if self.raw_mask is not None:
                m = self.raw_mask == 0
            else:
                m = 1
            for i in range(len(self.raw_data)):
                # Find crazy lines frame by frame
                for i0 in [0, 515]:
                    frame = (self.raw_data[i] * m)[:, i0:i0 + 515]
                    # Find where the difference between close lines is suspiciously close to 2**16-1
                    # (or a multiple of 2**16-1...)
                    for ii in range(1, len(frame)):
                        while ((frame[ii] - frame[ii - 1]) >= 60000).sum() > 300:
                            print("Crazy Eiger ! Correcting frame #%d @[%d, %d:%d]" % (i, ii, i0, i0 + 515))
                            self.raw_data[i, ii, i0:i0 + 515] -= 65535
                            self.raw_data[i, ii, i0:i0 + 2] += 32768
                            self.raw_data[i, ii, i0 + 255:i0 + 259] += 32768
                            self.raw_data[i, ii, i0 + 513:i0 + 515] += 32768
                            frame = (self.raw_data[i] * m)[:, i0:i0 + 515]
                    # In case the very top lines are affected
                    for ii in range(20, -1, -1):
                        while ((frame[ii] - frame[ii + 1]) >= 60000).sum() > 300:
                            print("Crazy Eiger ! Correcting frame #%d @[%d, %d:%d]" % (i, ii, i0, i0 + 515))
                            self.raw_data[i, ii, i0:i0 + 515] -= 65535
                            self.raw_data[i, ii, i0] += 32768
                            self.raw_data[i, ii, i0 + 255:i0 + 259] += 32768
                            self.raw_data[i, ii, i0 + 513:i0 + 514] += 32768
                            frame = (self.raw_data[i] * m)[:, i0:i0 + 515]

        super().load_data_post_process()


class PtychoRunnerID01(PtychoRunner):
    """
    Class to process a series of scans with a series of algorithms, given from the command-line
    """

    def __init__(self, argv, params, *args, **kwargs):
        super().__init__(argv, default_params if params is None else params)
        self.PtychoRunnerScan = PtychoRunnerScanID01

    @classmethod
    def make_parser(cls, default_par, description=None, script_name="pynx-ptycho-id01", epilog=None):
        if epilog is None:
            epilog = helptext_epilog
        if description is None:
            description = "Script to perform a ptychography analysis on data from ID01@ESRF"
        p = default_par

        parser = super().make_parser(default_par, script_name, description, epilog)
        grp = parser.add_argument_group("ID01 parameters")
        grp.add_argument('--data', type=str, default=None,
                         help='path to the bliss file, e.g. /some/dir/to/data.h5 '
                              '[mandatory unless spec is used]')

        grp.add_argument('--detector', type=str, default=None,
                         help='name of the detector to use (mpx1x4 or eiger2M, '
                              'this is normally auto-detected if only one is present '
                              'in the data)')

        grp.add_argument('--ptycho_motors', '--ptychomotors', type=str, default=p['ptychomotors'],
                         required=True, nargs='+', dest='ptychomotors', action=ActionPtychoMotors,
                         help="name of the two motors used for ptychography, optionally followed "
                              "by a mathematical expression to be used to calculate the actual "
                              "motor positions (axis convention, angle..). Values will be read "
                              "from the bliss/spec files, and are assumed to be in microns.\n"
                              "Examples:\n\n"
                              "* ``--ptychomotors pix piz``\n"
                              "* ``--ptychomotors=pix,piz,-x,y`` [using '=...' is necessary so "
                              "  the '-x' is not interpreted as a separate command-line parameter\n"
                              "* ``--ptychomotors=pix,piz,-x,y`` \n"
                              "Note that if the ``--xy=-y,x`` command-line argument is used, "
                              "it is applied _after_ this, so using ``--ptychomotors=pix,piz,-x,y`` "
                              "is equivalent to ``--ptychomotors pix piz --xy=-x,y``")
        grp.add_argument('--xyrange', '--xy_range', type=float, default=None,
                         nargs=4,
                         help='range where the data points will be taken into '
                              'account - needs 4 values i.e. xmin xmax ymin ymax. '
                              'All scan positions outside this range are ignored.'
                              'This must be given in original coordinates, in meters')
        grp.add_argument('--monitor', type=str, default=p['monitor'],
                         help='optional name for the monitor counter. The frames will '
                              'be normalized by the ratio of the counter value divided '
                              'by the median value of the counter over the entire'
                              'scan (so as to remain close to Poisson statistics). '
                              'A monitor intensity lower than 10%% of the median value '
                              'will be interpreted as an image taken without beam'
                              'and will be skipped.')
        grp.add_argument('--crazy_eiger', '--crazyeiger', action='store_true', default=p['crazy_eiger'],
                         help='if this option is given, crazy pixel lines affecting '
                              'entire modules shifted by 2**16-1 (or 2**15-1 in small '
                              'gaps) will be detected and corrected.')

        grp = parser.add_argument_group('ID01 parameters (obsolete-SPEC)')
        grp.add_argument('--specfile', '--spec_file', type=str, default=p['specfile'],
                         help='spec scan file for pre-bliss datasets')
        grp.add_argument('--imgcounter', '--img_counter', type=str, default=p['imgcounter'],
                         help='name of the image counter for spec')
        grp.add_argument('--imgname', '--img_name', type=str, default=p['imgname'],
                         help='images location pattern, e.g:\n'
                              '    /dir/to/images/prefix%%05d.edf.gz\n'
                              'By default this is automatically extracted '
                              'from the ULIMA spec scan header')
        grp.add_argument('--kmapfile', '--kmap_file', type=str, default=p['kmapfile'],
                         help='if images are saved in a multiframe data file, e.g. '
                              'detector/kmap/kmap_00000.edf.gz. '
                              'This supersedes imgname=...')
        grp.add_argument('--live_scan', '--livescan', action='store', type=str,
                         const=True, default=False, dest='livescan', nargs='?',
                         help="use this option to trigger the analysis of all scans "
                              "of a given type (by default 'spiralscan') in the "
                              "supplied spec file. If '--scan NN' is also given,"
                              "the analysis will start at scan #NN."
                              "Once all scans are processed, the script will wait "
                              "for new ones to arrive.")

        return parser

    def process_scans(self):
        """
        Run all the analysis on the supplied scan list. This derived function handles the livescan option.

        :return: Nothing
        """
        if self.params['livescan'] is False:
            super(PtychoRunnerID01, self).process_scans()
        else:
            # Type of scan which will be searched for ptycho data
            scan_type = 'spiralscan'
            if self.params['livescan'] is not True:
                scan_type = self.params['livescan'].lower()
            iscan = -1
            if self.params['scan'] is not None:
                iscan = int(self.params['scan']) - 1
            while True:
                sf = SpecFile(self.params['specfile'])
                if len(sf.keys()) > iscan + 1:
                    for iscan in range(iscan + 1, len(sf.keys())):
                        s = sf[iscan]
                        h = s.scan_header_dict
                        if scan_type in h['S']:  # Match scan title ?
                            scan = sf.list()[iscan]
                            if iscan == len(sf.keys()) - 1:
                                # Last scan ? Wait 20s for new data
                                while True:
                                    nb = s.data.shape[1]
                                    print('#            LIVESCAN: scan #%d: %s (waiting 20s for new data)' %
                                          (sf.list()[iscan], h['S']))
                                    time.sleep(20)
                                    sf = SpecFile(self.params['specfile'])
                                    s = sf[iscan]
                                    if s.data.shape[1] == nb:
                                        break

                            nb = s.data.shape[1]
                            nb_min = 16
                            if nb < nb_min:
                                print('#' * 100)
                                print('#            LIVESCAN: skipping scan #%d (only %d<%d data points): %s' %
                                      (sf.list()[iscan], nb, nb_min, h['S']))
                                print('#' * 100)
                                continue

                            if self.params['data2cxi']:
                                filename = os.path.join(os.path.dirname(self.params['saveprefix'] % (scan, 0)),
                                                        'data.cxi')
                                if os.path.isfile(filename):
                                    print('#' * 100)
                                    print('#            LIVESCAN: data2cxi: skip scan #%d (no overwriting): %s' %
                                          (sf.list()[iscan], h['S']))
                                    print('#' * 100)
                                    continue

                            print('#' * 100)
                            print('#            LIVESCAN: reading scan #%d: %s' % (sf.list()[iscan], h['S']))
                            print('#' * 100)
                            try:
                                self.ws = self.PtychoRunnerScan(self.params, scan)
                                self.ws.prepare_processing_unit()
                                self.ws.load_data()
                                if self.params['data2cxi']:
                                    if self.params['data2cxi'] == 'crop':
                                        self.ws.center_crop_data()
                                        self.ws.ƒ(crop=True, verbose=True)
                                    else:
                                        self.ws.save_data_cxi(crop=False, verbose=True)
                                else:
                                    self.ws.center_crop_data()
                                    self.ws.prepare()
                                    self.ws.run()
                            except PtychoRunnerException as ex:
                                print('\n\n Caught exception for scan %d: %s    \n' % (scan, str(ex)))
                                sys.exit(1)
                print("#            LIVESCAN: waiting for more scans (type='%s')..." % scan_type)
                time.sleep(10)


def make_parser_sphinx():
    """Returns the argparse for sphinx documentation"""
    return PtychoRunnerID01.make_parser(default_params)
