#!/usr/bin/env python

# 2014-02-14 to ignore cases for the sky prefix and to remove _ before the sky prefix

import glob
import os
import sys
import re
import math
import pyfits
from pyraf import iraf

class readparam:
    
    def __init__(self):

        if os.access('sirius.param', os.R_OK):
            omode = 'sirius'
        elif os.access('sirpol.param', os.R_OK):
            omode = 'sirpol'
        else:
            print 'irsf.readparam : No param file found'
            sys.exit()

        self.omode = omode

        flag = 0
        f = open(omode+'.param')
        for line in f:
            varr = line.split()
            exec("self."+varr[0]+"='"+varr[1]+"'")
        f.close()

        if self.band == 'all':
            self.blist = ['j', 'h', 'k']
        elif self.band == 'j':
            self.blist = ['j']
        elif self.band == 'h':
            self.blist = ['h']
        elif self.band == 'k':
            self.blist = ['k']
        else:
            print '%s : invalid value for band' % self.band
            flag = 1

        if len(self.yymmdd) != 6 or not varr[1].isdigit():
            print '%s : invalid value for yymmdd' % self.yymmdd
            flag = 1
        if self.twflat == '#':
            print '# : invalid value for twflat'
            flag = 1
        if self.stop != '0' and self.stop != '0.5' and self.stop != '1' and self.stop != '2':
            print '%s : invalid value for stop' % self.stop
            flag = 1
        #if self.slope != '0' and self.slope != '1':
        #    print '%s : invalid value for slope' % self.slope
        #    flag = 1
        if self.crremove != '0' and self.crremove != '1':
            print '%s : invalid value for crremove' % self.crremove
            flag = 1
        if self.linear != '0' and self.linear != '1':
            print '%s : invalid value for linear' % self.linear
            flag = 1
        if self.darkopt != '0' and self.darkopt != '1' and self.darkopt != '2': 
            print '%s : invalid value for darkopt' % self.darkopt
            flag = 1
        if self.comsky != '0' and self.comsky != '1': 
            print '%s : invalid value for comsky' % self.comsky
            flag = 1
        if self.allcom != '0' and self.allcom != '1': 
            print '%s : invalid value for allcom' % self.allcom
            flag = 1
        if self.cfitsio != '0' and self.cfitsio != '1': 
            print '%s : invalid value for cfitsio' % self.cfitsio
            flag = 1
        if self.keepd != '0' and self.keepd != '1': 
            print '%s : invalid value for keepd' % self.keepd
            flag = 1
        if self.keeps != '0' and self.keeps != '1': 
            print '%s : invalid value for keeps' % self.keeps
            flag = 1
        if self.keepm != '0' and self.keepm != '1': 
            print '%s : invalid value for keepm' % self.keepm
            flag = 1

        self.flag = flag


class readobslog():

    def __init__(self):

        self.fnum = []
        self.oname = []
        self.itime = []
        self.raoff = []
        self.decoff = []
        self.ra = []
        self.dec = []

        f = open('obslog')
        for line in f:
            v = line.split()
            if not re.match('flat', v[1], re.I) and not re.match('dark', v[1], re.I) and v[0].isdigit():
                if v[2][-1:] == '.':
                    v[2] = v[2][:-1]
                if re.search('sky$', v[1], re.I):      #  2014-02-14  to ignore cases for the sky prefix
                    v[1] = v[1][:-3] + 'sky'           
                if re.search('_sky$', v[1], re.I):     #  2014-02-14  to remove _ before the sky prefix
                    v[1] = v[1][:-4] + 'sky'           

                self.fnum.append(v[0])
                self.oname.append(v[1])
                self.itime.append(v[2])
                self.raoff.append(v[3])
                self.decoff.append(v[4])
                self.ra.append(v[11])
                self.dec.append(v[12])

        f.close()


def yymmdd():
    
    #    
    # read yymmdd
    #

    curdir = os.getcwd()
    darr = curdir.split('/')
    yymmdd = darr[len(darr)-2]

    if not yymmdd.isdigit():
        print 'irregular directory structure:'
        print 'the parent directory is '+yymmdd
        print 'which should be all digit'
        sys.exit()
    else:
        print 'yymmdd is '+yymmdd

    return yymmdd


def chkflat(band):

    SCRIPTDIR = os.path.realpath(os.path.dirname(sys.argv[0]))

    #
    # assumption 
    # FITS files in the working directory are 
    # user's prepared dark or flat frames. 
    #
    dlist = os.listdir('.')
    flist = []
    for v in dlist:
        if re.search('fits$', v) and not re.search('dark', v):
            flist.append(v)

    if len(flist) == 0:
        mflag = 1
    else:
        if band == 'all':
            if len(flist) == 3:
                common = flist[0][1:]
                if os.path.exists('j'+common) and os.path.exists('h'+common) and os.path.exists('k'+common):
                    twfname = common[:-5]
                    mflag = 0
                else:
                    mflag = 2
            else:
                mflag = 2
        else:
            if len(flist) == 1:
                if flist[0][:1] == band:
                    twfname = flist[0][1:-5]
                    mflag = 0
                else:
                    mflag = 2
            else:
                flag = 0
                common = flist[0][1:]
                for i in range(1, len(flist)):
                    if flist[i][1:] != common:
                        flag = 1
                if flag == 0 and os.path.exists(band+common):
                    twfname = common[:-5]
                    mflag = 0
                else:
                    mflag = 2
                
    if mflag == 0:
        print 'user-parepared flat frames have been detected.'
        print 'twfname is '+twfname
        if band == 'all':
            bloop = ['j', 'h', 'k']
        else:
            bloop = band

        for filt in bloop:
            hdulist = pyfits.open(filt+twfname+'.fits')
            if hdulist[0].header['bitpix'] == -32:
                pass
            elif hdulist[0].header['bitpix'] == -64:
                print 'flat FITS file : data type changed to 32bit float'
                iraf.chpixtype(filt+twfname+'.fits', filt+twfname+'.fits', 'real')
            else:
                print 'Illegal data type for the prepared flat FITS file : %s' % filt+twfname+'.fits'

            hdulist.close()

    elif mflag == 1:
        print 'the master flat frames are deployed'
        if band == 'all':
            os.system('ln -s %s/lib/jmflat.fits ./jmflat.fits' % SCRIPTDIR)
            os.system('ln -s %s/lib/hmflat.fits ./hmflat.fits' % SCRIPTDIR)
            os.system('ln -s %s/lib/kmflat.fits ./kmflat.fits' % SCRIPTDIR)
        else:
            os.system('ln -s %s/lib/%smflat.fits ./%smflat.fits' % (SCRIPTDIR, band, band))

        twfname = 'mflat'
    else:
        print 'I am confused.'
        print 'which is(are) the prepared flat file(s)?'
        sys.exit()

    return twfname


def mkdarklist(darkopt, yymmdd, blist):
    
    #
    # get exposure times for dark and other objects 
    #
    dtime = []
    dfnum = []
    otime = []
    f = open('obslog')
    for line in f:
        v = line.split()
        if re.search('dark', v[1], re.I):
            if v[2][-1:] == '.':
                v[2] = v[2][:-1]
            dtime.append(v[2])
            dfnum.append(v[0])
        elif v[0].isdigit():
            otime.append(v[2])
    f.close()

    if darkopt == '1':

        odtime = dtime
        uodtime = sorted(set(odtime), key = odtime.index)  #  this does not include dtime of prepared dark

        #
        # prepared dark 
        #
        flist = os.listdir('.')
        pdband = []
        pdtime = []
        for v in flist:
            if re.search('fits$', v) and re.search('dark', v):  # debugged 2013.4.10 match -> search
                hdulist = pyfits.open(v)
                hdulist.close()
                if hdulist[0].header['bitpix'] == -32:
                    pass
                else:
                    print 'dark FITS file %s : data type changed to 32bit float' % v
                    iraf.chpixtype(v, v, 'real')

                # band+'dark'+pdtime+'.fits'
                pdband.append(v[:1])
                pdtime.append(v[5:-5])

        if blist == ['j', 'h', 'k']:
            spdtime = list(set(pdtime))
            for pdt in spdtime:
                allband = ''
                for i in range(len(pdtime)):
                    if pdtime[i] == pdt:
                        allband = allband + pdband[i]
                if allband == 'jhk' or allband == 'jkh' or allband == 'hjk' or allband == 'hkj' or allband == 'kjh' or allband == 'khj':
                    dtime.append(pdt)
        else:
            for i in range(len(pdtime)):
                if pdband[i] == blist[0]:
                    dtime.append(pdtime[i]) 

    # if-darkopt ends

    #
    # unique list of otime and dtime
    #
    uotime = sorted(set(otime), key = otime.index)
    udtime = sorted(set(dtime), key = dtime.index)

    #
    # check if each otime has a corresponding dtime
    #
    for ot in uotime:
        flag = 0
        for dt in udtime:
            if float(ot) == float(dt):
                flag = 1
        if flag == 0:
            print 'No dark frame for %s sec' % ot
            return 1

    if darkopt == '1':     #    added  2013.4.19 
        return 0           #    No need to make dark lists when prepared dark drames exist. 

    #
    # make dark lists
    #

    dtimelist = udtime
    fol = open('darklist', 'w')
    for v in blist:
        for dt in dtimelist:
            listname = v + 'dark' + dt + '.list'
            print >> fol, listname
            fo = open(listname, 'w')
            for i in range(len(dtime)):
                if dtime[i] == dt:
                    if os.access('../rawdata/' + v + yymmdd + '_' + dfnum[i] + '.fits', os.R_OK):
                        print >> fo, '../rawdata/%s%s_%s.fits' % (v, yymmdd, dfnum[i])
                    elif os.access('../rawdata/' + v + yymmdd + '_' + dfnum[i] + '.fits.fz', os.R_OK):
                        print >> fo, '../rawdata/%s%s_%s.fits.fz[1]' % (v, yymmdd, dfnum[i])
                    elif os.access('../rawdata/' + v + yymmdd + '_' + dfnum[i] + '.fits.ic', os.R_OK):
                        print >> fo, '../rawdata/%s%s_%s.fits.ic[1]' % (v, yymmdd, dfnum[i])

            fo.close()
    fol.close()

    return 0


def mkdarkflat(twflat, yymmdd, band, obslog):

    uotime = sorted(set(obslog.itime), key = obslog.itime.index)

    fo = open(band + 'darkflatlist', 'w')
    for uo in uotime:
        print >> fo, '%s%sdflist %s %s' % (band, uo, uo, twflat) 
        fod = open(band + uo + 'dflist', 'w')
        for i in range(len(obslog.fnum)):
            if obslog.itime[i] == uo:
                if os.access('../rawdata/'+band+yymmdd+'_'+obslog.fnum[i]+'.fits', os.R_OK):
                    print >> fod, '../rawdata/%s%s_%s.fits ./ffiles/%sf%s.fits' % (band, yymmdd, obslog.fnum[i], band, obslog.fnum[i])
                elif os.access('../rawdata/'+band+yymmdd+'_'+obslog.fnum[i]+'.fits.fz', os.R_OK):
                    print >> fod, '../rawdata/%s%s_%s.fits.fz ./ffiles/%sf%s.fits' % (band, yymmdd, obslog.fnum[i], band, obslog.fnum[i])
                elif os.access('../rawdata/'+band+yymmdd+'_'+obslog.fnum[i]+'.fits.ic', os.R_OK):
                    print >> fod, '../rawdata/%s%s_%s.fits.ic ./ffiles/%sf%s.fits' % (band, yymmdd, obslog.fnum[i], band, obslog.fnum[i])
                else:
                    print 'No rawdata for %s-band %s' % (band, obslog.fnum[i])
                    print 'this frame wil be ignored in the following processes.'
        fod.close()
    fo.close()



def skygroup(setfnum, setoname, setitime):

    groupnum = -1
    ptime = 0
    pstd = 0
    pmap = 0
    pself = 0    
    pcsset = 0
    gnum = []
    sarr = []

    for i in range(len(setfnum)):

        suffix = 'none'

        if re.search('std$', setoname[i]):
            std = 1
            suffix = 'std'
        else:
            std = 0

        if re.search('map$', setoname[i]):
            map = 1
            suffix = 'map'
        else:
            map = 0

        if re.search('self$', setoname[i]):
            self = 1
            suffix = 'self'
        else:
            self = 0

        if re.search('cset$', setoname[i]) or re.search('sset$', setoname[i]):
            csset = 1
            if re.search('cset$', setoname[i]):
                suffix = 'cset'
            elif re.search('sset$', setoname[i]):
                suffix = 'sset'                
        else:
            csset = 0

        if re.search('sky$', setoname[i]):
            suffix = 'sky'

        if setitime[i] != ptime or std != pstd or map != pmap or self != pself or csset != pcsset:
            groupnum += 1
            
        ptime = setitime[i]
        pstd = std
        pmap = map
        pself = self
        pcsset = csset

        gnum.append(groupnum)
        sarr.append(suffix)

    return sarr, gnum, groupnum


def categorize(gfnum, gname, gtime, gsuff):

    eflag = 0

    global o2sky0, o2sky1, o2sky2
    global omap0, omap1, omap2, omap3
    global ocsset0, ocsset1, ocsset2, ocsset3

    omap0 = []
    omap1 = []
    omap2 = []
    omap3 = []

    ocsset0 = []
    ocsset1 = []
    ocsset2 = []
    ocsset3 = []

    mapflag = 0
    cssetflag = 0

    if gsuff[0] == 'std':
        for i in range(len(gfnum)):
            o2sky0.append(gfnum[i])
            o2sky1.append(gname[i])
            o2sky2.append(gfnum[i])
        
    elif gsuff[0] == 'self':
        for i in range(len(gfnum)):
            subname = gname[i][:-4]
            o2sky0.append(gfnum[i])
            o2sky1.append(subname)
            o2sky2.append(gfnum[i])
        
    elif gsuff[0] == 'map':
        for i in range(len(gfnum)):
            omap0.append(str(i))
            omap1.append(gfnum[i])
            omap2.append(gname[i])
            omap3.append(gtime[i])
        mapflag = 1
        
    elif gsuff[0] == 'cset' or gsuff[0] == 'sset':
        for i in range(len(gfnum)):
            ocsset0.append(str(i))
            ocsset1.append(gfnum[i])
            ocsset2.append(gname[i])
            ocsset3.append(gtime[i])
        cssetflag = 1

    else:   #  sky or object --- normal mode 
        for i in range(len(gfnum)):
            if gsuff[i] == 'sky':
                o2sky0.append(gfnum[i])
                o2sky1.append(gname[i])
                o2sky2.append(gfnum[i])

                # sky frames will be processed with self sky subtraction

            else:
                # if there is a dither set with a object name of 'object'+'sky' in the same 'group', 
                # it will be assigned to 'sky' of the object
                # if it is adjacent to the object set. 
                skyname = gname[i] + 'sky'
                flag = 0
                ssnum = []
                for j in range(len(gfnum)):
                    if gname[j] == skyname and abs(i - j)<=1:
                        flag += 1
                        ssnum.append(gfnum[j])

                if flag == 1:
                    o2sky0.append(gfnum[i])
                    o2sky1.append(gname[i])
                    o2sky2.append(ssnum[0])                        

                elif flag == 2:
                    o2sky0.append(gfnum[i])
                    o2sky1.append(gname[i])
                    o2sky2.append(ssnum[0]+':'+ssnum[1])

                elif flag == 0:
                    # In a case that there is no set with a object name of 'object'+'sky' in the same 'group',
                    # if an adjacent set has a suffix of 'sky', 
                    # the set will be assinged to 'sky' of the object.
                    if gsuff[i-1] == 'sky':
                        o2sky0.append(gfnum[i])
                        o2sky1.append(gname[i])
                        o2sky2.append(gfnum[i-1])
                    elif i< len(gfnum)-1 and gsuff[i+1] == 'sky':
                        o2sky0.append(gfnum[i])
                        o2sky1.append(gname[i])
                        o2sky2.append(gfnum[i+1])
                    else:
                        print 'No sky for %s' % gname[i]
                        eflag = 1


    return mapflag, cssetflag, eflag


def maplist(mapnum):

    global o2sky0, o2sky1, o2sky2
    global omap0, omap1, omap2, omap3

    mnum = len(omap0)
    if mnum == 1:
        # there is only one map set in this group  
        # self mode is adopted instead of map 
        mapname = omap2[0][:-3]
        o2sky0.append(omap1[0])
        o2sky1.append(mapname)
        o2sky2.append(omap1[0])
        print 'self mode is applied for %s~ %s' % (omap1[0], omap2[0])

        return 

    jump = []
    prev = 0
    for i in range(mnum):
        if i != 0 and int(omap0[i]) != prev + 1:
            jump.append(str(i))

        prev = int(omap0[i])

    jnum = len(jump)

    if jnum == 0:
        for i in range(mnum):
            mapname = omap2[i][:-3]
            if i == 0:
                o2sky0.append(omap1[i])
                o2sky1.append(mapname)
                o2sky2.append(omap1[i]+':'+omap1[i+1])
            elif i == mnum -1:
                o2sky0.append(omap1[i])
                o2sky1.append(mapname)
                o2sky2.append(omap1[i-1]+':'+omap1[i])
            else:
                o2sky0.append(omap1[i])
                o2sky1.append(mapname)
                o2sky2.append(omap1[i-1]+':'+omap1[i]+':'+omap1[i+1])
    else:
        jump.insert(0,'0')
        jump.append(mnum)
        for i in range(0, jnum+1):
            snum = int(jump[i])
            enum = int(jump[i+1])
            for j in range(snum, enum):
                mapname = omap2[j][:-3]
                if j == snum:
                    o2sky0.append(omap1[j])
                    o2sky1.append(mapname)
                    o2sky2.append(omap1[j]+':'+omap1[j+1])
                elif j == enum -1:
                    o2sky0.append(omap1[j])
                    o2sky1.append(mapname)
                    o2sky2.append(omap1[j-1]+':'+omap1[j])
                else:
                    o2sky0.append(omap1[j])
                    o2sky1.append(mapname)
                    o2sky2.append(omap1[j-1]+':'+omap1[j]+':'+omap1[j+1])


def cssetlist(cssetnum):

    global o2sky0, o2sky1, o2sky2
    global ocsset0, ocsset1, ocsset2, ocsset3

    csnum = len(ocsset0)
    suborder = 0
    suboname = ['','','','']
    subfnum = ['','','','']

    for i in range(0, csnum+1):

        if suborder == 0:
            suboname[suborder] = ocsset2[i]
            subfnum[suborder] = ocsset1[i]
            nameorg1 = ocsset2[i][:-5] # object name
            stypeorg1 = ocsset2[i][-4:] # cset or sset 
            suborder += 1

        elif suborder ==1:
            suboname[suborder] = ocsset2[i]
            subfnum[suborder] = ocsset1[i]
            nameorg2 = ocsset2[i][:-5]
            stypeorg2 =ocsset2[i][-4:]
            suborder += 1

        else:
            nametmp = ocsset2[i][:-5]
            stypetmp =ocsset2[i][-4:]
            
            if nametmp == nameorg1 and stypetmp == stypeorg1:
                suboname[suborder] = ocsset2[i]
                subfnum[suborder] = ocsset1[i]
                suborder = suborder + 1
            elif nametmp == nameorg2 and stypetmp == stypeorg2:
                suboname[suborder] = ocsset2[i]
                subfnum[suborder] = ocsset1[i]
                
                if stypeorg1 == 'cset' and stypeorg2 == 'sset':
                    o2sky0.append(subfnum[0])
                    o2sky1.append(suboname[0][:-4])
                    o2sky2.append(subfname[1])

                    o2sky0.append(subfnum[1])
                    o2sky1.append(suboname[1][:-4])
                    o2sky2.append(subfname[1])

                    sfnum3 = str(int(subfnum[1]) + 5).zfill(4)

                    o2sky0.append(subfnum[2])
                    o2sky1.append(suboname[2][:-4])
                    o2sky2.append(sfnum3 + ':' + subfname[3])

                    o2sky0.append(subfnum[3])
                    o2sky1.append(suboname[3][:-4])
                    o2sky2.append(sfnum3 + ':' + subfname[3])

                else:
                    print 'Not a regular set'
                    print ocsset2

                suborder = 0
                suboname = ['','','','']
                subfnum = ['','','','']

            else:
                if stypeorg1 == 'cset' and stypeorg2 =='sset':
                    o2sky0.append(subfnum[0])
                    o2sky1.append(suboname[0][:-4])
                    o2sky2.append(subfname[1])

                    o2sky0.append(subfnum[1])
                    o2sky1.append(suboname[1][:-4])
                    o2sky2.append(subfname[1])

                elif stypeorg1 =='sset' and stypeorg2 =='cset':
                    o2sky0.append(subfnum[0])
                    o2sky1.append(suboname[0][:-4])
                    o2sky2.append(subfname[0])

                    o2sky0.append(subfnum[1])
                    o2sky1.append(suboname[1][:-4])
                    o2sky2.append(subfname[0])

                suborder = 0
                suboname = ['','','','']
                subfnum = ['','','','']


def mkobj2skylist(fnum, oname, itime):

    class END(Exception):
        pass

    global o2sky0, o2sky1, o2sky2

    f = open('objectskylist', 'w')

    for i in range(len(o2sky0)):

        skysnum = o2sky2[i].split(':')
        tmponum = []

        for j in range(len(fnum)):
            if fnum[j] == o2sky0[i]:
                tmponum.append(fnum[j])
                for k in range(j+1, len(fnum)):
                    if oname[k] == oname[j] and itime[k] == itime[j]:
                        tmponum.append(fnum[k])
                    else:
                        break
                break
            
        oendnum = tmponum.pop()

        nrange=''
        for ii in range(len(skysnum)):
            tmpsnum = []
            try:
                for j in range(len(fnum)):
                    if fnum[j] == skysnum[ii]:
                        tmpsnum.append(fnum[j])
                        for k in range(j+1, len(fnum)):
                            if oname[k] == oname[j] and itime[k] == itime[j]:
                                tmpsnum.append(fnum[k])
                            else:
                                break
                        raise END()
            except END:
                pass
            
            sendnum = tmpsnum.pop()

            if ii == 0:
                nrange = skysnum[ii]+'-'+sendnum
            else:
                nrange = nrange+':'+skysnum[ii]+'-'+sendnum
    
        #if o2sky1[i][-3:] != 'sky': when not to process sky frames
        # print to objectskylist
        print >> f, '%s-%s %s %s sky%d' % (o2sky0[i], oendnum, o2sky1[i], nrange, i)

    f.close()


def modobj2skylist():

    skydict = {}
    f = open('objectskylist')
    for line in f:
        v = line[:-1].split()
        if not v[2] in skydict:
            skydict[v[2]] = v[3]
    f.close()

    fos = open('objectskylist.new', 'w')
    f = open('objectskylist')
    for line in f:
        v = line[:-1].split()
        print >> fos, v[0], v[1], v[2], skydict[v[2]]
    f.close()
    fos.close()

    os.rename('objectskylist.new', 'objectskylist')


def mkskylist(fnum, oname):

    osize = len(fnum)

    # skyllt stands for skylistlist.tmp in the sirius09 pipeline
    skyllt = []
    skylltu = []
    # skylt stands for skylist.tmp in the sirius09 pipeline
    skylt1 = []
    skylt2 = []
    skyltu = []

    fo = open('obj2sky', 'w')
    f = open('objectskylist')
    for line in f:
        values = line[:-1].split()
        objse = values[0].split('-')
        for i in range(osize):
            if fnum[i] == objse[0]:
                initoname = oname[i]
                print >> fo, '%s %s %s' % (fnum[i], oname[i], values[3])
            elif int(fnum[i]) > int(objse[0]) and int(fnum[i]) <= int(objse[1]):
                if initoname == oname[i]:
                    print >> fo, '%s %s %s' % (fnum[i], oname[i], values[3])
            elif int(fnum[i]) > int(objse[1]):
                break

        skyllt.append(values[3])
        skylltu.append('0')

        skylt1.append(values[2])
        skylt2.append(values[3])
        skyltu.append('0')

    f.close()
    fo.close()

    # sort -u skylistlist.tmp > skylistlist
    for i in range(len(skyllt)):
        for j in range(i+1, len(skyllt)):
            if skyllt[i] == skyllt[j]:
                skylltu[j] = '1'

    skyll = []
    for i in range(len(skyllt)):
        if skylltu[i] == '0':
            skyll.append(skyllt[i])

    # sort -u skylist.tmp > skylist.tmp2
    for i in range(len(skylt1)):
        for j in range(i+1, len(skylt1)):
            if skylt1[i] == skylt1[j] and skylt2[i] == skylt2[j]:
                skyltu[j] = '1'
            
    skylt21 = []
    skylt22 = []
    for i in range(len(skylt1)):
        if skyltu[i] == '0':
            skylt21.append(skylt1[i])
            skylt22.append(skylt2[i])

    for i in range(len(skyll)):
        fo = open(skyll[i]+'.skylist', 'w')        
        for j in range(len(skylt21)):
            if skylt22[j] == skyll[i]:
                sncomp = skylt21[j].split(':')
                snnum = len(sncomp)
                for ii in range(snnum):
                    skyse = sncomp[ii].split('-')
                    for jj in range(osize):
                        if fnum[jj] == skyse[0]:
                            initskyname = oname[jj]
                            print >> fo, fnum[jj]
                        elif int(fnum[jj]) > int(skyse[0]) and int(fnum[jj]) <= int(skyse[1]):
                            if initskyname == oname[jj]:
                                print >> fo, fnum[jj]
        fo.close()


def object2sky(obslog):

    global o2sky0, o2sky1, o2sky2
    global omap0, omap1, omap2, omap3
    global ocsset0, ocsset1, ocsset2, ocsset3

    # setfnum, setoname, and setitime are frame number, object name, and integration time
    # for the first frame of each dither set
    # a sequential frames with the same object name and integration time is regarded as a dither set. 
    setfnum = []
    setoname = []
    setitime = []
    poname = ''
    pitime = '0'
    for i in range(len(obslog.fnum)):
        if obslog.oname[i] != poname or obslog.itime[i] != pitime:
            setfnum.append(obslog.fnum[i])
            setoname.append(obslog.oname[i])
            setitime.append(obslog.itime[i])
            poname = obslog.oname[i]
            pitime = obslog.itime[i]

    suffix, grpnum, gmax = skygroup(setfnum, setoname, setitime)

    #for j in range(len(setoname)):
    #    print j, setfnum[j], setoname[j], setitime[j], grpnum[j], suffix[j]

    o2sky0 = []
    o2sky1 = []
    o2sky2 = []

    for i in range(gmax+1):
        gfnum = []
        gname = []
        gtime = []
        gsuff = []
        for j in range(len(setoname)):
            if grpnum[j] == i:
                gfnum.append(setfnum[j])
                gname.append(setoname[j])
                gtime.append(setitime[j])
                gsuff.append(suffix[j])

        mapflag, cssetflag, eflag = categorize(gfnum, gname, gtime, gsuff)

        if eflag == 1:
            return 1

        if mapflag == 1:
            maplist(i)
            mapflag = 0
        elif cssetflag == 1:
            cssetlist(i)
            cssetflag = 0

    mkobj2skylist(obslog.fnum, obslog.oname, obslog.itime)
    # creates objectskylist

    modobj2skylist()
    # removes duplication in objectskylist

    mkskylist(obslog.fnum, obslog.oname)
    
    return 0



def mkdithlist(allcom, obslog, comsky):

    dlarr1 = []
    dlarr2 = []
    dlarr3 = []
    f = open('objectskylist')
    for i, line in enumerate(f):
        v = line[:-1].split()
        listname = 'o'+str(i)+'.'+v[1]+'.list'
        sefnum = v[0].split('-')
        
        fo2 = open(listname, 'aw')
        for j in range(len(obslog.fnum)):
            if obslog.fnum[j] == sefnum[0]:
                dlarr1.append(i)
                dlarr2.append(v[1])
                dlarr3.append(obslog.itime[j])

            if int(obslog.fnum[j]) >= int(sefnum[0]) and int(obslog.fnum[j]) <= int(sefnum[1]):
                xoff = float(obslog.raoff[j]) 
                yoff = float(obslog.decoff[j])
                print >> fo2, '%s %.2f %.2f' % (obslog.fnum[j], xoff, yoff)
            elif obslog.fnum[j] > sefnum[1]:
                break

        fo2.close()

    f.close()

    fo1 = open('dithlistlist', 'w')
    if allcom == '0':
        if comsky == '0':
            for i in range(len(dlarr1)):
                if not dlarr2[i].endswith('sky'):
                    print >> fo1, 'o.%d%s.list o%d.%s %s' % (dlarr1[i], dlarr2[i], dlarr1[i], dlarr2[i], dlarr3[i])
        else:
            for i in range(len(dlarr1)):
                print >> fo1, 'o.%d%s.list o%d.%s %s' % (dlarr1[i], dlarr2[i], dlarr1[i], dlarr2[i], dlarr3[i])

    else:
        dflag = []
        for i in range(len(dlarr1)):
            dflag.append(0)

        for i in range(len(dlarr1)):
            if dlarr2[i].endswith('sky'):
                continue
            for j in range(i+1, len(dlarr1)):
                if dlarr2[i] == dlarr2[j] and dlarr3[i] == dlarr3[j]:
                    dflag[j] = 1

        for i in range(len(dlarr1)):
            if dflag[i] == 1:
                os.remove('o'+str(i)+'.'+dlarr2[i]+'.list')
                continue
            else:
                #char = dlarr2[i]+str(i)
                if comsky == '0':
                    if not dlarr2[i].endswith('sky'):
                        print >> fo1, 'o%d.%s.list o%d.%s %s' % (dlarr1[i], dlarr2[i], dlarr1[i], dlarr2[i], dlarr3[i])
                else:
                    print >> fo1, 'o%d.%s.list o%d.%s %s' % (dlarr1[i], dlarr2[i], dlarr1[i], dlarr2[i], dlarr3[i])

                listname = 'o'+str(i)+'.'+dlarr2[i]+'.list'

            if dlarr2[i].endswith('sky'):
                continue
            for j in range(i+1, len(dlarr1)):
                if dlarr2[i] == dlarr2[j] and dlarr3[i] == dlarr3[j]:
                    fo = open(listname, 'aw')
                    fi = open('o'+str(j)+'.'+dlarr2[j]+'.list')
                    for line in fi:
                        print >> fo, line.strip()
                    fi.close()
                    fo.close()
    fo1.close()


def chkobslog():

    o = readobslog()

    # object name and target position

    dflag = []
    for i in range(len(o.fnum)):
        dflag.append(0)

    for i in range(len(o.fnum)):
        if dflag[i] == 1:
            continue
        for j in range(i+1, len(o.fnum)):
            if o.oname[i] == o.oname[j] and o.itime[i] == o.itime[j]:
                dflag[j] = 1

    # should extract self and  map
    for i in range(len(o.fnum)):
        if o.oname[i].endswith('sky'):
            continue
        if dflag[i] == 0:
            if o.oname[i].endswith('self'):
                oname1 = o.oname[i][:-4]
            elif o.oname[i].endswith('map'):
                oname1 = o.oname[i][:-3]
            else:
                oname1 = o.oname[i]
            x1 = o.ra[i]
            y1 = o.dec[i]
            for j in range(i+1, len(o.fnum)):
                if o.oname[j].endswith('self'):
                    oname2 = o.oname[j][:-4]
                elif o.oname[j].endswith('map'):
                    oname2 = o.oname[j][:-3]
                else:
                    oname2 = o.oname[j]

                if oname1 == oname2 and o.itime[i] == o.itime[j]:
                    x2 = o.ra[j]
                    y2 = o.dec[j]
                    angm = calcdist(x1, y1, x2, y2)
                    if angm > 5:
                        print 'warning : '
                        print 'the same object name %s for frames %s and %s' % (oname1, o.fnum[i], o.fnum[j])
                        print 'but the central coordinates are separated by more than 5 arcmin'
                        print ''


def calcdist(x1, y1, x2, y2):

    # ra = xx:xx:xx.x
    # dec = +/- xx:xx:xx.x

    if x1 == x2 and y1 == y2:
        return 0

    xarr1 = x1.split(':')
    yarr1 = y1.split(':')

    xarr2 = x2.split(':')
    yarr2 = y2.split(':')

    ra1 = math.radians(15 * (int(xarr1[0]) + float(xarr1[1]) / 60 + float(xarr1[2]) / 3600))
    if yarr1[0][:1] == '-':
        sign1 = -1
    else:
        sign1 = 1
    dec1 = math.radians(int(yarr1[0]) + sign1 * float(yarr1[1]) / 60 + sign1 * float(yarr1[2]) / 3600)

    ra2 = math.radians(15 * (int(xarr2[0]) + float(xarr2[1]) / 60 + float(xarr2[2]) / 3600))
    if yarr2[0][:1] == '-':
        sign2 = -1
    else:
        sign2 = 1
    dec2 = math.radians(int(yarr2[0]) + sign2 * float(yarr2[1]) / 60 + sign2 * float(yarr2[2]) / 3600)

    xvalue = math.cos(dec1) * math.cos(dec2) * (math.cos(ra1) * math.cos(ra2) + math.sin(ra1) * math.sin(ra2)) + math.sin(dec1) * math.sin(dec2)
    angm = 60 * math.degrees(math.acos(xvalue)) 

    return angm


def comdithlist():

    os.rename('dithlistlist', 'dithlistlist.org')

    fout = open('dithlistlist', 'w')
    fin = open('dithlistlist.org')
    
    i = 0
    for line in fin:
        if not re.search('\S', line[:-1]):
            continue

        v = line[:-1].split()
        if i == 0:
            if v[0].startswith('*'):
                print 'Illegal format of dithlistlist. '
                print 'Do not attach * to the first line.'
                sys.exit()
            else:
                i += 1

        if not v[0].startswith('*'):
            print >> fout, '%s %s %s' % (v[0], v[1], v[2])
            catintofile = v[0]
        else:
            fname = v[0][1:]
            fadd = open(catintofile, 'aw')
            fcat = open(fname)
            for line in fcat:
                print >> fadd, line[:-1]
            os.remove(fname)
            fcat.close()
            fadd.close()

    fin.close()
    fout.close()


def revskylist():

    if os.path.exists('objectskylist'):
        pass
    else:
        print 'There is no objectskylist here.'
        sys.exit()

    for file in glob.glob('*.skylist'):
        os.remove(file)

    f = open('dithlistlist')
    for line in f:
        v = line[:-1].split()
        os.remove(v[0])
    f.close()

    para = readparam()
    obslog = readobslog()

    modobj2skylist()
    mkskylist(obslog.fnum, obslog.oname)

    if para.omode == 'sirpol':
        numpoldict = numpol(para.yymmdd, para.blist, para.polmode, obslog)
        polsky(para.polmode, numpoldict)

    mkdithlist(para.allcom, obslog, para.comsky)



def zerosky(band, skyname):

    iraf.imstat.fields = 'midpt'
    iraf.imstat.nclip = 10
    iraf.imstat.cache = 'yes'

    fo1 = open('comlist', 'w')
    fo2 = open('zerolist', 'w')
    fo3 = open(band + 'skybg.list', 'aw')
    fin = open(skyname + '.skylist')
    i = 0
    for line in fin:
        frame = './ffiles/' + band + 'f' + line[:-1]
        if i == 0:
            frame0 = frame
            iraf.imstat.lower = -5000
            iraf.imstat.upper = 20000
            imout = iraf.imstat(frame, Stdout=1)
            med0 = float(imout[1])
            print >> fo1, frame
            print >> fo2, 0
            print >> fo3, '%s %.1f' % (frame, med0)
            i += 1
        else:
            iraf.imstat.lower = -5000
            iraf.imstat.upper = 20000
            imout = iraf.imstat(frame, Stdout=1)
            med = float(imout[1])
            iraf.imstat.lower = med0 - med - 1000
            iraf.imstat.upper = med0 - med + 1000
            iraf.imarith(frame0, '-', frame, 'dframe')
            imout = iraf.imstat('dframe', Stdout=1)
            if re.search('INDEF', imout[1]):
                print 'imstat failed for %s' % frame
                print 'imout[1] was %s' % imout[1]
                print '*** please send the files %s.fits and %s.fits to y.nakajima ***' % (frame0, frame)
                iraf.imdel('dframe')     #    added 2013.4.30
                continue
            med = float(imout[1])
            print >> fo1, frame
            print >> fo2, med
            print >> fo3, '%s %.1f' % (frame, med0 - med)
            iraf.imdel('dframe')
    fin.close()
    fo1.close()
    fo2.close()
    fo3.close()

    iraf.imcombine.unlearn()
    iraf.imcombine.combine = 'median'
    iraf.imcombine.reject = 'sigclip'
    iraf.imcombine.zero = '@zerolist'
    iraf.imcombine.mclip = 'yes'
    iraf.imcombine('@comlist', band+skyname, Stdout=1)
    os.remove('comlist')
    os.remove('zerolist')

    iraf.imstat.lower = 'INDEF'
    iraf.imstat.upper = 'INDEF'
    imout = iraf.imstat(band+skyname, Stdout=1)
    med = imout[1]

    iraf.imarith(band+skyname, '/', med, 'n'+band+skyname)


def skybglist(band):

    fin = open(band+'skybg.list')
    namearr = []
    bgarr = []
    for line in fin:
        v = line[:-1].split()
        dflag = 0
        for fid in namearr:
            if fid == v[0]:
                dflag = 1
                break
        if dflag == 0:
            namearr.append(v[0])
            bgarr.append(v[1])
    fin.close()

    fo = open(band+'skybg.list', 'w')
    for i in range(len(namearr)):
        print >> fo, namearr[i], bgarr[i]
    fo.close()


def getstat(infits):

    iraf.imstat.unlearn()
    iraf.imstat.fields = 'midpt, stddev'
    iraf.imstat.lower = -1000
    iraf.imstat.upper = 15000
    iraf.imstat.nclip = 5
    iraf.imstat.lsigma = 3
    iraf.imstat.usigma = 3
    iraf.imstat.binwidth = 0.1
    iraf.imstat.format = 'yes'
    iraf.imstat.cache = 'yes'
                                        
    out_imstat = iraf.imstat(infits, Stdout=1)
    med, stdev = out_imstat[1].split()
        
    return med, stdev


def numpol(yymmdd, blist, polmode, obslog):

    numpoldict = {}
    for num in obslog.fnum:
        ffile = ''
        for band in blist:
            if os.access('../rawdata/' + band + yymmdd + '_' + num + '.fits.fz', os.R_OK):
                ffile = '../rawdata/' + band + yymmdd + '_' + num + '.fits.fz'
                hdunum = 1
                break
            elif os.access('../rawdata/' + band + yymmdd + '_' + num + '.fits.ic', os.R_OK):
                ffile = '../rawdata/' + band + yymmdd + '_' + num + '.fits.ic'
                hdunum = 1
                break
            elif os.access('../rawdata/' + band + yymmdd + '_' + num + '.fits', os.R_OK):
                ffile = '../rawdata/' + band + yymmdd + '_' + num + '.fits'
                hdunum = 0
                break

        if ffile == '':
            continue 
        else:
            hdu = pyfits.open(ffile)
            if polmode == '1':
                pang = hdu[hdunum].header['POL-AGL1']
            else:
                if not hdu[hdunum].header['POL-AGL1'] == 0.0:
                    print 'numpol : POL-AGL1 is not 0.0 for %s' % ffile
                pang = hdu[hdunum].header['POL-AGL2']
            hdu.close()

        numpoldict[num] = pang
    
    return numpoldict



def polsky(polmode, numpoldict):

    skyarr = []
    fo = open('obj2sky.new', 'w')
    f = open('obj2sky')
    for line in f:
        v = line[:-1].split()
        pang = numpoldict[v[0]]

        if polmode == '1':
            if pang == 0.0:
                cpolagl = 'a00'
            elif pang == 22.5:
                cpolagl = 'a22'
            elif pang == 45.0:
                cpolagl = 'a45'
            elif pang == 67.5:
                cpolagl = 'a67'
            else:
                print 'polsky : bad POL-AGL1 in the header of %s' % ffile
        else:
            if pang == 0.0:
                cpolagl = 'a000'
            elif pang == 90.0:
                cpolagl = 'a090'
            elif pang == 45.0:
                cpolagl = 'a045'
            elif pang == 135.0:
                cpolagl = 'a135'
            else:
                print 'polsky : bad POL-AGL2 in the header of %s' % ffile

        print >> fo, v[0], v[1], v[2]+cpolagl

        if not v[2] in skyarr:
            skyarr.append(v[2])

    f.close()
    fo.close()

    os.rename('obj2sky.new', 'obj2sky')

    if polmode == '1':

        for skyn in skyarr:
            fo1 = open(skyn+'a00.skylist', 'w')
            fo2 = open(skyn+'a45.skylist', 'w')
            fo3 = open(skyn+'a22.skylist', 'w')
            fo4 = open(skyn+'a67.skylist', 'w')
            fin = open(skyn+'.skylist')
            for line in fin:
                fnum = line[:-1]
                if numpoldict[fnum] == 0.0:
                    print >> fo1, fnum
                elif numpoldict[fnum] == 45.0:
                    print >> fo2, fnum
                elif numpoldict[fnum] == 22.5:
                    print >> fo3, fnum
                elif numpoldict[fnum] == 67.5:
                    print >> fo4, fnum
            fin.close()
            fo1.close()
            fo2.close()
            fo3.close()
            fo4.close()

    elif polmode == '2':

        for skyn in skyarr:
            fo1 = open(skyn+'a000.skylist', 'w')
            fo2 = open(skyn+'a090.skylist', 'w')
            fin = open(skyn+'.skylist')
            for line in fin:
                fnum = line[:-1]
                if numpoldict[fnum] == 0.0:
                    print >> fo1, fnum
                elif numpoldict[fnum] == 90.0:
                    print >> fo2, fnum
            fin.close()
            fo1.close()
            fo2.close()

    else:

        for skyn in skyarr:
            fo1 = open(skyn+'a000.skylist', 'w')
            fo2 = open(skyn+'a090.skylist', 'w')
            fo3 = open(skyn+'a045.skylist', 'w')
            fo4 = open(skyn+'a135.skylist', 'w')
            fin = open(skyn+'.skylist')
            for line in fin:
                fnum = line[:-1]
                if numpoldict[fnum] == 0.0:
                    print >> fo1, fnum
                elif numpoldict[fnum] == 90.0:
                    print >> fo2, fnum
                elif numpoldict[fnum] == 45.0:
                    print >> fo3, fnum
                elif numpoldict[fnum] == 135.0:
                    print >> fo4, fnum
            fin.close()
            fo1.close()
            fo2.close()
            fo3.close()
            fo4.close()


def mkqui(numpoldict, blist):

    SCRIPTDIR = os.path.realpath(os.path.dirname(sys.argv[0]))

    fo1 = open('mkqu.sh', 'w')
    fo2 = open('mki.sh', 'w')
    fin = open('dithlistlist')
    for line in fin:
        v = line[:-1].split()
        qflag = 0
        uflag = 0
        fin2 = open(v[0])
        for line2 in fin2:
            v2 = line2[:-1].split()
            if numpoldict[v2[0]] == 0.0:
                q1 = v2[0]
            elif numpoldict[v2[0]] == 45.0:
                q2 = v2[0]
                if int(q2) - int(q1) == 1:
                    for band in blist:
                        print >> fo1, '%s/spmkqux %sf%s.fits %sf%s.fits %sq%s.fits' % (SCRIPTDIR, band, q1, band, q2, band, q1)
                    qflag = 1
            elif numpoldict[v2[0]] == 22.5:
                u1 = v2[0]
            elif numpoldict[v2[0]] == 67.5:
                u2 = v2[0]
                if int(u2) - int(u1) == 1:
                    for band in blist:
                        print >> fo1, '%s/spmkqux %sf%s.fits %sf%s.fits %su%s.fits' % (SCRIPTDIR, band, u1, band, u2, band, u1)
                    uflag = 1

            if qflag == 1 and uflag == 1:
                for band in blist:
                    print >> fo2, '%s/spmkix %sf%s.fits %sf%s.fits %sf%s.fits %sf%s.fits %si%s.fits' % (SCRIPTDIR, band, q1, band, q2, band, u1, band, u2, band, q1)
                qflag = 0
                uflag = 0
            elif uflag == 1:
                uflag = 0

        fin2.close()
    fin.close()
    fo1.close()
    fo2.close()



def mkvi(numpoldict, blist):

    SCRIPTDIR = os.path.realpath(os.path.dirname(sys.argv[0]))

    fo1 = open('mkv.sh', 'w')
    fo2 = open('mki.sh', 'w')
    fin = open('dithlistlist')
    for line in fin:
        v = line[:-1].split()
        qflag = 0
        fin2 = open(v[0])
        for line2 in fin2:
            v2 = line2[:-1].split()
            if numpoldict[v2[0]] == 0.0:
                q1 = v2[0]
            elif numpoldict[v2[0]] == 90.0:
                q2 = v2[0]
                if int(q2) - int(q1) == 1:
                    for band in blist:
                        print >> fo1, '%s/spmkqux %sf%s.fits %sf%s.fits %sv%s.fits' % (SCRIPTDIR, band, q1, band, q2, band, q1)
                    qflag = 1

            if qflag == 1:
                for band in blist:
                    print >> fo2, '%s/addfitsf %sf%s.fits %sf%s.fits %si%s.fits' % (SCRIPTDIR, band, q1, band, q2, band, q1)
                qflag = 0

        fin2.close()
    fin.close()
    fo1.close()
    fo2.close()



def mki(numpoldict, blist):

    SCRIPTDIR = os.path.realpath(os.path.dirname(sys.argv[0]))

    fo1 = open('mki.sh', 'w')
    fin = open('dithlistlist')
    for line in fin:
        v = line[:-1].split()
        qflag = 0
        uflag = 0
        fin2 = open(v[0])
        for line2 in fin2:
            v2 = line2[:-1].split()
            if numpoldict[v2[0]] == 0.0:
                q1 = v2[0]
            elif numpoldict[v2[0]] == 90.0:
                q2 = v2[0]
                if int(q2) - int(q1) == 1:
                    qflag = 1
            elif numpoldict[v2[0]] == 45.0:
                u1 = v2[0]
            elif numpoldict[v2[0]] == 135.0:
                u2 = v2[0]
                if int(u2) - int(u1) == 1:
                    uflag = 1

            if qflag == 1 and uflag ==1:
                for band in blist:
                    print >> fo1, '%s/spmkix %sf%s.fits %sf%s.fits %sf%s.fits %sf%s.fits %si%s.fits' % (SCRIPTDIR, band, q1, band, q2, band, u1, band, u2, band, q1)
                qflag = 0
                uflag = 0
            elif uflag == 1:
                uflag = 0

        fin2.close()
    fin.close()
    fo1.close()



def clean(keepd, keeps, keepm):
    
    print 'cleaning the working directory... '

    if keepd == 1 or keeps ==1 or keepm ==1:
        os.mkdir('calib')

    if keepd ==1:
        for file in glob.glob('*dark*.fits'):
            shutil.move(file, 'calib')
    else:
        for file in glob.glob('*dark*.fits'):
            os.remove(file)

    if keeps ==1:
        for file in glob.glob('*sky*.fits'):
            shutil.move(file, 'calib')
    else:
        for file in glob.glob('*sky*.fits'):
            os.remove(file)
            
    if keepm ==1:
        for file in glob.glob('*mask.fits'):
            shutil.move(file, 'calib')
    else:
        for file in glob.glob('*mask*.fits'):
            os.remove(file)
            
    for file in glob.glob('*.skylist'):
        os.remove(file)

    for file in glob.glob('*sky.list'):
        os.remove(file)


    os.remove('0.ffiles')




if __name__ == "__main__":

    print 'irsf.py is a module repository.'




