pro reduce_main,src,auto=auto
;+
; NAME:				REDUCE_MAIN
;
; PURPOSE:			Main program to search for new files created by
;				DDIS to perform pipeline processing on.
;
; CATEGORY:			REDUCTION
;
; CALLING SEQUENCE:		REDUCE_MAIN, Src
;
; INPUTS:			None
;
; OPTIONAL INPUTS:		Src = 0 for processing R/T files at GSFC
; 				      1 for processing R/T files at NRL
; 				      2 for processing PB files at NRL
;	
; KEYWORD PARAMETERS:		Auto = if set then use closed_img_file
;                                      if not set then use closed_img_file2
;
; OUTPUTS:			None
;
; OPTIONAL OUTPUTS:		None
;
; COMMON BLOCKS:		DBMS
;
; SIDE EFFECTS:
;
; RESTRICTIONS:
;
; PROCEDURE:
;
; EXAMPLE:
;
; MODIFICATION HISTORY:		Written  RA Howard, NRL
;    Version 1   RAH, 6 Nov 1995,    Initial Release
;    Version 2   rah, 15 Nov 1995,   Added closed_img_file flag file
;                                    Modified log printing
;    Version 3   rah, 18 Nov 1995,   Added source as optional parameter
;    Version 4   rah,  7 Dec 1995,   Minor corrections to version
;    Version 5   rah,  4 Jan 1996,   Correction to handling of .mem files
;    Version 6   rah, 15 Jan 1996,   Minor correction to file name and get_lun,luf
;    Version 7   rah, 18 Jan 1996,   Corrected handling of contents of closed_img_file
;				     to correct FP files 
;    Version 8   rah, 09 Apr 1996,   Added check of letter extension to filename
;    Version 9   rah, 19 Apr 1996,   More mods to account for letter extension
;    Version 10  rah, 27 Apr 1996,   Added error handling opening closed_img_file
;    Version 11  rah, 08 May 1996,   Added creation of file for update list
;    Version 12  nbr, 08 Apr 1997,   Changed target directories for .log and .db files
;    Version 13  rah, 26 Aug 1997,   Error handling opening closed_img_file
;    Version 14  rah, 27 Sep 1997,   Handling of files created out of time order
;    Version 15  rah, 16 Jan 1998,   Process mem files first before img files
;    Version 16  nbr, 03 Jun 1998,   Remove old .img dirs from previous session
;    Version 17  rah, 19 Oct 1998,   Remove old log files for GSFC processing
;    Version 18  nbr,  4 Jan 1999,   Add row to .db file to remove QL LEB_HDR
;    Version 19  nbr, 20 Oct 1999,   Send message to user if gap between current and last 
;					.img files is > 1 hour; add check for good date
;    Version 20  nbr,  4 Apr 2000,   Halt processing of QL for db update at 2AM; move save lastdate.sav
;    Version 21  nbr, 24 Jul 2000,   Add /SH to spawn commands
;    Version 22 nbr , 17 Oct 2001,   Change filename sent in email for db update halt
;
;
;
; @(#)reduce_main.pro	1.25 10/17/01 :LASCO IDL LIBRARY
;
;
;-
;
common dbms,ludb,lulog
ver = 'V17  19  Oct1998'
true=1
leb_img = getenv ('LEB_IMG')
earliest_tai=utc2tai(str2utc('1999/01/01'))
;
;  main loop to find and process an image file
;
fnclosed='closed_img_file2'
if keyword_set (auto) then fnclosed='closed_img_file'

rawdir = getenv('RAW')
;IF src EQ 1 OR src EQ 2 THEN BEGIN	;** 6/3/98, NBR
IF src EQ 9 THEN BEGIN	;** 10/16/98, NBR
	IF src EQ 1 THEN rawfile = rawdir+'/QL_RAW.contents' ELSE rawfile = rawdir+'/LZ_RAW.contents'
	spawn,'du -k '+rawdir+'/9* >> '+rawfile,/sh	;** 6/10/98, NBR
	spawn,'ll '+rawdir+'/9* >> '+rawfile,/sh
	print,'Removing '
	spawn,'ls -d '+rawdir+'/9*',/sh
	wait,5
	spawn,'/bin/rm -fr '+rawdir+'/9*',/sh
ENDIF

restore,leb_img+'/lastdate.sav'

repeat begin				; 1
   cd,leb_img
;
;  Find all mem or leb files in $LEB_IMG 
;  
   g = findfile('*.mem')		; check for mem files
   sg = size(g)
   files = findfile('*.img*')
   siz = size(files)
   sf  = siz
   if (sg(0) ne 0) then begin
      if (siz(0) eq 0)  then files=g else files=[g,files]
      siz=size(files)
   endif
   IF files(0) EQ '' THEN nfiles = 0 ELSE nfiles = siz(1)       ;** added 980423 SEP
   ifiles = 0
   ;PRINT,nfiles,ifiles,files(0)
   REPEAT BEGIN				; 2
;
;  Wait until 'closed_img_file' exists, open it, read the contents (file name)
;  'closed_img_file' is written by DDIS when it closes an img or mem file.
;  the contents are the file name that was just closed.  Since the file names
;  are constructed with the date and time of the file, we know when a file
;  has been closed by looking at the times and making sure that the img or mem
;  file is not dated after the time indicated by the name in 'closed_img_file'.
;
;  Also, if the last time through the loop, looking for a closed file name,
;  there were no files to be processed, then just loop until DDIS changes
;  the name in 'closed_img_file'
;
   lastfn = ''
   repeat begin				; 3
      cd,leb_img
      get_lun,luf
      repeat begin			; 4
	 a=systime()
	 hour = strmid(a,11,2)
	; ** Halt processing for QL before database update at 2 AM.
	 IF hour GE 1 and hour LT 3 and src EQ 1 THEN BEGIN
		spawn,'whoami',user,/sh
		user=user(0)
		message=user+', source='+TRIM(STRING(src))+': processing stopped at '+filename
		subject='reduce_main stopped'
		openw,tlun,'maildummy',/get_lun
		printf,tlun,message
		close,tlun
		free_lun,tlun
		spawn,'mail -s "'+subject+'" '+user+' < maildummy',/sh
		save, last_tai,date_str,filename=leb_img+'/lastdate.sav'
		return
	 ENDIF
	; **
         done = 1
         openr,luf,fnclosed,error=err
         if (err gt 0) then begin 
            print,'Error opening '+fnclosed
            wait,60.
            done = 0
         endif else begin
            ON_IOERROR, BADIO
            fn = ''
            IF (EOF(luf)) THEN BEGIN
               PRINT,'Error reading '+fnclosed
               WAIT,10.
               done = 0
            ENDIF ELSE BEGIN
               readf,luf,fn
               if (strpos(fn,'FP') ne -1) then begin
                  nc = strpos(fn,'.')
                  fn = strmid (fn,nc-13,14)+'img'
               endif
               close,luf
               if (fn eq lastfn) then begin
                  done=0
		  save, last_tai,date_str,filename=leb_img+'/lastdate.sav'
                  IF not keyword_set (auto) then return

                  print,'Filename in '+fnclosed+' same as last so must wait'
;
;	Check to see if GSFC processing and if so then remove log files from
;	dates that were dated at least 4 days ago.
;
		  IF (src EQ 0)  THEN BEGIN
		     GET_UTC,dte
		     dte.mjd = dte.mjd-4		; Remove 4 days ago
		     ymd = UTC2YYMMDD(dte)
;
;	First check for red files in $REDUCE_LOG/log
;	Find the filenames that match our date.
;	Then delete filenames that are even earlier than our date.
;	This takes care of any oddballs.
;
		     oldfn = GETENV('REDUCE_LOG')+'/log/red_*.log'
		     oldf = FINDFILE(oldfn)
		     s = SIZE(oldf)
		     IF (s(0) EQ 0)  THEN wait,60. ELSE BEGIN
		        fpos = STRPOS(oldf,ymd)		; Do any files match our date?
			wf = WHERE(fpos NE -1,nwf)
			IF (nwf EQ 0)  THEN WAIT,60. ELSE BEGIN
			   lastnw = wf(nwf-1)		; Pick up last file that matches
			   PRINT,'Removing old log files'
			   FOR iwf=0,lastnw DO SPAWN,'/bin/rm '+oldf(iwf)	; Delete all files up to last
;
;	Now check for unpk files in $REDUCE_LOG
;
			   oldfn = GETENV('REDUCE_LOG')+'/unpk_*.log'
			   oldf = FINDFILE(oldfn)
			   s = SIZE(oldf)
			   IF (s(0) NE 0)  THEN BEGIN
			      fpos = STRPOS(oldf,ymd)
				wf = WHERE(fpos NE -1,nwf)
				IF (nwf NE 0)  THEN BEGIN
				   lastnw = wf(nwf-1)
				   FOR iwf=0,lastnw DO SPAWN,'/bin/rm '+oldf(iwf)
				ENDIF
			   ENDIF
			ENDELSE
		     ENDELSE
		  ENDIF ELSE wait,60.
               endif
            ENDELSE
         endelse
         GOTO,CONT
BADIO:
         PRINT,'%ERROR: REDUCE_MAIN, IO ERROR, type='+!err_string+' on file '+fnclosed
         WAIT,10.
         done = 0
CONT:
      endrep until (done eq 1)			; 4
      ON_IOERROR, NULL
      free_lun,luf
      lastfn = fn
;
;  find the last occurence of the directory path, if any, and extract 
;  only the file name
;
      slash = str_index (fn,'/')		;  unix only
      s = size(slash)
      if (s(0) ne 0) then slash=slash(s(1)-1)
      fn=strmid(fn,slash+1,strlen(fn))
      date_fn = str2utc(ddistim2ecs(strmid(fn,0,strlen(fn)-4)))
;  
;  See if the first file in $LEB_IMG to be processed
;  is earlier than or equal to fn.
;  We know that findfile will return the earliest file first.  All mem
;  files will be processed before any img file.
;  DDIS adds a character A, B, ... to .img if the time is the same as the
;  preceeding file. (????)
;
      if (nfiles eq 0) then GOTO, CONT2 else begin
         filename = files(ifiles)
;
;  Find the root portion of the file name and form file names
;
         n = strpos(filename,'.')
         root = strmid (filename,0,n)
	 date_str = ddistim2ecs(root)
         date_file = str2utc(date_str)
	 cur_tai = utc2tai(date_file)
         ext = STRMID (filename,n+1,n)
         IF (STRUPCASE(STRMID(ext,0,3)) EQ 'MEM')  THEN done=1 ELSE BEGIN
            GET_UTC,dte		; **NBR, moved 11/1/99
            diff = diff2time(date_fn,date_file)
            CASE diff OF
            1:   BEGIN		; first file is later than closed_img_file
	           IF (sf(0) EQ 1)  THEN BEGIN	; number of .img files>0
;
;  Check to see if first file is a valid date, between 12/1/95 and current
;  date.  If it is an invalid date then process the image file.  IF it is
;  a valid date then wait.
;
                      dmjd=date_file.mjd
                      IF (dmjd GT 50052) AND (dmjd LE dte.mjd)  $
                         THEN done=0 ELSE done=1
                   ENDIF ELSE done=0
 	         END
            0:   BEGIN
;
;  root portion agrees with filename, check extensions
;  first check if filename has extra character and if not must
;  be done.
;
                    IF ( STRLEN(ext) EQ 3 ) THEN done=1 ELSE BEGIN
;
;  filename has extra character so check last character in fn
;
                       fnext = STRMID(fn, STRPOS(fn,'.')+1,n)
                       IF ( STRLEN(fnext) EQ 3 ) THEN done=0 ELSE BEGIN
;
;  if extension to fn only has 3 characters then not done
;
                          fnchar = STRMID ( fnext, STRLEN (fnext)-1,1)
                          fchar = STRMID ( ext, STRLEN (ext)-1,1)
                          IF ( fchar LE fnchar ) THEN done=1 ELSE done=0
                       ENDELSE
                    ENDELSE
                 END
            -1:  done=1		; img date is earlier, so process it
            ENDCASE
         ENDELSE
      ENDELSE
   ENDREP UNTIL (done EQ 1)			; 3
;
;  OK.  File must be closed, so begin processing
;
print,'Processing file: '+filename
;
;  Open a log file, and print pertinent information
;
   lastfn = ''   ;  process anything next time
   log = getenv_slash ('REDUCE_LOG')
   logroot = log+'log/red_'+root
   get_utc,dte,/ecs
   openw,lulog,logroot+'.log',/get_lun
   printf,lulog,'Procedure = reduce_main'
   printf,lulog,'Version   = '+ver
   printf,lulog,'Date      = '+dte
   spawn,'hostname',host,/sh
   printf,lulog,'Host      = '+host
   spawn,'domainname',dom,/sh
   printf,lulog,'Domain    = '+dom
;
;  Assign the source designation: 0=> GSFC R/T
;  1 => NRL R/T and 2=>NRL replay (either level 0 or quick look)
;
   if (n_params() eq 0) then begin
      if (dom(0) eq 'nascom.nasa.gov') then source=0 else begin 
         if (host(0) eq 'calliope') then source=1 else source=2
      endelse
   endif else source=src
   printf,lulog,'Source    =',source
;
;  Get the processing options from environment string
;
   opt = strupcase(getenv ('REDUCE_OPTS'))
   printf,lulog,'Reduce options = '+opt
;
;  now open the file to get the file size
;
   openr,lu,filename,/get_lun
   st=fstat(lu)
   oldsize=st.size
   close,lu
   free_lun,lu
   printf,lulog,'Reducing image file  ='+filename
   printf,lulog,'File size (bytes)    =',oldsize
;
;  If generating the DBMS update commands, then open a file
;
   if (strpos(opt,'DBMS') ge 0)   then begin
      dbfile = log+'db/red_'+root+'.db'
      openw,ludb,dbfile,/get_lun
	IF src EQ 2 THEN BEGIN
	   printf,ludb,'use lasco
	   printf,ludb,'go'
	   ;obsdate = utc2str(date_file,/ecs,/date_only)
	   printf,ludb,'delete img_leb_hdr from img_files,img_leb_hdr where '
	   printf,ludb,'img_files.fileorig= "'+filename+'" and'
	   printf,ludb,'img_leb_hdr.date_mod = img_files.date_mod and'
	   printf,ludb,'img_leb_hdr.fileorig = img_files.fileorig and source = 1'
	   printf,ludb,'go'
	ENDIF
      printf,lulog,'DB update file = '+dbfile
   endif
;
;  Now process the image file after changing default directory to
;  $IMAGES
;
   images = getenv_slash ('IMAGES')
   cd,images
   printf,lulog,'Changing to directory ='+images
   for i=0,5 do printf,lulog

; **NBR, 10/20/99
   num_s =cur_tai - last_tai
   now_tai = utc2tai(dte)
  IF cur_tai LT now_tai and cur_tai GT earliest_tai and STRMID(ext,0,3) EQ 'img' THEN BEGIN
   IF num_s GT 60*60 THEN BEGIN		; IF time difference GT 1 hour and .img file
					; then send a message
	minutes=STRING(num_s/60., FORMAT='(I4)')
	spawn,'whoami',user,/sh
	user=user(0)
	message=user+', source='+TRIM(STRING(src))+':'+minutes+' minute gap at '+date_str
	subject=user+TRIM(STRING(src))+':'+minutes+' minute gap'
	openw,tlun,'maildummy',/get_lun
	printf,tlun,message
	close,tlun
	free_lun,tlun
	spawn,'mail -s "'+subject+'" '+user+' < maildummy',/sh
   ENDIF
   last_tai=cur_tai
  ENDIF
	
   reduce_image,filename,source
   if (strpos(opt,'DBMS') ge 0)   then begin
      close,ludb
      get_utc,dte,/date_only,/ecs
      dte = strmid(dte,0,4)+strmid(dte,5,2)+strmid(dte,8,2)
      IF (src eq 2) THEN tt='.final_lst' ELSE tt='.quick_lst'
      openw,ludb,log+'updates/update_'+dte+tt,/append
      printf,ludb,'$1 < '+dbfile    
      close,ludb
      spawn,'chmod a+x '+log+'updates/update_'+dte+tt,/sh
      free_lun,ludb
      if (strpos(opt,'UPDATE') ge 0)   then $
         spawn,'isql '+dbfile    ,/sh 
   endif
   get_utc,dte,/ecs
   printf,lulog,'Reduce_main completed at '+dte
   close,lulog
   free_lun,lulog
   ifiles = ifiles+1
   ENDREP UNTIL (ifiles GE nfiles)		; 2
 CONT2:
endrep until (true eq 0)			; 1

;return
end