pro undefine, varname tempvar = SIZE(TEMPORARY(varname)) end pro xrt_missing_imgs, start_time, end_time, n_miss, n_images, n_outliers,$ dv=dv, metrics=metrics, archive_prefix=archive_prefix,$ timeline_dir = timeline_dir, qstop = qstop ; ========================================================================= ; ;+ ; PROJECT: ; Solar-B / XRT ; ; NAME: ; ; XRT_MISSING_IMGS ; ; CATEGORY: ; ; XRT Data Verification utilities ; ; ; PURPOSE: ; ; Print to the screen a list of missing XRT images in a given ; time interval ; ; ; CALLING SEQUENCE: ; ; XRT_MISSING_IMGS,start_time,end_time, n_miss, n_images, ; n_outliers, /dv ; ; INPUTS: ; ; start_time: (string or double) start of the time ; interval that is checked for missing images. Can be in ; any format accepeted by ANYTIM. ; end_time: (string or double) end of the time ; interval that is checked for missing images. Can be in ; any format accepeted by ANYTIM. ; ; KEYWORD INPUTS: ; ; dv: set this keyword to examine QuickLook data, output info to ; screen for DV report. ; metrics: set this keyword to examine Level0 data, output ; info to screen for XTR Monthly Metrics report ; ; ; OPTIONAL INPUTS: ; ; archive_prefix: (string) Root of the directory trees where XRT ; images reside. The default is '/archive/hinode/xrt/'. ; Directory structure is assumed to be in the form ; archive_prefix+'QLraw/yyyy/mm/dd/Hxx00/' ; timeline_dir: (string) directory where timelines live (optional) ; qstop: Set for debugging mode. ; ; OUTPUTS: ; ; n_miss: number of missing images over time interval ; n_images: number of total images taken ; n_outliers: number of images with incorrect time stamps ; ; EXAMPLE: ; ; find all missing images between 2008/10/03 10:00 UT and ; 2008/10/04 10:00 UT ; xrt_dv_find_missing_imgs,'03-OCT-2008 10:00','04-OCT-2008 10:00' ; ; COMMON BLOCKS: ; ; none ; ; PROCEDURE: ; ; This routine calls xrt_id_ecids to do the heavy lifting. ; See the comments in that programs for algorithmic details. ; ; KNOWN PROBLEMS: ; ; 1) When the program is started in the middle of a time period where ; no data was intentially taken, an error message is returned: ; ; ------------------------------------------------------------ ; | No times between t0 and t1, returning record closest to t0 | ; | DeltaT (catrec,t0) = 25381 seconds | ; ------------------------------------------------------------ ; ------------------------------------------ ; | No XRT records meet time/search criteria | ; ------------------------------------------ ; ; The solution is to expand the time range of the search. ; ; ; NOTES: ; 1) Warning: since this program reads in the FITS file header, ; it will be slow for time intervals spanning more than a few days, ; where a sizable number of FITS header needs to be read in. ; ; 2) Either the DV or the METRICS keyword must be set, but not ; both. ; ; CONTACT: ; ; Comments, feedback, and bug reports regarding this routine may be ; directed to this email address: ; xrt_manager ~at~ head.cfa.harvard.edu ; ; ; ; MODIFICATION HISTORY: ; progver='v2011.Feb.10' ;--- (K. Reeves. SAO) written ; progver='v2011.July.22' ;--- (K. Reeves, SAO) added timelines_dir keyword ; progver='v2011.Oct.06' ;--- (K. Reeves, SAO) added logic to catch missing images between XOBs. ; progver='v2011.Oct.06a' ;--- (K. Reeves, SAO) Fixed bug in above logic. ; progver='v2011.Oct.07' ;--- (K. Reeves, SAO) Moved assignemnt of ecid_last and index_last ; variables to end of qabort=1 section. Don't look for gaps between flare XOB and regular XOB ; progver='v2011.Oct.18' ;--- (K. Reeves, SAO) Fixed bug in XOB gap logic, added stuff to qabort=3 section to handle mislabeled flare images ; progver='v2011.Nov.1' ;--- (K. Reeves, SAO) Removed stop left over from debugging ; progver='v2011.Nov.23' ;--- (K. Reeves, SAO) included case where ecid_last is greater than ecid_list[0] - usually due to FLD images. ; progver='v2011.Nov.28' ;--- (K. Reeves, SAO) I mean where ecid_last is greater than *or equal* to ecid_list... ; progver='v2012.Feb.5' ;--- (K. Reeves, SAO) Fixed instance where ecid_last is greater than ecid_list[0] due to rollover of ECID counter. ; progver='v2012.Mar.12' ;--- (K. Reeves, SAO) Fixed bug introduced by last fix. ; should be: if (ecid_last - ecid_list[0]) ge 0 and (ecid_last - ecid_list[0]) le 2000 then begin ; progver='v2012.Jul.10' ;--- (K. Reeves, SAO) Changed definition of 'flare mode' so that ; either index.flfg = 'FLR' or index.obs_mode='FL'. This accomplishes two things: ; 1) flare mode is not interrupted by flare patrol images (which have flflg='FLR', but 'obs_mode' = 'QT'.) ; 2) It catches flare images that are mislabeled as outside of flare mode because ; of timing mismatches between Hinode clocks. These images have flflg='NON', but 'obs_mode' = 'FL'. ; progver='v2012.Dec.3' ;--- (K. Reeves, SAO) - Added case statement for qabort=1. This can happen when the start ; time of the next timeline file is ; significantly earlier than the end time from ; the last file, causing xrt_tim2xobt to return different start and stop times. ;- ; ========================================================================= ;; the DV keyword causes the code to search quicklook data, the ;; METRICS keyword causes it to search level0 data. Can't do both! if keyword_set(dv) and keyword_set(metrics) then begin print, 'You can only set one of the DV or METRICS keywords. Aborting.' return endif day = 3600.*24 ;; one day in seconds int_end_time = anytim(end_time) xob_stop = start_time n_miss = 0 n_images = 0 n_outliers = 0 nmin = 1 flare_mode = 0 while anytim(xob_stop) le int_end_time do begin if keyword_set(qstop) then print, 'xob_stop at while loop start: ', xob_stop ;; Parse the time period into xob times ;; xrt_time2xobt doesn't work for flare mode if n_elements(qabort) ne 0 then undefine, qabort ;; clear qerror if n_elements(qerror) ne 0 then undefine, qerror if not flare_mode then begin xrt_tim2xobt, start_time, xob_start=xob_start, xob_stop=xob_stop, timeline_dir=timeline_dir,$ qerror=qerror while n_elements(qerror) gt 0 do begin start_time = atime(anytim(start_time)+day) ;;advance start time by a day print, '***no sim file or timeline file found. Advancing by one day to ' +start_time+'***' undefine, qerror xrt_tim2xobt, start_time, xob_start=xob_start, xob_stop=xob_stop, timeline_dir=timeline_dir,$ qerror=qerror endwhile ;; move stop time a minutes earlier so it won't get flagged ;; by xrt_id_ecids as a different xob stop_time = atime(anytim(xob_stop)-nmin*60) endif ;; stop at requested end time if it is earlier than xob stop time if anytim(stop_time) ge int_end_time then stop_time = end_time if keyword_set(qstop) then begin print, 'before id_ecids...' print, anytim(start_time, /vms) print, anytim(stop_time, /vms) endif xrt_id_ecids, start_time, stop_time, ecid_list, ecid_codes, $ fits_index=index, qabort=qabort, quicklook=dv, $ level0=metrics, archive_prefix=archive_prefix,$ ss_outliers = ss_outliers, /quiet, qstop=qstop if keyword_set(qstop) then print, 'qabort = ', qabort ;; continue only if xrt_id_ecids executed successfully. A ;; qabort value of 2 indicates that no data was found. ;; qabort value of 3 indicates input timerange spans more than one XOB case 1 of (qabort eq 0): begin ;; print out output for metrics if keyword_set(metrics) then begin print, anytim(start_time, /vms) print, anytim(stop_time, /vms) endif ;; if ecid last is bigger than ecid_list[0], then truncate ;; ecid_list, but if it's a lot bigger, probably ;; there was a rollover. if n_elements(ecid_last) ne 0 then begin if (ecid_last - ecid_list[0]) ge 0 and (ecid_last - ecid_list[0]) le 2000 then begin minec = min(abs(ecid_list - ecid_last),nec_id) ecid_codes = ecid_codes[nec_id+1:*] ecid_list = ecid_list[nec_id+1:*] endif endif ;; find index numbers of all images labeled as missing that ;; are not positively identified as preflare buffer images match = where(strmatch(ecid_codes, 'MISS/Pat*') eq 1 or $ strmatch(ecid_codes, 'MISS/Reg*') eq 1 or $ strmatch(ecid_codes, 'MISS/Flr.') eq 1 or $ strmatch(ecid_codes, 'MISS/\?\?\?*') eq 1) test=match ;; find index numbers of images labeled as present match_prsnt = where(strmatch(ecid_codes, 'PRST*') eq 1) if n_elements(ecid_last) ne 0 then begin ;; this part checks to see if there are missing images ;; *between* XOBs. The fancy check for preflare images ;; is not included, so the count will be high. KKR ;; 2011/10/06 if not flare_mode then begin if (ecid_list[0] - ecid_last) gt 1 then begin ;; if there are missing images between the last image ;; of the last XOB and the first image of the next ;; one, add last XOB last image index to the array of ;; index, and doctor missing and present index arrays ;; accordingly. ;; added code to deal with ECID counter rollover - ;; KKR 2012/02/06 index = concat_struct(index_last, index) ngap = ecid_list[0] - ecid_last if ngap lt 0 then ngap = ngap + 32768L ecid_list_first = ecid_list[0] ecid_list = [findgen(ngap)+ecid_list[0]-ngap,ecid_list] match_prsnt = [0,match_prsnt+ngap] if ss_outliers[0] ne -1 then ss_outliers = ss_outliers+ngap if match[0] ne -1 then match = [findgen(ngap),match+ngap] else match = findgen(ngap-1)+1 endif endif endif ;; print out each missing image for DV report if keyword_set(dv) then $ xrt_create_dv_output, ecid_list,match,match_prsnt,$ index,ss_outliers ;; keep track of missing images found on this time through ;; the loop if match[0] eq -1 then new_miss = 0 else new_miss = n_elements(match) ;; total number of images is given by the number of images ;; marked as "present" new_images = n_elements(match_prsnt) ;; record bad timestamp images if ss_outliers[0] ne -1 then $ new_outliers = n_elements(ss_outliers) $ else new_outliers = 0 ;; make the start time equal to the current XOB end time for the ;; next iteration through the loop if flare_mode then begin if anytim(start_time) lt anytim(flr_mode_start)-10 then begin start_time = flr_mode_start stop_time = flr_mode_stop endif else begin start_time = atime(anytim(flr_mode_stop)+10) stop_time = stop_time_orig endelse endif else begin start_time = xob_stop endelse ;; add missing images and total images to totals n_miss = n_miss + new_miss n_images = n_images + new_images n_outliers = n_outliers + new_outliers ;; get rid of qabort definition so that it isn't used ;; as an input next time around. undefine, qabort ;; set minute interval to 1 for next time around nmin = 1 ;; unset flare mode if it was set last time flare_mode = 0 ecid_last = ecid_list[n_elements(ecid_list)-1] index_last =index[n_elements(index)-1] end ;; qabort = 1 means start_time and stop_time are in different ;; xobs, probably because there is a overlap in sim files, and ;; the later sim file starts significantly before the earlier ;; one ends (qabort eq 1): begin ;; get rid of qabort definition so that it isn't used ;; as an input next time around. undefine, qabort ;; increment minute interval until we find the right end time nmin = nmin+1 ;; unset flare mode if it was set last time flare_mode = 0 ;ecid_last = ecid_list[n_elements(ecid_list)-1] ;index_last =index[n_elements(index)-1] end ;; qabort = 2 means no data found at start time (qabort eq 2): begin print, 'No data found at start time' ;; get rid of qabort definition so that it isn't used ;; as an input next time around. undefine, qabort ;; make the start time equal to the current XOB end time for the ;; next iteration through the loop start_time = xob_stop ;; set minute interval to 1 for next time around nmin = 1 ;; unset flare mode if it was set last time flare_mode = 0 ;ecid_last = ecid_list[n_elements(ecid_list)-1] ;index_last =index[n_elements(index)-1] end (qabort eq 3): begin ;; check for flare mode, which is one reason that there ;; could be two XOBs in the specified time period ;; changed from requiring only obs_mode = 'FL' to either ;; that, or flflg = 'FLR'. See notes in header. KKR ;; 07/10/2012 match = where(strmatch(index.obs_mode, 'FL') eq 1 or strmatch(index.flflg,'FLR') eq 1) if match[0] ne -1 then begin ;; check to see if flare mode went off more than ;; once. If flare mode went off only once, elements ;; of match will be sequential, and there will be no ;; element of match_diff greater than one. If there ;; is more than one instance of flare mode, ;; use the stop time for the first set of flare mode images match_diff = shift(match, -1) - match nn = where(match_diff gt 1) if nn[0] ne -1 then $ flr_mode_stop = index[match[nn[0]]].date_obs $ else $ flr_mode_stop = index[match[n_elements(match)-1]].date_obs flare_mode=1 flr_mode_start = index[match[0]].date_obs stop_time_orig = stop_time if anytim(start_time) lt anytim(flr_mode_start)-5 then begin ;; stop time 5 sec before first flare mode image stop_time = atime(anytim(flr_mode_start)-5) endif else begin stop_time = atime(anytim(flr_mode_stop)) endelse xob_stop = stop_time print, 'Flare mode detected: ' + anytim(flr_mode_start,/vms) $ +' - ' + anytim(flr_mode_stop, /vms) endif else begin ;; check to see if first image is bad one ;; this can happen if last image in flare mode ;; is mislabeled as QT obs_no = index[0].prog_no match = where(obs_no eq index[1:*].prog_no) if match[0] eq -1 then begin ;; if first image is rogue, take the next ;; image's start time start_time = index[1].date_obs endif ;; if there's no flare mode, it's possible that the end ;; time is overlapping the next XOB. This happens ;; sometimes at the boundary between timelines. ;; increment minutes to decrease stop time nmin = nmin +1 ;; reset xob_stop so that program will iterate again xob_stop = start_time endelse end endcase if n_elements(qabort) ne 1 then begin ;; print out output for metrics if keyword_set(metrics) then begin print, 'Number missing: ',n_miss print, 'Total images: ',n_images endif endif endwhile ;; print out output for metrics if keyword_set(metrics) then begin print, string(n_outliers,format='(i3)')+ $ ' image(s) with incorrect time stamps' print, strtrim(n_miss,1) +' missing images out of '+$ strtrim(n_images,1)+': '+ $ strtrim(100*n_miss/float(n_images),1)+'% missing.' endif end