pro go_mk_nfi_flat, t0, t1, $ level0=level0, use_fg_prep=use_fg_prep, $ mk_flat=mk_flat, do_read=do_read, do_print=do_print, $ bin_siz=bin_siz, n_files_sot_set_min=n_files_sot_set_min, $$ maxiter=maxiter, max_error=max_error, notvshow=notvshow, $ _extra=_extra t_bench0 = !stime if not exist(level0) then level0 = 1 if not exist(t_delt_max_sec) then t_delt_max_sec = 300d0 ; 3*3600d if not exist(t_cat_sec_delt_max) then t_cat_sec_delt_max = 48 ;12 if not exist(use_fg_prep) then use_fg_prep = 1 if not exist(mk_flat) then mk_flat = 1 if not exist(n_files_sot_set_min) then n_files_sot_set_min = 10 ;5 if not exist(maxiter) then maxiter = 100 if not exist(max_err) then max_err = .003 if not exist(min_frac) then min_frac = .001 if not exist(max_n_zero) then max_n_zero = 10 if not exist(fake_nfi_fg_start_string) then fake_nfi_fg_start_string = $ '/archive1/hinode/sot/ops/timeline/monthly_file_2020_01.tim:' + $ 'ABS: 2020/01/01.00:00:00 START FG Table NN, f N.NN,1 NFI flat field WAVE FAKE' fake_nfi_fg_start_arr = str2arr(strcompress(fake_nfi_fg_start_string), ' ') if not exist(fake_bfi_fg_start_string) then fake_bfi_fg_start_string = $ '/archive1/hinode/sot/ops/timeline/monthly_file_2020_01.tim:' + $ 'ABS: 2020/02/01.00:00:00 START FG Table NN, f N.NN,1 BFI flat field WAVE FAKE' fake_bfi_fg_start_arr = str2arr(strcompress(fake_bfi_fg_start_string), ' ') if not exist(fake_nfi_fg_stop_string) then fake_nfi_fg_stop_string = $ '/archive1/hinode/sot/ops/timeline/monthly_file_2020_01.tim:' + $ 'ABS: 2020/01/01.01:00:00 STOP FG, SP, CT FAKE' fake_nfi_fg_stop_arr = str2arr(strcompress(fake_nfi_fg_stop_string), ' ') if not exist(fake_bfi_fg_stop_string) then fake_bfi_fg_stop_string = $ '/archive1/hinode/sot/ops/timeline/monthly_file_2020_01.tim:' + $ 'ABS: 2020/02/01.01:00:00 STOP FG, SP, CT FAKE' fake_bfi_fg_stop_arr = str2arr(strcompress(fake_bfi_fg_stop_string), ' ') if not exist(dir_top) then $ dir_top = '/sanhome/slater/public_html/missions/hinode/sot/nfi/flats' if not exist(dir_sot_event_lists) then $ dir_sot_event_lists = '/net/topaz/Users/slater/soft/idl/idl_startup/sot' ;filnam_nfi_flat_start = 'flat_grep_blanknfiblank_.lis' ; 'nfi_flat.lis' ;filnam_fg_start = 'start_fg.lis' ;filnam_fg_stop = 'stop_fg.lis' filnam_fg_start = concat_dir(dir_sot_event_lists, 'start_fg.lis') filnam_fg_stop = concat_dir(dir_sot_event_lists, 'stop_fg.lis') nbfw_range_arr = [[030,045],[070,085],[110,125],[150,165],[190,205],[230,245]] wave_arr = [ '6302', '5576', '5896', '5250', '5172', '6563', 'no move' ] wave_num_arr = wave_arr ; Restore NFI mask save file. Mask array will be in variable 'mask': restore, file = '/net/topaz/Users/slater/soft/idl/idl_startup/sot/nfi_mask_20100805.save' ;restore, file = '/net/tarbell//Users/tarbell/solarb/flats/nfi_mask_20100805.save' ; Create list of all fg_start times: buff_fg_start_no_parse = rd_tfile(filnam_fg_start) buff_fg_start = rd_tfile(filnam_fg_start, 13, delim=' ') ; Append final fake record to buff arrays: buff_fg_start_no_parse = [buff_fg_start_no_parse, fake_nfi_fg_start_string] buff_fg_start = [[buff_fg_start], [fake_nfi_fg_start_arr]] t_fg_start = str_replace(reform(buff_fg_start[1,*]), '.', ' ') ss_valid_time = where( (strmid(t_fg_start, 0,2) eq '20') and $ (strmid(t_fg_start, 4,1) eq '/') and $ (strmid(t_fg_start, 7,1) eq '/') and $ (strmid(t_fg_start,10,1) eq ' ') and $ (strmid(t_fg_start,13,1) eq ':') and $ (strmid(t_fg_start,16,1) eq ':') ) buff_fg_start_no_parse = buff_fg_start_no_parse[ss_valid_time] buff_fg_start = buff_fg_start[*, ss_valid_time] t_fg_start = t_fg_start[ss_valid_time] t_fg_start_sec = anytim(t_fg_start) ss_uniq_sort_fg_start = uniq(t_fg_start_sec, sort(t_fg_start_sec)) buff_fg_start_no_parse = buff_fg_start_no_parse[ss_uniq_sort_fg_start] buff_fg_start = buff_fg_start[*, ss_uniq_sort_fg_start] t_fg_start = t_fg_start[ss_uniq_sort_fg_start] t_fg_start_sec = t_fg_start_sec[ss_uniq_sort_fg_start] t_fg_start_int = anytim(t_fg_start, /int) n_t_fg_start = n_elements(t_fg_start) t_fg_start_sec_delt = t_fg_start_sec[1:*] - t_fg_start_sec ; Create list of nfi flat start times: ;ss_bfi_flat_start = where( ( (strpos(strlowcase(buff_fg_start_no_parse), 'nfi' ) ne -1) and $ ; (strpos(strlowcase(buff_fg_start_no_parse), 'flat') ne -1) ), n_nfi_flat_start) ; NB - Some timeline comments for flat runs do not include 'BFI' or 'NFI', so we will not search on these: ;ss_nfi_flat_start = where( (strpos(strlowcase(buff_fg_start_no_parse), 'flat') ne -1), n_bfi_flat_start) ss_nfi_flat_start = where( ( (strpos(strlowcase(buff_fg_start_no_parse), 'flat' ) ne -1) and $ (strpos(strlowcase(buff_fg_start_no_parse), 'bfi') eq -1) ), n_nfi_flat_start) buff_nfi_flat_start_no_parse = buff_fg_start_no_parse[ss_nfi_flat_start] buff_nfi_flat_start = buff_fg_start[*, ss_nfi_flat_start] t_start = t_fg_start[ss_nfi_flat_start] t_start_sec = t_fg_start_sec[ss_nfi_flat_start] t_start_int = anytim(t_start, /int) n_t_start = n_elements(t_start) t_start_sec_delt = t_start_sec[1:*] - t_start_sec ; If t0 was passed, then limit t_start to times greater than t0: if exist(t0) then begin ss_ge_t0 = where(t_start_sec ge (anytim(t0)-60), n_ge_t0) ; Since the last t_start FG START record is fake, must have more than one: if n_ge_t0 gt 1 then begin buff_nfi_flat_start_no_parse = buff_nfi_flat_start_no_parse[ss_ge_t0] buff_nfi_flat_start = buff_nfi_flat_start[*, ss_ge_t0] t_start = t_start[ss_ge_t0] t_start_sec = t_start_sec[ss_ge_t0] t_start_int = anytim(t_start, /int) n_t_start = n_elements(t_start) t_start_sec_delt = t_start_sec[1:*] - t_start_sec endif else begin print, " No 'START FG' times greater than passed t0. Returning." return endelse endif ; If t1 was passed, then limit t_start to times less than t1: ; Since the last t_start FG START record is fake, must have more than one: if n_elements(t_start_sec) gt 1 then begin if exist(t1) then begin ss_le_t1 = where(t_start_sec le (anytim(t1)), n_le_t1) if n_le_t1 gt 0 then begin buff_nfi_flat_start_no_parse = buff_nfi_flat_start_no_parse[ss_le_t1] buff_nfi_flat_start = buff_nfi_flat_start[*, ss_le_t1] t_start = t_start[ss_le_t1] t_start_sec = t_start_sec[ss_le_t1] t_start_int = anytim(t_start, /int) n_t_start = n_elements(t_start) t_start_sec_delt = t_start_sec[1:*] - t_start_sec endif else begin print, " No 'START FG' times less than passed t1. Returning." return endelse endif endif else begin print, " No 'START FG' times less than passed t1. Returning." return endelse ; Create list of all fg stop times: buff_stop_no_parse = rd_tfile(filnam_fg_stop) buff_stop = rd_tfile(filnam_fg_stop, 6, delim=' ') ; Append final fake record to buff arrays: buff_stop_no_parse = [buff_stop_no_parse, fake_nfi_fg_stop_string] buff_stop = [[buff_stop], [fake_nfi_fg_stop_arr[0:5]]] t_stop = str_replace(reform(buff_stop[1,*]), '.', ' ') ss_valid_time = where( (strmid(t_stop, 0,2) eq '20') and $ (strmid(t_stop, 4,1) eq '/') and $ (strmid(t_stop, 7,1) eq '/') and $ (strmid(t_stop,10,1) eq ' ') and $ (strmid(t_stop,13,1) eq ':') and $ (strmid(t_stop,16,1) eq ':') ) buff_stop_no_parse = buff_stop_no_parse[ss_valid_time] buff_stop = buff_stop[*, ss_valid_time] t_stop = t_stop[ss_valid_time] t_stop_sec = anytim(t_stop) ss_uniq_sort_stop = uniq(t_stop_sec, sort(t_stop_sec)) buff_stop_no_parse = buff_stop_no_parse[ss_uniq_sort_stop] buff_stop = buff_stop[*, ss_uniq_sort_stop] t_stop = t_stop[ss_uniq_sort_stop] t_stop_sec = t_stop_sec[ss_uniq_sort_stop] t_stop_int = anytim(t_stop, /int) n_t_stop = n_elements(t_stop) t_stop_sec_delt = t_stop_sec[1:*] - t_stop_sec ; Create combined list of all fg start and stop times: buff_start_stop_no_parse = [buff_fg_start_no_parse, buff_stop_no_parse] t_start_stop = [t_fg_start, t_stop] t_start_stop_sec = [t_fg_start_sec, t_stop_sec] ss_uniq_sort_start_stop = uniq(t_start_stop_sec, sort(t_start_stop_sec)) buff_stop_no_parse = buff_start_stop_no_parse[ss_uniq_sort_start_stop] t_stop = t_start_stop[ss_uniq_sort_start_stop] t_stop_sec = t_start_stop_sec[ss_uniq_sort_start_stop] t_stop_int = anytim(t_start_stop, /int) n_t_stop = n_elements(t_stop) t_stop_sec_delt = t_stop_sec[1:*] - t_stop_sec ; ss = tim2dset(t_ref_arr, t_samp_arr, delta_sec=delta_sec) for i=0, n_t_start-1 do begin t_offset_sec = (t_stop_sec - t_start_sec[i]) ss_pos_offset = where(t_offset_sec gt 0, n_pos_offset) if n_pos_offset gt 0 then begin t_pos_offset_min = min(t_offset_sec[ss_pos_offset], ss_pos_offset_min) if not exist(t_stop_match_arr) then begin buff_stop_no_parse_match_arr = buff_stop_no_parse[ss_pos_offset[ss_pos_offset_min]] t_stop_match_arr = t_stop[ss_pos_offset[ss_pos_offset_min]] endif else begin t_stop_match_arr = [t_stop_match_arr, t_stop[ss_pos_offset[ss_pos_offset_min]]] buff_stop_no_parse_match_arr = $ [buff_stop_no_parse_match_arr, buff_stop_no_parse[ss_pos_offset[ss_pos_offset_min]]] endelse endif else begin stop, " No 'FG_STOP' found after this FG_START." endelse endfor ; Clustering of start times: i = 0 while i lt (n_t_start-1) do begin n_clust = 1 while ( (t_start_sec_delt[i] le t_delt_max_sec) and $ (i lt (n_t_start-1)) ) do begin n_clust = n_clust + 1 i = i + 1 endwhile i = i + 1 if not exist(n_clust_arr) then begin n_clust_arr = n_clust endif else begin n_clust_arr = [n_clust_arr, n_clust] endelse endwhile n_clusters = n_elements(n_clust_arr) ss_start_clust = total(n_clust_arr, /cum) ss_start_clust = [0, ss_start_clust[0:n_clusters-1]] ; Now, loop through n_clust_arr, reading in all records in each cluster: for i = 0, n_clusters-1 do begin t_clust_start = t_start[ss_start_clust[i]] t_clust_stop = t_stop_match_arr[ss_start_clust[i]] buff_start_no_parse_clust = buff_nfi_flat_start_no_parse[ss_start_clust[i]] buff_stop_no_parse_clust = buff_stop_no_parse_match_arr[ss_start_clust[i]] if keyword_set(do_print) then begin print, ' Timeline messages for this cluster:' prstr, [buff_start_no_parse_clust, buff_stop_no_parse_clust] endif ; xrt_cat, t_clust_start, t_clust_stop, cat_xrt, files_xrt sot_cat, t_clust_start, t_clust_stop, cat_sot, files_sot, level0=level0, /init if size(cat_sot, /type) eq 8 then begin only_tags = 'date_obs, nbfw' mreadfits_header, files_sot, index_sot ;, only_tags=only_tags if ( (not tag_exist(index_sot, 'waveid')) or $ (not tag_exist(index_sot, 'fgccdix0')) or $ (not tag_exist(index_sot, 'fgccdix1')) ) then begin print, ' Index for this cluster missing one or more needed tags. Skipping.' goto, skip1 endif print, index_sot.wave print, 'NFI: ', wave_arr fmt_timer, index_sot.date_obs print, 'Waiting after reading all headers for this super cluster.' ;dum = get_kbrd() ; Break up this cluster if necessary: n_cat = n_elements(cat_sot) t_cat = cat_sot.date_obs t_cat_sec = anytim(t_cat) t_cat_sec_delt = t_cat_sec[1:*] - t_cat_sec ix0_index = index_sot.fgccdix0 ix1_index = index_sot.fgccdix1 waveid_index = index_sot.waveid print, 't_cat_sec_delt', t_cat_sec_delt print, 'waveid', index_sot.waveid print, 'fgccdix0', index_sot.fgccdix0 print, 'fgccdix1', index_sot.fgccdix1 file_append, './cluster_data.txt', '' file_append, './cluster_data.txt', 't_cat_sec_delt: ' + arr2str(strtrim(t_cat_sec_delt,2)) file_append, './cluster_data.txt', 'waveid: ' + arr2str(strtrim(index_sot.waveid,2)) file_append, './cluster_data.txt', 'fgccdix0: ' + arr2str(strtrim(index_sot.fgccdix0,2)) file_append, './cluster_data.txt', 'fgccdix1: ' + arr2str(strtrim(index_sot.fgccdix1,2)) k = 0 delvarx, n_sub_clust_arr while k lt (n_cat-2) do begin n_sub_clust = 1 ix1_index_last = ix1_index[k] waveid_index_last = waveid_index[k] while ( (t_cat_sec_delt[k] le t_cat_sec_delt_max) and $ (ix1_index[k] eq ix1_index_last) and $ (waveid_index[k] eq waveid_index_last) and $ (k lt (n_cat-2)) ) do begin n_sub_clust = n_sub_clust + 1 k = k + 1 ix1_index_last = ix1_index[k] endwhile k = k + 1 if not exist(n_sub_clust_arr) then begin n_sub_clust_arr = n_sub_clust endif else begin n_sub_clust_arr = [n_sub_clust_arr, n_sub_clust] endelse endwhile n_sub_clusters = n_elements(n_sub_clust_arr) if n_sub_clusters eq 1 then begin ss_start_sub_clust = 0 ss_end_sub_clust = n_cat-1 endif else begin tot_sub_clust_arr = total(n_sub_clust_arr, /cum) ss_start_sub_clust = [0, tot_sub_clust_arr[0:n_sub_clusters-2]] ; ss_end_sub_clust = tot_sub_clust_arr-1 ss_end_sub_clust = [ss_start_sub_clust[1:*]-1, n_cat-1] endelse for l = 0, n_sub_clusters-1 do begin cat_sot_sub_clust = cat_sot[ss_start_sub_clust[l]:ss_end_sub_clust[l]] n_rec_cat_sot_sub_clust = n_elements(cat_sot_sub_clust) file_append, './cluster_data.txt', 'n_sub_clusters' + strtrim(n_sub_clusters,2) file_append, './cluster_data.txt', $ 'n_rec_cat_sot_sub_clust' + strtrim(n_rec_cat_sot_sub_clust,2) ; Skip this sub-cluster if it has only one element: if n_rec_cat_sot_sub_clust eq 1 then begin print, ' Only one image in this subset. Skipping.' goto, skip2 endif files_sot_sub_clust = files_sot[ss_start_sub_clust[l]:ss_end_sub_clust[l]] index_sot_sub_clust = index_sot[ss_start_sub_clust[l]:ss_end_sub_clust[l]] waveid_median = median(cat_sot_sub_clust.waveid) ss_waveid_median = where(cat_sot_sub_clust.waveid eq waveid_median, n_waveid_median) ; Skip this sub-cluster if it has only one element where the waveid is equal to the median: if n_waveid_median eq 1 then begin print, ' Only one image in this subset. Skipping.' goto, skip2 endif ss_diff = ss_waveid_median[1:*] - ss_waveid_median ss_big_diff = where(ss_diff gt 1, n_big_diff) if n_big_diff gt 0 then begin for j=0, n_big_diff-1 do begin if j eq 0 then begin ; ss_set_begin = [0, ss_set_begin = waveid_median[0] ss_set_end = waveid_median[ss_big_diff[j]] endif else begin ss_set_begin = waveid_median[ss_big_diff[j-1]+1] ss_set_end = waveid_median[ss_big_diff[j ] ] endelse ; ss_set = ss_waveid_median[ss_set_begin:ss_set_end] ss_set = ss_set_begin + lindgen(ss_set_end - ss_set_begin + 1) n_elem_set = n_elements(ss_set) if keyword_set(do_print) then begin fmt_timer, cat_sot_sub_clust[ss_set].date_obs ; prstr, strtrim(cat_sot[ss_set].naxis1,2) + ' ' + cat_sot_sub_clust[ss_set].wave print, 'BFI waveid_median count: ' + string(n_elem_set, format='$(i3.3)') endif ; Attempt to generate flat from this ss_set: if keyword_set(mk_flat) then begin cat_sot_set = cat_sot_sub_clust[ss_set] files_sot_set = files_sot_sub_clust[ss_set] index_sot_set = index_sot_sub_clust[ss_set] if keyword_set(do_print) then prstr, files_sot_set, /nomore n_files_sot_set = n_elements(files_sot_set) ; Check uniformity of image dimensions and that they are among the allowed set: if ( (n_elements(all_vals(cat_sot_set.waveid )) eq 1) and $ (n_elements(all_vals(cat_sot_set.naxis1 )) eq 1) and $ (n_elements(all_vals(cat_sot_set.naxis2 )) eq 1) and $ (n_elements(all_vals(index_sot_set.fgccdix1)) eq 1) and $ ( ( (cat_sot_set[0].naxis1 eq 2048) and $ (cat_sot_set[0].naxis2 eq 1024) ) or $ ( (cat_sot_set[0].naxis1 eq 4096) and $ (cat_sot_set[0].naxis2 eq 2048) ) ) ) ne 1 then begin print, ' Image dimensions not uniform or not among the allowed set. Skipping.' print, ' WAVEID: ', all_vals(cat_sot_set.waveid) print, ' NAXIS1: ', all_vals(cat_sot_set.naxis1) print, ' NAXIS2: ', all_vals(cat_sot_set.naxis2) print, ' FCCDIX1: ', all_vals(index_sot_set.fgccdix1) goto, skip3a endif ; Determine BFI filter position: nbfw_sot_set = index_sot_set.nbfw ss_6302 = where(((nbfw_sot_set ge nbfw_range_arr(0,0)) and $ (nbfw_sot_set le nbfw_range_arr(1,0))), n_6302) ss_5576 = where(((nbfw_sot_set ge nbfw_range_arr(0,1)) and $ (nbfw_sot_set le nbfw_range_arr(1,1))), n_5576) ss_5896 = where(((nbfw_sot_set ge nbfw_range_arr(0,2)) and $ (nbfw_sot_set le nbfw_range_arr(1,2))), n_5896) ss_5250 = where(((nbfw_sot_set ge nbfw_range_arr(0,3)) and $ (nbfw_sot_set le nbfw_range_arr(1,3))), n_5250) ss_5172 = where(((nbfw_sot_set ge nbfw_range_arr(0,4)) and $ (nbfw_sot_set le nbfw_range_arr(1,4))), n_5172) ss_6563 = where(((nbfw_sot_set ge nbfw_range_arr(0,5)) and $ (nbfw_sot_set le nbfw_range_arr(1,5))), n_6563) n_match_wave_arr = [n_6302, n_5576, n_5896, n_5250, n_5172, n_6563] max_match_wave = max(n_match_wave_arr, ss_match_wave) wave = wave_arr[ss_match_wave] ; TODO: Salvage case where not all nbfw values are the same: if n_match_wave_arr[ss_match_wave] ne n_elem_set then begin print, ' Not all NB filter wheel positions are the same for this set. Skipping.' ;STOP goto, skip3a endif pos_6302 = strpos(cat_sot_sub_clust[ss_set[0]-1].wave, wave_arr[0]) pos_5576 = strpos(cat_sot_sub_clust[ss_set[0]-1].wave, wave_arr[1]) pos_5896 = strpos(cat_sot_sub_clust[ss_set[0]-1].wave, wave_arr[2]) pos_5250 = strpos(cat_sot_sub_clust[ss_set[0]-1].wave, wave_arr[3]) pos_5172 = strpos(cat_sot_sub_clust[ss_set[0]-1].wave, wave_arr[4]) pos_6563 = strpos(cat_sot_sub_clust[ss_set[0]-1].wave, wave_arr[5]) pos_no_move = strpos(cat_sot_sub_clust[ss_set[0]-1].wave, wave_arr[6]) pos_arr = [pos_6302, pos_5576, pos_5896, pos_5250, pos_5172, pos_6563, pos_no_move] ss_match_wave_cat = where(pos_arr ne -1, n_match_wave_cat) print, CAT_SOT_SUB_CLUST.wave print, CAT_SOT_SUB_CLUST.waveid print, n_match_wave_arr STOP if n_match_wave_cat eq 0 then begin print, 'WAVE tag values: ', all_vals(cat_sot_sub_clust.wave) print, ' No match to cat_sot.wave. Skipping.' goto, skip3a endif if n_match_wave_cat gt 1 then begin print, 'WAVE tag values: ', all_vals(cat_sot_sub_clust.wave) print, ' No uniq match to cat_sot.wave. Skipping.' goto, skip3a endif wave_match_cat = wave_arr[ss_match_wave_cat] if ( (wave_match_cat ne wave) and (wave_match_cat ne 'no_move') ) then $ print, ' wave_match_cat does not equal wave. Fix this!' ; stop, ' wave_match_cat does not equal wave. Stopping.' prstr, [wave, wave_match_cat] ; Finally, eliminate incomplete images: delvarx, ss_complete ; if tag_exist(index_sot_set, 'percentd') then $ ; ss_complete = where(index_sot_set.percentd eq 100, n_complete) if not exist(ss_complete) then begin read_sot, files_sot_set, index_sot_test, data_sot_test ; data_sot_summed = total(total(data_sot_test,1),1) for k=0, n_elements(files_sot_set)-1 do begin ss_zero = where(data_sot_test[*,*,k] eq 0, n_zero) if n_zero le max_n_zero then begin if not exist(ss_complete) then ss_complete = k else $ ss_complete = [ss_complete, k] endif endfor endif if not exist(ss_complete) then goto, skip3a if n_elements(ss_complete) lt n_files_sot_set_min then begin print, ' Not enough files in this set. Skipping.' goto, skip3a endif index_sot_set = index_sot_set[ss_complete] cat_sot_set = cat_sot_set[ss_complete] files_sot_set = files_sot_set[ss_complete] n_files_sot_set = n_elements(files_sot_set) if keyword_set(do_print) then begin prstr, files_sot_set, /nomore endif x_binning = cat_sot_set[0].campsum if exist(bin_siz) then x_binning = x_binning*bin_siz y_binning = cat_sot_set[0].camssum if exist(bin_siz) then y_binning = y_binning*bin_siz sum_string = string(x_binning, format='$(i1.1)') + 'x' + $ string(y_binning, format='$(i1.1)') if ( (index_sot_set[0].fgccdix0 eq 0) and (index_sot_set[0].fgccdix1 eq 4095) ) then begin ccd_side = 'f' endif else begin if index_sot_set[0].fgccdix0 gt 2047 then ccd_side = 'l' else ccd_side = 'r' endelse PRINT, 'FGCCDIX0: ' + strtrim(index_sot_set[0].fgccdix0,2) PRINT, 'FGCCDIX1: ' + strtrim(index_sot_set[0].fgccdix1,2) if not exist(prefix_in) then $ prefix = 'sot_nfi_flat_' + wave + 'A_' + ccd_side + '_' + sum_string else $ prefix = prefix_in outfil_flat = prefix + '_' + time2file(cat_sot_set[0].date_obs) + '.fits' outdir_flat = concat_dir(dir_top, prefix + '_' + time2file(cat_sot_set[0].date_obs)) nfi_flat = fg_make_flat_chae(files_sot_set, outfil_flat, outdir_flat=outdir_flat, bfi=0, $ use_fg_prep=use_fg_prep, $ wave=wave, mask0=mask, bin_siz=bin_siz, object=object, c=c, $ maxiter=maxiter, min_frac=min_frac, max_error=max_error, $ do_print=do_print, notvshow=notvshow, $ do_make_top_html_files=do_make_top_html_files, $ do_make_flat_dir_html_files=do_make_flat_dir_html_files, $ do_make_only_current_flat_dir_html_file=do_make_only_current_flat_dir_html_file, $ _extra=_extra) endif SKIP3A: print, 'Bottom of tertiary loop (big difference sets).' endfor ss_set_begin = ss_waveid_median[ss_big_diff[n_big_diff-1]+1] ss_set_end = ss_waveid_median[n_waveid_median-1] ; ss_set = ss_waveid_median[ss_set_begin:ss_set_end] ss_set = ss_set_begin + lindgen(ss_set_end - ss_set_begin + 1) n_elem_set = n_elements(ss_set) if keyword_set(do_print) then begin fmt_timer, cat_sot_sub_clust[ss_set].date_obs ; prstr, strtrim(cat_sot_sub_clust[ss_set].naxis1,2) + ' ' + cat_sot_sub_clust[ss_set].wave print, 'BFI waveid_median count: ' + string(n_elem_set, format='$(i3.3)') endif ; Attempt to generate flat from this ss_set: if keyword_set(mk_flat) then begin cat_sot_set = cat_sot_sub_clust[ss_set] files_sot_set = files_sot_sub_clust[ss_set] index_sot_set = index_sot_sub_clust[ss_set] if keyword_set(do_print) then prstr, files_sot_set, /nomore n_files_sot_set = n_elements(files_sot_set) ; Check uniformity of image dimensions and that they are among the allowed set: if ( (n_elements(all_vals(cat_sot_set.waveid )) eq 1) and $ (n_elements(all_vals(cat_sot_set.naxis1 )) eq 1) and $ (n_elements(all_vals(cat_sot_set.naxis2 )) eq 1) and $ (n_elements(all_vals(index_sot_set.fgccdix1)) eq 1) and $ ( ( (cat_sot_set[0].naxis1 eq 2048) and $ (cat_sot_set[0].naxis2 eq 1024) ) or $ ( (cat_sot_set[0].naxis1 eq 4096) and $ (cat_sot_set[0].naxis2 eq 2048) ) ) ) ne 1 then begin print, ' Image dimensions not uniform or not among the allowed set. Skipping.' print, ' WAVEID: ', all_vals(cat_sot_set.waveid) print, ' NAXIS1: ', all_vals(cat_sot_set.naxis1) print, ' NAXIS2: ', all_vals(cat_sot_set.naxis2) print, ' FCCDIX1: ', all_vals(index_sot_set.fgccdix1) goto, skip3b endif ; Determine BFI filter position: nbfw_sot_set = index_sot_set.nbfw ss_6302 = where(((nbfw_sot_set ge nbfw_range_arr(0,0)) and $ (nbfw_sot_set le nbfw_range_arr(1,0))), n_6302) ss_5576 = where(((nbfw_sot_set ge nbfw_range_arr(0,1)) and $ (nbfw_sot_set le nbfw_range_arr(1,1))), n_5576) ss_5896 = where(((nbfw_sot_set ge nbfw_range_arr(0,2)) and $ (nbfw_sot_set le nbfw_range_arr(1,2))), n_5896) ss_5250 = where(((nbfw_sot_set ge nbfw_range_arr(0,3)) and $ (nbfw_sot_set le nbfw_range_arr(1,3))), n_5250) ss_5172 = where(((nbfw_sot_set ge nbfw_range_arr(0,4)) and $ (nbfw_sot_set le nbfw_range_arr(1,4))), n_5172) ss_6563 = where(((nbfw_sot_set ge nbfw_range_arr(0,5)) and $ (nbfw_sot_set le nbfw_range_arr(1,5))), n_6563) n_match_wave_arr = [n_6302, n_5576, n_5896, n_5250, n_5172, n_6563] max_match_wave = max(n_match_wave_arr, ss_match_wave) wave = wave_arr[ss_match_wave] ; TODO: Salvage case where not all nbfw values are the same: if n_match_wave_arr[ss_match_wave] ne n_elem_set then begin print, ' Not all NB filter wheel positions are the same for this set. Skipping.' ;STOP goto, skip3b endif pos_6302 = strpos(cat_sot_sub_clust[ss_set[0]-1].wave, wave_arr[0]) pos_5576 = strpos(cat_sot_sub_clust[ss_set[0]-1].wave, wave_arr[1]) pos_5896 = strpos(cat_sot_sub_clust[ss_set[0]-1].wave, wave_arr[2]) pos_5250 = strpos(cat_sot_sub_clust[ss_set[0]-1].wave, wave_arr[3]) pos_5172 = strpos(cat_sot_sub_clust[ss_set[0]-1].wave, wave_arr[4]) pos_6563 = strpos(cat_sot_sub_clust[ss_set[0]-1].wave, wave_arr[5]) pos_no_move = strpos(cat_sot_sub_clust[ss_set[0]-1].wave, wave_arr[6]) pos_arr = [pos_6302, pos_5576, pos_5896, pos_5250, pos_5172, pos_6563, pos_no_move] ss_match_wave_cat = where(pos_arr ne -1, n_match_wave_cat) print, CAT_SOT_SUB_CLUST.wave print, CAT_SOT_SUB_CLUST.waveid print, n_match_wave_arr STOP if n_match_wave_cat eq 0 then begin print, 'WAVE tag values: ', all_vals(cat_sot_sub_clust.wave) print, ' No match to cat_sot.wave. Stopping.' goto, skip3b endif if n_match_wave_cat gt 1 then begin print, 'WAVE tag values: ', all_vals(cat_sot_sub_clust.wave) print, ' No uniq match to cat_sot.wave. Stopping.' goto, skip3b endif wave_match_cat = wave_arr[ss_match_wave_cat] if ( (wave_match_cat ne wave) and (wave_match_cat ne 'no_move') ) then $ print, ' wave_match_cat does not equal wave. Fix this!' ; stop, ' wave_match_cat does not equal wave. Stopping.' prstr, [wave, wave_match_cat] ; Finally, eliminate incomplete images: delvarx, ss_complete ; if tag_exist(index_sot_set, 'percentd') then $ ; ss_complete = where(index_sot_set.percentd eq 100, n_complete) if not exist(ss_complete) then begin read_sot, files_sot_set, index_sot_test, data_sot_test ; data_sot_summed = total(total(data_sot_test,1),1) for k=0, n_elements(files_sot_set)-1 do begin ss_zero = where(data_sot_test[*,*,k] eq 0, n_zero) if n_zero le max_n_zero then begin if not exist(ss_complete) then ss_complete = k else $ ss_complete = [ss_complete, k] endif endfor endif if not exist(ss_complete) then goto, skip3b if n_elements(ss_complete) lt n_files_sot_set_min then begin print, ' Not enough files in this set. Skipping.' goto, skip3b endif index_sot_set = index_sot_set[ss_complete] cat_sot_set = cat_sot_set[ss_complete] files_sot_set = files_sot_set[ss_complete] n_files_sot_set = n_elements(files_sot_set) if keyword_set(do_print) then begin prstr, files_sot_set, /nomore endif x_binning = cat_sot_set[0].campsum if exist(bin_siz) then x_binning = x_binning*bin_siz y_binning = cat_sot_set[0].camssum if exist(bin_siz) then y_binning = y_binning*bin_siz sum_string = string(x_binning, format='$(i1.1)') + 'x' + $ string(y_binning, format='$(i1.1)') if ( (index_sot_set[0].fgccdix0 eq 0) and (index_sot_set[0].fgccdix1 eq 4095) ) then begin ccd_side = 'f' endif else begin if index_sot_set[0].fgccdix0 gt 2047 then ccd_side = 'l' else ccd_side = 'r' endelse PRINT, 'FGCCDIX0: ' + strtrim(index_sot_set[0].fgccdix0,2) PRINT, 'FGCCDIX1: ' + strtrim(index_sot_set[0].fgccdix1,2) if not exist(prefix_in) then $ prefix = 'sot_nfi_flat_' + wave + 'A_' + ccd_side + '_' + sum_string else $ prefix = prefix_in outfil_flat = prefix + '_' + time2file(cat_sot_set[0].date_obs) + '.fits' outdir_flat = concat_dir(dir_top, prefix + '_' + time2file(cat_sot_set[0].date_obs)) nfi_flat = fg_make_flat_chae(files_sot_set, outfil_flat, outdir_flat=outdir_flat, bfi=0, $ use_fg_prep=use_fg_prep, $ wave=wave, mask0=mask, bin_siz=bin_siz, object=object, c=c, $ maxiter=maxiter, min_frac=min_frac, max_error=max_error, $ do_print=do_print, notvshow=notvshow, $ do_make_top_html_files=do_make_top_html_files, $ do_make_flat_dir_html_files=do_make_flat_dir_html_files, $ do_make_only_current_flat_dir_html_file=do_make_only_current_flat_dir_html_file, $ _extra=_extra) endif endif else begin ss_set = ss_waveid_median n_elem_set = n_elements(ss_set) if keyword_set(do_print) then begin fmt_timer, cat_sot_sub_clust[ss_set].date_obs ; prstr, strtrim(cat_sot_sub_clust[ss_set].naxis1,2) + ' ' + cat_sot_sub_clust[ss_set].wave print, 'BFI waveid_median count: ' + string(n_elem_set, format='$(i3.3)') endif ; Attempt to generate flat from this ss_set: if keyword_set(mk_flat) then begin cat_sot_set = cat_sot_sub_clust[ss_set] files_sot_set = files_sot_sub_clust[ss_set] index_sot_set = index_sot_sub_clust[ss_set] if keyword_set(do_print) then prstr, files_sot_set, /nomore n_files_sot_set = n_elements(files_sot_set) ; Check uniformity of image dimensions and that they are among the allowed set: if ( (n_elements(all_vals(cat_sot_set.waveid )) eq 1) and $ (n_elements(all_vals(cat_sot_set.naxis1 )) eq 1) and $ (n_elements(all_vals(cat_sot_set.naxis2 )) eq 1) and $ (n_elements(all_vals(index_sot_set.fgccdix1)) eq 1) and $ ( ( (cat_sot_set[0].naxis1 eq 2048) and $ (cat_sot_set[0].naxis2 eq 1024) ) or $ ( (cat_sot_set[0].naxis1 eq 4096) and $ (cat_sot_set[0].naxis2 eq 2048) ) ) ) ne 1 then begin print, ' Image dimensions not uniform or not among the allowed set. Skipping.' print, ' WAVEID: ', all_vals(cat_sot_set.waveid) print, ' NAXIS1: ', all_vals(cat_sot_set.naxis1) print, ' NAXIS2: ', all_vals(cat_sot_set.naxis2) print, ' FCCDIX1: ', all_vals(index_sot_set.fgccdix1) goto, skip3b endif ; Determine BFI filter position: nbfw_sot_set = index_sot_set.nbfw ss_6302 = where(((nbfw_sot_set ge nbfw_range_arr(0,0)) and $ (nbfw_sot_set le nbfw_range_arr(1,0))), n_6302) ss_5576 = where(((nbfw_sot_set ge nbfw_range_arr(0,1)) and $ (nbfw_sot_set le nbfw_range_arr(1,1))), n_5576) ss_5896 = where(((nbfw_sot_set ge nbfw_range_arr(0,2)) and $ (nbfw_sot_set le nbfw_range_arr(1,2))), n_5896) ss_5250 = where(((nbfw_sot_set ge nbfw_range_arr(0,3)) and $ (nbfw_sot_set le nbfw_range_arr(1,3))), n_5250) ss_5172 = where(((nbfw_sot_set ge nbfw_range_arr(0,4)) and $ (nbfw_sot_set le nbfw_range_arr(1,4))), n_5172) ss_6563 = where(((nbfw_sot_set ge nbfw_range_arr(0,5)) and $ (nbfw_sot_set le nbfw_range_arr(1,5))), n_6563) n_match_wave_arr = [n_6302, n_5576, n_5896, n_5250, n_5172, n_6563] max_match_wave = max(n_match_wave_arr, ss_match_wave) wave = wave_arr[ss_match_wave] ; TODO: Salvage case where not all nbfw values are the same: if n_match_wave_arr[ss_match_wave] ne n_elem_set then begin print, ' Not all NB filter wheel positions are the same for this set. Skipping.' ;STOP goto, skip3b endif pos_6302 = strpos(cat_sot_sub_clust[ss_set[0]-1].wave, wave_arr[0]) pos_5576 = strpos(cat_sot_sub_clust[ss_set[0]-1].wave, wave_arr[1]) pos_5896 = strpos(cat_sot_sub_clust[ss_set[0]-1].wave, wave_arr[2]) pos_5250 = strpos(cat_sot_sub_clust[ss_set[0]-1].wave, wave_arr[3]) pos_5172 = strpos(cat_sot_sub_clust[ss_set[0]-1].wave, wave_arr[4]) pos_6563 = strpos(cat_sot_sub_clust[ss_set[0]-1].wave, wave_arr[5]) pos_no_move = strpos(cat_sot_sub_clust[ss_set[0]-1].wave, wave_arr[6]) pos_arr = [pos_6302, pos_5576, pos_5896, pos_5250, pos_5172, pos_6563, pos_no_move] ss_match_wave_cat = where(pos_arr ne -1, n_match_wave_cat) print, CAT_SOT_SUB_CLUST.wave print, CAT_SOT_SUB_CLUST.waveid print, n_match_wave_arr ;STOP if n_match_wave_cat eq 0 then begin print, 'WAVE tag values: ', all_vals(cat_sot_sub_clust.wave) print, ' No match to cat_sot.wave. Stopping.' goto, skip3b endif if n_match_wave_cat gt 1 then begin print, 'WAVE tag values: ', all_vals(cat_sot_sub_clust.wave) print, ' No uniq match to cat_sot.wave. Stopping.' goto, skip3b endif wave_match_cat = wave_arr[ss_match_wave_cat] if ( (wave_match_cat ne wave) and (wave_match_cat ne 'no_move') ) then $ print, ' wave_match_cat does not equal wave. Fix this!' ; stop, ' wave_match_cat does not equal wave. Stopping.' prstr, [wave, wave_match_cat] ; Finally, eliminate incomplete images: delvarx, ss_complete ; if tag_exist(index_sot_set, 'percentd') then $ ; ss_complete = where(index_sot_set.percentd eq 100, n_complete) if not exist(ss_complete) then begin read_sot, files_sot_set, index_sot_test, data_sot_test ; data_sot_summed = total(total(data_sot_test,1),1) for k=0, n_elements(files_sot_set)-1 do begin ss_zero = where(data_sot_test[*,*,k] eq 0, n_zero) if n_zero le max_n_zero then begin if not exist(ss_complete) then ss_complete = k else $ ss_complete = [ss_complete, k] endif endfor endif if not exist(ss_complete) then goto, skip3b if n_elements(ss_complete) lt n_files_sot_set_min then begin print, ' Not enough files in this set. Skipping.' goto, skip3b endif index_sot_set = index_sot_set[ss_complete] cat_sot_set = cat_sot_set[ss_complete] files_sot_set = files_sot_set[ss_complete] n_files_sot_set = n_elements(files_sot_set) if keyword_set(do_print) then begin prstr, files_sot_set, /nomore endif x_binning = cat_sot_set[0].campsum if exist(bin_siz) then x_binning = x_binning*bin_siz y_binning = cat_sot_set[0].camssum if exist(bin_siz) then y_binning = y_binning*bin_siz sum_string = string(x_binning, format='$(i1.1)') + 'x' + $ string(y_binning, format='$(i1.1)') if ( (index_sot_set[0].fgccdix0 eq 0) and (index_sot_set[0].fgccdix1 eq 4095) ) then begin ccd_side = 'f' endif else begin if index_sot_set[0].fgccdix0 gt 2047 then ccd_side = 'l' else ccd_side = 'r' endelse PRINT, 'FGCCDIX0: ' + strtrim(index_sot_set[0].fgccdix0,2) PRINT, 'FGCCDIX1: ' + strtrim(index_sot_set[0].fgccdix1,2) if not exist(prefix_in) then $ prefix = 'sot_nfi_flat_' + wave + 'A_' + ccd_side + '_' + sum_string else $ prefix = prefix_in outfil_flat = prefix + '_' + time2file(cat_sot_set[0].date_obs) + '.fits' outdir_flat = concat_dir(dir_top, prefix + '_' + time2file(cat_sot_set[0].date_obs)) nfi_flat = fg_make_flat_chae(files_sot_set, outfil_flat, outdir_flat=outdir_flat, bfi=0, $ use_fg_prep=use_fg_prep, $ wave=wave, mask0=mask, bin_siz=bin_siz, object=object, c=c, $ maxiter=maxiter, min_frac=min_frac, max_error=max_error, $ do_print=do_print, notvshow=notvshow, $ do_make_top_html_files=do_make_top_html_files, $ do_make_flat_dir_html_files=do_make_flat_dir_html_files, $ do_make_only_current_flat_dir_html_file=do_make_only_current_flat_dir_html_file, $ _extra=_extra) endif endelse if keyword_set(do_read) then $ mreadfits_header, files_sot, index_sot ; if keyword_set(do_print) then begin ; fmt_timer, cat_sot.date_obs ; prstr, strtrim(cat_sot.naxis1,2) + ' ' + cat_sot.wave ; print, 'BFI waveid_median count: ' + string(n_waveid_median, format='$(i3.3)') ; endif SKIP3B: print, 'Bottom of big difference sets last case.' SKIP2: print, 'Bottom of secondary loop (sub-clusters).' endfor ; end of sub cluster loop endif else begin print, 'No SOT catalog records found for this time interval.' endelse ; STOP SKIP1: print, 'Bottom of primary loop (clusters).' endfor end