pro sp_madness_data_ops, t0, t1, cat_dir=cat_dir, struct_sp=struct_sp, $ tim_arr=tim_arr, median_arr=median_arr, $ ysiz_vals=ysiz_vals, ysiz_arr=ysiz_arr, level0=level0, nfil_max=nfil_max, pnt_lim=pnt_lim, $ do_prep=do_prep, save_old=save_old, quiet=quiet ; TODO: ; - Enable inclusion of binned and summed images if not exist(interval1) then interval1 = 1 ; Days if not exist(level0) then level0 = 1 if not exist(nfil_max) then nfil_max = 10 ; 100 if not exist(pnt_lim) then pnt_lim = 600 pnt_lim_str = strtrim(pnt_lim,2) if not exist(tags_keep) then $ tags_keep = ['date_obs','naxis1','naxis2','xcen','ycen','spnint','spbshft','camdaca','camdacb','camamp', $ 'camgain','campsum','camssum','t_spccd','t_spceb','scn_sum'] ;dark_vals = [0d0, 0d0, 0d0, 0d0, 0d0, 135.4, 221.1, 307.0, 394.0] dark_vals = [0d0, 0d0, 0d0, 0d0, 0d0, 135.4, 221.1, 307.0, 394.0, $ 0d0, 0d0, 0d0, 0d0, 0d0, 0d0, 0d0, 0d0, 0d0, 0d0, 0d0, $ 0d0, 0d0, 0d0, 0d0, 0d0, 0d0, 0d0, 0d0, 0d0, 0d0, $ 0d0, 0d0, 0d0, 0d0, 0d0, 0d0, 0d0, 0d0, 0d0, 0d0, $ 0d0, 0d0, 0d0, 0d0, 0d0, 0d0, 0d0, 0d0, 0d0, 0d0, $ 0d0, 0d0, 0d0, 0d0, 0d0, 0d0, 0d0, 0d0, 0d0, 0d0, $ 0d0, 0d0, 0d0, 0d0, 0d0, 0d0, 0d0, 0d0, 0d0, 0d0, $ 0d0, 0d0, 0d0, 0d0, 0d0, 0d0, 0d0, 0d0, 0d0, 0d0, $ 0d0, 0d0, 0d0, 0d0, 0d0, 0d0, 0d0, 0d0, 0d0, 0d0, $ 0d0, 0d0, 0d0, 0d0, 0d0, 0d0, 0d0, 0d0, 0d0, 0d0] bad_dates = ['15-mar-2007', '10-sep-2007', '29-feb-2008', '22-sep-2008', '23-dec-2021'] bad_dates_sec = anytim(bad_dates) if level0 eq 1 then set_logenv,'SOT_DATA','/archive1/hinode/sot/level0' else $ set_logenv,'SOT_DATA','/archive1/hinode/sot/QuickLook' ;filnam_sp_sav = '~/soft/idl/idl_startup/sp_throughput_mission.sav' ;filnam_sp_sav_backup = '~/soft/idl/idl_startup/sp_throughput_mission.sav_' + time2file(!stime) ;filnam_sp_sav = '/sanhome/slater/public_html/sot/instrument/mechanisms_data_ops/sp_throughput_mission.sav' ;filnam_sp_sav_backup = '/sanhome/slater/public_html/sot/instrument/mechanisms_data_ops/sp_throughput_mission.sav_' + $ ; time2file(!stime) filnam_sp_sav = '/archive1/hinode/sot/instrument/mechanisms_data_ops/sp_throughput_mission.sav' filnam_sp_sav_backup = '/archive1/hinode/sot/instrument/mechanisms_data_ops/sp_throughput_mission.sav_' + $ time2file(!stime) if file_exist(filnam_sp_sav) then begin if keyword_set(save_old) then $ spawn, 'cp -p ' + filnam_sp_sav + ' ' + filnam_sp_sav_backup restore, filnam_sp_sav endif if not exist(t0) then begin if exist(struct_sp) then begin t_sec = anytim(struct_sp.date_obs) t_sort = t_sec[sort(t_sec)] t0 = anytim(t_sort[n_elements(t_sort)-1],/yoh) endif else begin t0 = '16-oct-2006' endelse endif if not exist(t1) then t1 = anytim(!stime,/yoh) t_big_grid = anytim(timegrid(t0,t1,days=interval1),/yoh) np_big_grid = n_elements(t_big_grid) for i=0, np_big_grid-2 do begin delvarx, files_sp, index_sp_samp, data_sp_samp nfil_samp=0 sttim_loop = !stime search_array = ['obs_type=SP*IQUV*4D*array', $ 'xcen=-'+pnt_lim_str+'~'+pnt_lim_str, $ 'ycen=-'+pnt_lim_str+'~'+pnt_lim_str, $ 'slitpos=-300~300'] sot_cat, t_big_grid(i), t_big_grid(i+1), cat_sp, files_sp, level0=level0, $ search_array=search_array, topdir=cat_dir if exist(files_sp) then begin print, ' Range of naxis1 values = ' print, all_vals(cat_sp.naxis1) print, ' Range of naxis2 values = ' print, all_vals(cat_sp.naxis2) exist_arr = file_exist(files_sp) ss_exist = where(exist_arr eq 1, n_exist) if n_exist gt 0 then begin files_sp = files_sp[ss_exist] cat_sp = cat_sp[ss_exist] n_times = n_elements(cat_sp) ss_good_arr = lonarr(n_times) + 1l t_sec = anytim(cat_sp.date_obs) t_date = long(t_sec) mod 86400l ss_bad_times = wc_where_arr(t_date, bad_dates, n_bad_times) if n_bad_times gt 0 then begin ss_good_arr[ss_bad_times] = 0l ss_good_times = where(ss_good_arr eq 1, n_good_times) files_sp = files_sp[ss_good_times] cat_sp = cat_sp[ss_good_times] endif ; Figure out which naxis2 to use: siz_vec = [1024, 816, 512, 384, 256] ss_1024 = where(cat_sp.naxis2 eq 1024, n_1024) ss_816 = where(cat_sp.naxis2 eq 816, n_816) ss_512 = where(cat_sp.naxis2 eq 512, n_512) ss_384 = where(cat_sp.naxis2 eq 384, n_384) ss_256 = where(cat_sp.naxis2 eq 256, n_256) num_vec = long([n_1024, n_816, n_512, n_384, n_256]) if total(num_vec) ne 0 then begin flag_vec = num_vec/nfil_max ysiz_sel = -1 if flag_vec[0] gt 0 then ysiz_sel = 1024 else $ if flag_vec[1] gt 0 then ysiz_sel = 816 else $ if flag_vec[2] gt 0 then ysiz_sel = 512 else $ if flag_vec[3] gt 0 then ysiz_sel = 384 else $ if flag_vec[4] gt 0 then ysiz_sel = 256 if ysiz_sel eq -1 then begin max_num = max(num_vec, ss_max_num) ysiz_sel = siz_vec[ss_max_num] endif case ysiz_sel of 1024: ss_sel = ss_1024 816 : ss_sel = ss_816 512 : ss_sel = ss_512 384 : ss_sel = ss_384 256 : ss_sel = ss_256 endcase files_sp = files_sp[ss_sel] cat_sp = cat_sp[ss_sel] nfil_sp = n_elements(files_sp) nfil_samp = min([nfil_sp, nfil_max]) ind_arr = (lindgen(nfil_samp)*nfil_sp)/nfil_samp ; Read all indices first, and remove files which have naxis3 gt 1: files_sp_samp = files_sp[ind_arr] read_sot, files_sp_samp, index_sp_samp ss_good_naxis3 = where(index_sp_samp.naxis3 eq 1, n_good_naxis3) if n_good_naxis3 gt 0 then begin files_sp_samp = files_sp_samp[ss_good_naxis3] nfil_samp = n_elements(files_sp_samp) ; Now read the data: read_sot, files_sp_samp, index_sp_samp, data_sp_samp sot_iquv_3d, index_sp_samp, data_sp_samp, oindex_sp_samp, odata_sp_samp, /i_only, /noscale if exist(odata_sp_samp) then begin odata_sp_samp = uint(odata_sp_samp) for j=0,nfil_samp-1 do begin index0 = index_sp_samp[j] data0 = reform(odata_sp_samp[*,*,j]) ; Skip if array has any zero values: if min(data0) gt 0 then begin ; Otherwise process: data0 = data0 * (index0.spbshft+1d0) / index0.scn_sum / (index0.spnint*8d0) data0 = ( data0 - dark_vals[index0.camdaca] ) / index0.camssum sz_data0 = size(data0) roi_samp0 = data0[4:sz_data0[1]-4, 4:sz_data0[2]-4] tim0 = index0.date_obs mean0 = mean(roi_samp0) median0 = median(roi_samp0) ysiz0 = index0.naxis2 struct_sp0 = str_subset(index0, tags_keep) struct_sp0 = add_tag(struct_sp0, mean0, 'mean_sp') struct_sp0 = add_tag(struct_sp0, median0, 'median_sp') str_tim_arr = strtrim(tim0,2) str_med_arr = string(median0, format='$(d10.1)') str_rows = str_tim_arr + str_med_arr ; if not file_exist('~/soft/idl/idl_startup/sp_tp_tab.txt') then $ ; file_append, '~/soft/idl/idl_startup/sp_tp_tab.txt', str_rows, /newfile else $ ; file_append, '~/soft/idl/idl_startup/sp_tp_tab.txt', str_rows if not exist(struct_sp) then begin struct_sp = struct_sp0 ; tim_arr = tim0 ; median_arr = median0 ; ysiz_arr = ysiz0 endif else begin struct_sp = concat_struct(struct_sp, struct_sp0) ; tim_arr = [tim_arr, tim0] ; median_arr = [median_arr, median0] ; ysiz_arr = [ysiz_arr, ysiz0] endelse endif endfor endif endif endif else begin nfil_samp = 0 if not exist(ysiz_vals) then ysiz_vals = all_vals(cat_sp.naxis2) else $ ysiz_vals = [ysiz_vals, all_vals(cat_sp.naxis2)] endelse endif endif else begin nfil_samp = 0 endelse entim_loop = !stime tim_loop = addtime(entim_loop, diff=sttim_loop, /sec) if nfil_samp gt 0 then begin print, 'Finished for ' + strmid(t_big_grid(i),0,9) + '. ' + $ strtrim(tim_loop,2) + ' sec total. ' + $ string(nfil_samp,format='$(i5.5)') + ' files. ' + $ strtrim(float(tim_loop)/nfil_samp,2) + ' sec per file.' endif else begin print, 'Finished for ' + strmid(t_big_grid(i),0,9) + '. No matches found.' endelse endfor save, struct_sp, file=filnam_sp_sav chmod, filnam_sp_sav, '0664'o end