pro son_of_go_sot_cruiser, t0, t1, obs_arr=obs_arr, hcr_arr=hcr_arr, $ sloop=sloop, eloop=eloop, $ hinode_required=hinode_required, dirnam_data=dirnam_data, $ n_samp_max=n_samp_max, $ no_links=no_links, symlink_parent=symlink_parent, $ recurs=recurs, remove_old=remove_old, dir_parent_cruiser=dir_parent_cruiser, $ no_mk_dir=no_mk_dir, $ no_skip=no_skip, no_drotate=no_drotate, fast=fast, do_log=do_log, $ do_write=do_write, do_test=do_test, file_type=file_type, $ do_color=do_color, quality=quality, do_xstep=do_xstep, $ _extra=_extra if ( exist(t0) and (not exist(t1)) ) then t1 = reltime(t0, days=31, out='ccsds') if not exist(n_samp_max) then n_samp_max = 10000 if not exist(no_drotate) then no_drotate = 0 if not exist(quality) then quality = 70 if not exist(do_log) then do_log = 1 if not exist(log_pipeline) then $ log_pipeline = concat_dir('/sanhome/slater/logs/log_pipeline_cruiser' if not exist(log_fits_read) then log_fits_read = '/sanhome/slater/logs/log_fits_read_cruiser_01.txt' if not exist(file_type) then file_type = 'jpeg' case file_type of 'jpeg': file_suffix = 'jpg' 'gif': file_suffix = 'gif' else: file_suffix = 'jpg' endcase ; if t0, t1 defined then find all IRIS obs in [t0,t1] interval: if (exist(t0) and exist(t1)) then hcr = iris_time2hcr(t0, t1, /struct) if exist(obs_arr) then hcr = iris_obs2hcr(obs_arr) if exist(hcr_arr) then hcr = hcr_arr if not exist(hcr) then begin STOP, 'None of "t0/t1", "obs_arr", "hcr_arr" passed. Stopping.' ; Returning' ; return, -1 endif obsid_full = hcr.full_obsid obsid = obsid_full n_obs = n_elements(hcr) ; ============================================= ; Main loop: Process each IRIS obs, one by one: ; ============================================= if n_obs gt 0 then begin if ~exist(sloop) then sloop = 0 if ~exist(eloop) then eloop = n_obs-1 for i=sloop, eloop do begin sttim_obs = anytim(!stime, /ccsds) ; For loop execution time logging t_obs_ex = anytim(file2time(obsid[i]), /ex) yr_obs = string(t_obs_ex[6], '$(i4.4)') mo_obs = string(t_obs_ex[5], '$(i2.2)') dom_obs = string(t_obs_ex[4], '$(i2.2)') ; ************************************************************************ ; Define input and output directories and url for this obs, and create the ; output cruiser data directory: ; ************************************************************************ ; Define IRIS obs directory for input level2 FITS files: dir_obs_in = '/irisa/data/level2' + $ '/' + yr_obs + '/' + mo_obs + '/' + dom_obs + '/' + obsid[i] if ~exist(dir_parent_cruiser) then $ dir_parent_cruiser_remote = '/sanhome/slater/public_html/cruiser' if ~exist(dirnam_data) then begin if ~keyword_set(do_color) then dirnam_data = 'data' else dirnam_data = 'data_color' endif dir_data = concat_dir(dir_parent_cruiser, concat_dir('observatory/iris_hinode/' + dirnam_data, obsid[i])) url_base = concat_dir('observatory/iris_hinode/' + dirnam_data, obsid[i]) ; Optionally remove previous cruiser directory for this obs, and re-make an empty directory: if not exist(remove_old) then remove_old = 0 if keyword_set(remove_old) then begin print, ' **********************' print, ' Now Removing directory ' + dir_data print, ' **********************' spawn, 'rm -rf ' + dir_data ; + '*' endif ; *************************************************** ; END DIRECTORY DEFINITION/CREATION CODE FOR THIS OBS ; *************************************************** ; **************************************************************** ; BEGIN SECTION TO DETERMINE AVAILABILITY OF HINODE AND AIA IMAGES ; MATCHING THIS IRIS OBS. NOTE THAT THESE CRUISER MOVIES ARE FOR ; HINODE SOT, SO THE IRIS OBS IS SKIPPED IF THERE ARE NO MATCHING ; HINODE IMAGES: ; **************************************************************** ; Find all instruments with FITS cubes for this IRIS obs: ; IRIS files: dir_iris = dir_obs_in files_iris = file_list(dir_iris, 'iris_l2_*SJI*.fits') if files_iris[0] ne '' then $ n_files_iris = n_elements(files_iris) else n_files_iris = 0 ; If no IRIS cubes then skip this obs: if n_files_iris eq 0 then begin print, ' No IRIS images found for this obs.' + $ ' Skipping to AIA section for this obs.' do_skip = 1 waves_iris = '' n_waves_iris = 0 n_img_arr_iris = 0 GOTO, SKIP_IRIS endif ; AIA files: dir_aia = concat_dir(dir_obs_in, 'aia') files_aia = file_list(dir_aia, 'aia_l2_*.fits') if files_aia[0] ne '' then $ n_files_aia = n_elements(files_aia) else n_files_aia = 0 ; SOT files: dir_sot = concat_dir(dir_obs_in, 'hinode') files_sot = file_list(dir_sot, 'sot_l2_*.fits') if files_sot[0] ne '' then begin n_files_sot = n_elements(files_sot) endif else begin if ~keyword_set(hinode_required) then begin n_files_sot = 0 endif else begin print, ' Hinode required flag set and no hinode images found' + $ ' for this obs. Skipping this obs.' do_skip = 1 GOTO, SKIPCASE endelse endelse ; ******************************************** ; END HINODE, AIA MATCHING FILE CHECK SECTION: ; ******************************************** ; Create output cruiser dir for this obs: print, ' Creating directory ' + dir_data mk_dir, dir_data file_json = concat_dir(dir_data, 'cruiser.json') ; Choose which instrument will be used as the reference for alignment. ; (default is IRIS): if not exist(instr_ref) then instr_ref = 'iris' ; Determine all IRIS waves present: pos_sji = strpos(files_iris[0], 'SJI') waves_iris = strmid(files_iris, pos_sji+4, 4) n_waves_iris = n_elements(waves_iris) ; Read in headers of all IRIS cubes to determine highest cadence wave, for use as ; the reference wave: for j=0, n_files_iris-1 do begin read_iris_l2, files_iris[j], index_iris0, /nodata ss_bad0 = where(index_iris0.exptimes eq 0, n_bad0) ss_good0 = where(index_iris0.exptimes ne 0, n_good0) if n_good0 gt 0 then begin index_iris0 = index_iris0[ss_good0] if j eq 0 then begin n_bad_tot = n_bad0 ss_good_img_arr_iris = ss_good0 n_img_arr_iris = n_elements(index_iris0) endif else begin n_bad_tot = n_bad_tot + n_bad0 ss_good_img_arr_iris = [ss_good_img_arr_iris, ss_good0] n_img_arr_iris = [n_img_arr_iris, n_elements(index_iris0)] endelse endif else begin print, ' No good IRIS images found for IRIS wave ' + waves_iris[j] + '.' + $ ' Skipping to AIA section for this obs.' waves_iris = '' n_waves_iris = 0 n_img_arr_iris = 0 do_skip = 1 GOTO, SKIP_IRIS ; If no good IRIS images found endelse endfor ; Use highest cadence IRIS wave for reference: n_img_max_iris = max(n_img_arr_iris, ss_n_img_max_iris) wave_ref_iris = waves_iris[ss_n_img_max_iris] ; Use the times of the IRIS reference wave to define a (possibly sub-sampled) array ; of reference times to which all waves of all instruments will be matched: read_iris_l2, files_iris[ss_n_img_max_iris], index_iris_ref, /nodata ss_good_ref = where(index_iris_ref.exptimes ne 0, n_good_ref) index_iris_ref = index_iris_ref[ss_good_ref] t_ref_arr = anytim(index_iris_ref.date_obs, /ccsds) t_ref_arr_int = anytim(t_ref_arr, /int) t_ref_arr_sec = anytim(t_ref_arr) t0_ref = min(t_ref_arr) t1_ref = max(t_ref_arr) t_span_sec_ref = anytim(t1_ref) - anytim(t0_ref) ; If necessary, sub-sample the reference time array to have no more elements than ; the maximum allowed: if n_img_max_iris gt n_samp_max then begin t_grid_ref = timegrid(t0_ref, t1_ref, nsamp = n_samp_max) ss_samp_ref = tim2dset(t_ref_arr_int, t_grid_ref) ; ss_samp_ref = grid_data(t_ref_arr_int, nsamp=n_samp_max, /not_uniq) t_samp_ref = t_ref_arr_int[ss_samp_ref] endif else begin ss_samp_ref = lindgen(n_img_max_iris) t_samp_ref = t_ref_arr_int endelse n_samp = n_elements(t_samp_ref) t_samp_ref_int = anytim(t_samp_ref, /int) fovx_ref = index_iris_ref[0].fovx fovy_ref = index_iris_ref[0].fovy t0_read_sec = anytim(!stime) read_iris_l2, files_iris[ss_n_img_max_iris], index0, data0 ; This section logs the FITS read time for the IRIS image sample set: t1_read_sec = anytim(!stime) n_img_read = n_elements(index0) tsec_read_per_img = double(t1_read_sec - t0_read_sec) / n_img_read t_read = anytim(!stime, /ccsds) buff_read = t_read spawn, 'uname -n', node buff_read = buff_read + ' node: ' + strtrim(node,2) buff_read = buff_read + ' IRIS L2 FITS cube read time per image = ' + $ string(tsec_read_per_img, '$(f6.3)') + ' seconds.' buff_read = buff_read + $ ' Total read time for ' + string(n_img_read, '$(i5.5)') + ' images = ' + $ string(double(t1_read_sec - t0_read_sec), '$(f6.3)') + ' seconds.' if not keyword_set(no_logging) then file_append, log_fits_read, buff_read prstr, buff_read ; End of IRIS L2 FITS read time section index0 = index0[ss_samp_ref] data0 = data0[*,*,ss_samp_ref] if n_elements(index0) gt 1 then data0 = iris_dustbuster(index0, temporary(data0)) data0 = iris_intscale(temporary(data0), index0) ; , /limb) data0 = bytscl(data0) ; Create a map array from the reference index0 and data0 array: index2map, index0, data0, map_ref wave0 = waves_iris[ss_n_img_max_iris] n_files0 = n_elements(index0) if not exist(n_img_arr) then n_img_arr = n_files0 else n_img_arr = [n_img_arr, n_files0] file_times0 = index0.date_obs files_out0 = 'sdo_iris_' + string(wave0, '(i4.4)') + 'A_' + time2file(file_times0, /sec) + $ '.' + file_suffix files_out0 = concat_dir(dir_data, files_out0) ; Write out IRIS images for reference wave in desired format (jpeg, gif, etc): if keyword_set(do_write) then begin ; Create directory for this obsid, if it doesn't already exist: if ~keyword_set(no_mk_dir) then begin if ~file_exist(dir_data) then begin mk_dir, dir_data print, ' **********************' print, ' Creating directory ' + dir_data print, ' **********************' endif endif iris_lct, index0[0], r0, g0, b0, /noload for k=0,n_files0-1 do begin case file_type of 'jpeg': begin data0_single = reform(data0[*,*,k]) if keyword_set(do_color) then begin datar = r0[data0_single] & datag = g0[data0_single] & datab = b0[data0_single] ; datat = [[[datar]],[[datag]],[[datab]]] write_jpeg, files_out0[k], truecolor(data0_single, r0, g0, b0), true=1, quality=quality endif else begin write_jpeg, files_out0[k], data0_single, quality=quality endelse end 'gif': write_gif, files_out0[k], data0_single, r0, g0, b0 else: write_jpeg, files_out0[k], data0_single, quality=quality endcase endfor endif ; Add output file names for this wave to array of file names for all waves: if not exist(files_out_arr) then files_out_arr = files_out0 else $ files_out_arr = [files_out_arr, files_out0] ; Delete the last index and data arrays, and working variables, before processing another wave: delvarx, files_out0, index0, data0 print, 'Completed IRIS wave ' + wave0 ;================================= ; Now process remaining IRIS waves: ;================================= ;HELP, n_img_arr, n_img_arr_iris, n_img_arr_aia, n_img_arr_sot, $ ; n_img_max_iris, n_img_max_aia, n_img_max_sot, $ ; instr_arr, n_waves_iris, n_waves_aia, n_waves_sot, $ ; waves_arr, waves_iris, waves_aia, waves_sot ss_waves_iris_non_ref = where(indgen(n_waves_iris) ne ss_n_img_max_iris, n_waves_iris_non_ref) for j=0, n_waves_iris_non_ref-1 do begin t0_read_sec = anytim(!stime) read_iris_l2, files_iris[ss_waves_iris_non_ref[j]], index0, data0 ; ss_good0 = where(index0.exptimes ne 0, n_good0) ; index0 = index0[ss_good0] ; index0 = index0[ss_good_img_arr_iris[ss_waves_iris_non_ref[j]]] t1_read_sec = anytim(!stime) n_img_read = n_elements(index0) tsec_read_per_img = double(t1_read_sec - t0_read_sec) / n_img_read buff_read = ' IRIS L2 FITS cube read time per image = ' + $ string(tsec_read_per_img, '$(f6.3)') + ' seconds.' buff_read = [buff_read, $ ' Total read time for ' + string(n_img_read, '$(i5.5)') + ' images = ' + $ string(double(t1_read_sec - t0_read_sec), '$(f6.3)') + ' seconds.'] if not keyword_set(no_logging) then file_append, log_fits_read, buff_read prstr, buff_read t_wave0 = anytim(index0.date_obs, /ccsds) ss_samp_wave0 = tim2dset(anytim(t_wave0, /int), t_samp_ref) ; ss_samp_wave0 = grid_data(t_wave0, nsamp=n_samp_ref, /not_uniq) index0 = index0[ss_samp_wave0] data0 = data0[*,*,ss_samp_wave0] wave0 = waves_iris[ss_waves_iris_non_ref[j]] t_wave0 = anytim(index0.date_obs, /ccsds) n_files0 = n_elements(index0) if not exist(n_img_arr) then n_img_arr = n_files0 else n_img_arr = [n_img_arr, n_files0] if n_elements(index0) gt 1 then data0 = iris_dustbuster(index0, temporary(data0)) data0 = iris_intscale(temporary(data0), index0) ; , /limb) data0 = bytscl(data0) file_times0 = index0.date_obs files_out0 = 'sdo_iris_' + string(wave0, '(i4.4)') + 'A_' + time2file(file_times0, /sec) + $ '.' + file_suffix files_out0 = concat_dir(dir_data, files_out0) ; Write out IRIS images for this wave in desired format (jpeg, gif, etc): if keyword_set(do_write) then begin ; Create directory for this obsid, if it doesn't already exist: if ~keyword_set(no_mk_dir) then begin if ~file_exist(dir_data) then begin mk_dir, dir_data print, ' **********************' print, ' Creating directory ' + dir_data print, ' **********************' endif endif iris_lct, index0[0], r0, g0, b0, /noload print, 'Wave = ' + string(wave0, '(i4.4)') + ' .' help, n_files0 for k=0,n_files0-1 do begin case file_type of 'jpeg': begin data0_single = reform(data0[*,*,k]) if keyword_set(do_color) then begin datar = r0[data0_single] & datag = g0[data0_single] & datab = b0[data0_single] ; datat = [[[datar]],[[datag]],[[datab]]] write_jpeg, files_out0[k], truecolor(data0_single, r0, g0, b0), true=1, quality=quality endif else begin write_jpeg, files_out0[k], data0_single, quality=quality endelse end 'gif': write_gif, files_out0[k], data0_single, r0, g0, b0 else: write_jpeg, files_out0[k], data0_single, quality=quality endcase endfor endif ; Add output file names for this wave to array of file names for all waves: if not exist(files_out_arr) then files_out_arr = files_out0 else $ files_out_arr = [files_out_arr, files_out0] ; Delete the last index and data arrays, and working variables, before processing another wave: delvarx, files_out0, index0, data0 print, 'Completed IRIS wave ' + wave0 endfor SKIP_IRIS: ;====================== ; Now process AIA waves: ;====================== ;HELP, n_img_arr, n_img_arr_iris, n_img_arr_aia, n_img_arr_sot, $ ; n_img_max_iris, n_img_max_aia, n_img_max_sot, $ ; instr_arr, n_waves_iris, n_waves_aia, n_waves_sot, $ ; waves_arr, waves_iris, waves_aia, waves_sot if n_files_aia gt 0 then begin ; Determine all AIA waves present: filnams_aia = file_break(files_aia) pos_us = strpos(filnams_aia, '_', /reverse_search) pos_dot = strpos(filnams_aia, '.', /reverse_search) slen = pos_dot - pos_us - 1 ; sdo_l2_20140301_171928_3800111285_20140301181000_1700.fits n_waves_aia = n_elements(files_aia) for j=0, n_waves_aia-1 do begin if not exist(waves_aia) then $ waves_aia = strmid(filnams_aia[j], pos_us[j]+1, slen[j]) else $ waves_aia = [waves_aia, strmid(filnams_aia[j], pos_us[j]+1, slen[j])] endfor ; Loop through all AIA waves: delvarx, n_img_arr_aia for j=0, n_waves_aia-1 do begin t0_read_sec = anytim(!stime) read_iris_l2, files_aia[j], index0, data0 t1_read_sec = anytim(!stime) n_img_read = n_elements(index0) tsec_read_per_img = double(t1_read_sec - t0_read_sec) / n_img_read t_read = anytim(!stime, /ccsds) buff_read = t_read spawn, 'uname -n', node buff_read = buff_read + ' node: ' + strtrim(node,2) buff_read = buff_read + ' AIA L2 FITS cube read time per image = ' + $ string(tsec_read_per_img, '$(f6.3)') + ' seconds.' buff_read = buff_read + $ ' Total read time for ' + string(n_img_read, '$(i5.5)') + ' images = ' + $ string(double(t1_read_sec - t0_read_sec), '$(f6.3)') + ' seconds.' if not keyword_set(no_logging) then file_append, log_fits_read, buff_read prstr, buff_read if not exist(n_img_arr_aia) then n_img_arr_aia = n_elements(index0) else $ n_img_arr_aia = [n_img_arr_aia, n_elements(index0)] t_wave0 = anytim(index0.date_obs, /ccsds) ; ================ ; Begin kluge ; (Apparently a kluge if index_iris_ref not yet defined): ; ================ if not exist(index_iris_ref) then begin ; n_samp = n_img_arr_aia < n_samp_aia_max index_iris_ref = index0 t_ref_arr = anytim(index_iris_ref.date_obs, /ccsds) t_ref_arr_int = anytim(t_ref_arr, /int) t_ref_arr_sec = anytim(t_ref_arr) t0_ref = min(t_ref_arr) t1_ref = max(t_ref_arr) t_span_sec_ref = anytim(t1_ref) - anytim(t0_ref) ; If necessary, sub-sample the reference time array to have no more elements than ; the maximum allowed: if exist(n_img_max_iris) then $ n_img_max_ref = n_img_max_iris else $ n_img_max_ref = n_samp_max if n_img_max_ref gt n_samp_max then $ t_grid_ref = timegrid(t0_ref, t1_ref, nsamp = n_samp_max) else $ t_grid_ref = timegrid(t0_ref, t1_ref, nsamp = n_img_max_ref) ss_samp_ref = tim2dset(t_ref_arr_int, t_grid_ref) ; ss_samp_ref = grid_data(t_ref_arr_int, nsamp=n_samp_max, /not_uniq) t_samp_ref = t_ref_arr_int[ss_samp_ref] n_samp = n_elements(t_samp_ref) t_samp_ref_int = anytim(t_samp_ref, /int) fovx_ref = index_iris_ref[0].fovx fovy_ref = index_iris_ref[0].fovy index0 = index0[ss_samp_ref] data0 = data0[*,*,ss_samp_ref] ; Create a map array from the reference index0 and data0 array: index2map, index0, data0, map_ref endif ; ================ ; End kluge to defin index_iris_ref ; ================ ss_samp_wave0 = tim2dset(anytim(t_wave0, /int), t_samp_ref) ; ss_samp_wave0 = grid_data(t_wave0, nsamp=n_samp_ref, /not_uniq) index0 = index0[ss_samp_wave0] data0 = data0[*,*,ss_samp_wave0] wave0 = waves_aia[j] t_wave0 = anytim(index0.date_obs, /ccsds) n_files0 = n_elements(index0) if not exist(n_img_arr) then n_img_arr = n_files0 else n_img_arr = [n_img_arr, n_files0] ; Process each AIA image separately: for k=0, n_files0-1 do begin index0_single = index0[k] data0_single = reform(data0[*,*,k]) index2map, index0_single, data0_single, map0_single rmap0_single = drot_map(map0_single, ref_map=map_ref[k], no_drotate=no_drotate, $ /keep_limb, fast=fast, no_project=no_project, _extra=_extra) if k eq 0 then begin index0_fov = index0_single data0_fov = rmap0_single.data endif else begin index0_fov = concat_struct(index0_fov, index0_single) data0_fov = [[[data0_fov]], [[rmap0_single.data]]] endelse endfor ; Scale data cube for this wave: ; Set all zeros in the fov cutout cube equal to the median value: ; ss_z = where(data0_fov eq 0, n_z) ; ss_nz = where(data0_fov ne 0, n_nz) ; if n_nz gt 0 then begin ; median_nz = median(data0_fov[ss_nz]) ; data0_fov[ss_z] = median_nz ; endif ; Perform canonical AIA scaling according to wavelength: ; for k=0, n_files-1 do begin ; data0_fov[0,0,k] = $ ; aia_intscale(data0_fov[*,*,k], exptime=index0_fov[k].exptime, $ ; wavelnth=fix(wave), /bytescale) ; endfor ; if keyword_set(do_log) then data0_fov = safe_log10(data0_fov-(min(data0_fov)>0)) if keyword_set(do_log) then begin if fix(wave0) ne 304 then begin data0_fov = safe_log10(data0_fov) endif else begin data0_fov = safe_log10(data0_fov-(min(data0_fov)>0)) endelse endif data0_fov = bytscl(data0_fov) ; -(min(data0_fov)>0)) ;if fix(wave0) ne 304 then STOP file_times0 = index0.date_obs files_out0 = 'sdo_iris_' + string(wave0, '(i4.4)') + 'A_' + time2file(file_times0, /sec) + $ '.' + file_suffix files_out0 = concat_dir(dir_data, files_out0) ; Write out AIA images for this wave in desired format (jpeg, gif, etc): if keyword_set(do_write) then begin ; Create directory for this obsid, if it doesn't already exist: if ~keyword_set(no_mk_dir) then begin if ~file_exist(dir_data) then begin mk_dir, dir_data print, ' **********************' print, ' Creating directory ' + dir_data print, ' **********************' endif endif set_plot, 'z' aia_lct, r0, g0, b0, wavelnth=fix(wave0) set_plot, 'x' for k=0,n_files0-1 do begin case file_type of 'jpeg': begin data0_single = reform(data0_fov[*,*,k]) if keyword_set(do_color) then begin ; datar = r0[data0_single] & datag = g0[data0_single] & datab = b0[data0_single] ; datat = [[[datar]],[[datag]],[[datab]]] write_jpeg, files_out0[k], truecolor(data0_single, r0, g0, b0), true=1, quality=quality endif else begin write_jpeg, files_out0[k], data0_single, quality=quality endelse end 'gif': write_gif, files_out0[k], data0_single, r0, g0, b0 else: write_jpeg, files_out0[k], data0_single, quality=quality endcase endfor endif ; Add output file names for this wave to array of file names for all waves: if not exist(files_out_arr) then files_out_arr = files_out0 else $ files_out_arr = [files_out_arr, files_out0] ; Delete the last index and data arrays, and working variables, before processing another wave: delvarx, files_out0, index0, data0, index0_fov, data0_fov print, 'Completed AIA wave ' + wave0 endfor n_img_max_aia = max(n_img_arr_aia, ss_n_img_max_aia) endif else begin print, 'No AIA waves to process.' n_waves_aia = 0 n_img_max_aia = 0 endelse ;============================= ; Now process Hinode SOT waves: ;============================= ;HELP, n_img_arr, n_img_arr_iris, n_img_arr_aia, n_img_arr_sot, $ ; n_img_max_iris, n_img_max_aia, n_img_max_sot, $ ; instr_arr, n_waves_iris, n_waves_aia, n_waves_sot, $ ; waves_arr, waves_iris, waves_aia, waves_sot if n_files_sot gt 0 then begin ; Determine all SOT waves present: filnams_sot = file_break(files_sot) pos_dot = strpos(filnams_sot, '.', /reverse_search) slen = pos_dot - 50 ; sot_l2_20140427_233436_3800258458_20140428001200_CaIIHline_FG.fits n_waves_sot = n_elements(files_sot) for j=0, n_waves_sot-1 do begin if not exist(waves_sot) then $ waves_sot = strmid(filnams_sot[j], 49, slen[j]) else $ waves_sot = [waves_sot, strmid(filnams_sot[j], 49, slen[j])] endfor ; Loop through all SOT waves: for j=0, n_waves_sot-1 do begin t0_read_sec = anytim(!stime) read_iris_l2, files_sot[j], index0, data0 t1_read_sec = anytim(!stime) n_img_read = n_elements(index0) tsec_read_per_img = double(t1_read_sec - t0_read_sec) / n_img_read t_read = anytim(!stime, /ccsds) buff_read = t_read spawn, 'uname -n', node buff_read = buff_read + ' node: ' + strtrim(node,2) buff_read = buff_read + ' SOT L2 FITS cube read time per image = ' + $ string(tsec_read_per_img, '$(f6.3)') + ' seconds.' buff_read = buff_read + $ ' Total read time for ' + string(n_img_read, '$(i5.5)') + ' images = ' + $ string(double(t1_read_sec - t0_read_sec), '$(f6.3)') + ' seconds.' if not keyword_set(no_logging) then file_append, log_fits_read, buff_read prstr, buff_read if j eq 0 then n_img_arr_sot = n_elements(index0) else $ n_img_arr_sot = [n_img_arr_sot, n_elements(index0)] t_wave0 = anytim(index0.date_obs, /ccsds) ss_samp_wave0 = tim2dset(anytim(t_wave0, /int), t_samp_ref) ; ss_samp_wave0 = grid_data(t_wave0, nsamp=n_samp_ref, /not_uniq) index0 = index0[ss_samp_wave0] data0 = data0[*,*,ss_samp_wave0] wave0 = waves_sot[j] t_wave0 = anytim(index0.date_obs, /ccsds) n_files0 = n_elements(index0) if not exist(n_img_arr) then n_img_arr = n_files0 else n_img_arr = [n_img_arr, n_files0] ; Process each SOT image separately: for k=0, n_files0-1 do begin index0_single = index0[k] data0_single = reform(data0[*,*,k]) index2map, index0_single, data0_single, map0_single rmap0_single = drot_map(map0_single, ref_map=map_ref[k], no_drotate=no_drotate, $ /keep_limb, fast=fast, no_project=no_project, _extra=_extra) if k eq 0 then begin index0_fov = index0_single data0_fov = rmap0_single.data endif else begin index0_fov = concat_struct(index0_fov, index0_single) data0_fov = [[[data0_fov]], [[rmap0_single.data]]] endelse endfor ; Scale data cube for this wave: ; Set all zeros in the fov cutout cube equal to the median value: ; ss_z = where(data0_fov eq 0, n_z) ; ss_nz = where(data0_fov ne 0, n_nz) ; if n_nz gt 0 then begin ; median_nz = median(data0_fov[ss_nz]) ; data_test = data0_fov ; data0_fov[ss_z] = median_nz ; endif ; if keyword_set(do_log) then data0_fov = safe_log10(data0_fov) data0_fov = bytscl(data0_fov) file_times0 = index0.date_obs files_out0 = 'sdo_iris_' + strtrim(wave0,2) + '_' + time2file(file_times0, /sec) + $ '.' + file_suffix files_out0 = concat_dir(dir_data, files_out0) ; Write out SOT images for this wave in desired format (jpeg, gif, etc): if keyword_set(do_write) then begin ; Create directory for this obsid, if it doesn't already exist: if ~keyword_set(no_mk_dir) then begin if ~file_exist(dir_data) then begin mk_dir, dir_data print, ' **********************' print, ' Creating directory ' + dir_data print, ' **********************' endif endif for k=0,n_files0-1 do begin case file_type of 'jpeg': begin data0_single = reform(data0_fov[*,*,k]) if keyword_set(do_color) then begin ; datar = r0[data0_single] & datag = g0[data0_single] & datab = b0[data0_single] ; datat = [[[datar]],[[datag]],[[datab]]] write_jpeg, files_out0[k], truecolor(data0_single, r0, g0, b0), true=1, quality=quality endif else begin write_jpeg, files_out0[k], data0_single, quality=quality endelse end 'gif': write_gif, files_out0[k], data0_single, r0, g0, b0 else: write_jpeg, files_out0[k], data0_single, quality=quality endcase endfor endif ; Add output file names for this wave to array of file names for all waves: if not exist(files_out_arr) then files_out_arr = files_out0 else $ files_out_arr = [files_out_arr, files_out0] ; Delete the last index and data arrays, and working variables, before processing another wave: delvarx, files_out0, index0, data0, index0_fov, data0_fov print, 'Completed SOT wave ' + wave0 endfor n_img_max_sot = max(n_img_arr_sot, ss_n_img_max_sot) endif else begin print, 'No SOT waves to process.' n_waves_sot = 0 n_img_max_sot = 0 endelse ; =============== ; Make JSON file for this obs: ; =============== ;HELP, n_img_arr, n_img_arr_iris, n_img_arr_aia, n_img_arr_sot, $ ; n_img_max_iris, n_img_max_aia, n_img_max_sot, $ ; instr_arr, n_waves_iris, n_waves_aia, n_waves_sot, $ ; waves_arr, waves_iris, waves_aia, waves_sot filnams_arr = file_break(files_out_arr) if n_waves_iris gt 0 then begin instr_arr = replicate('iris', n_waves_iris) waves_arr = waves_iris endif if exist(instr_arr) then begin if n_waves_aia gt 0 then begin instr_arr = [instr_arr, replicate('aia', n_waves_aia)] waves_arr = [waves_arr, waves_aia] endif endif else begin if n_waves_aia gt 0 then begin instr_arr = replicate('aia', n_waves_aia) waves_arr = waves_aia endif endelse if exist(instr_arr) then begin if n_waves_sot gt 0 then begin instr_arr = [instr_arr, replicate('sot', n_waves_sot)] waves_arr = [waves_arr, waves_sot] endif endif else begin if n_waves_sot gt 0 then begin instr_arr = replicate('sot', n_waves_sot) waves_arr = waves_sot endif endelse if ~exist(instr_arr) then begin do_skip = 1 ;STOP GOTO, SKIPCASE endif if keyword_set(do_write) then $ mk_json_new, instr_arr, waves_arr, fovx_ref=fovx_ref, fovy_ref=fovy_ref, $ dir_hinode_centric=dir_hinode_centric, $ filnams_arr, n_img_arr, obsid0=obsid[i], $ dir_data=dir_data, file_json=file_json, url_base=url_base, $ use_local=use_local, $ _extra=_extra ; ======================== ; SKIPCASE Re-entry point: ; ======================== SKIPCASE: ; ======================== ; Do logging for this obs: ; ======================== ; Create and append log file record for this obs id: buff = 'run_date: ' + strtrim(sttim_obs,2) buff = buff + ' obsid: ' + strtrim(obsid[i],2) spawn, 'uptime', buff_out ss_load = strpos(buff_out, 'load') if ss_load[0] ne -1 then load_avg = strmid(buff_out, ss_load) buff = buff + ' ' + strtrim(load_avg,2) spawn, 'uname -n', node buff = buff + ' node: ' + strtrim(node,2) if keyword_set(do_skip) then begin buff = buff + 'Skipping obsid ' + strtrim(obsid[i],2) + $ ' - One or more problems with this obs.' endif else begin buff = buff + ' nimg_samps_per_wave: ' + string(n_img_arr[0], '$(i5.5)') buff = buff + ' nimg_total: ' + string(long(total(n_img_arr)), '$(i6.6)') buff = buff + ' waves: ' + arr2str(waves_arr) buff = buff + ' nimg_max_iris: ' + string(n_img_max_iris, '$(i5.5)') buff = buff + ' nimg_max_aia: ' + string(n_img_max_aia, '$(i5.5)') buff = buff + ' nimg_max_sot: ' + string(n_img_max_sot, '$(i5.5)') spawn, 'du -hs ' + dir_obs_in + '/', siz_obs ss_path = strpos(siz_obs, '/') siz_obs = strmid(siz_obs, 0, ss_path) buff = buff + ' siz_obs: ' + strtrim(siz_obs,2) spawn, 'du -hs ' + dir_data + '/', siz_cruiser ss_path = strpos(siz_cruiser, '/') siz_cruiser = strmid(siz_cruiser, 0, ss_path) buff = buff + ' siz_cruiser: ' + strtrim(siz_cruiser,2) entim_obs = anytim(!stime, /ccsds) t_sec_obs = long(anytim(entim_obs) - anytim(sttim_obs)) buff = buff + ' t_sec_obs: ' + strtrim(t_sec_obs,2) buff = buff + ' sec_per_img: ' + string(double(t_sec_obs)/total(n_img_arr), '$(f4.1)') buff = buff + ' sec_per_sec: ' + string(double(t_sec_obs)/t_span_sec_ref, '$(f5.2)') endelse if not keyword_set(no_logging) then file_append, log_pipeline, buff ; ========================= ; Loop Cleanup - Reset/delete variables from this obs, before starting next obs, ; and optionally announce that we are done with current obs: ; ========================= ; Clean up before next obs processing: delvarx, index_iris_all_waves, index_aia_all_waves, index_sot_all_waves delvarx, index_iris0, index_iris_ref, map_ref, files_out_arr delvarx, n_img_arr, n_img_arr_iris, n_img_arr_aia, n_img_arr_sot delvarx, n_img_max_iris, n_img_max_aia, n_img_max_sot delvarx, instr_arr, n_waves_iris, n_waves_aia, n_waves_sot delvarx, waves_arr, waves_iris, waves_aia, waves_sot delvarx, do_skip print, '' print, '' print, '' print, '' print, '***************' print, '***************' print, 'Done with obsid ' + strtrim(obsid[i],2) print, buff print, '***************' print, '***************' print, '' print, '' print, '' print, '' endfor endif end