;+
; NAME: sotsp_level2dsummary.pro
;
; PURPOSE: Generate summary webpage for SOT-SP level 2 (inverted) data
;
; INPUTS:
;   l2filesin - list of one or more SOT Level 2 directories
;
; KEYWORD PARAMETERS:
;   debug - switch that, if set, causes the routine to stop at various points 
;   table - switch that, if set, will add a suffix to outdir
;   qonly - switch that, if set, will just re-create the index.html file
;           (maybe useful in instances when this routine doesn't fully complete)
;   sot2ssw - switch that, if set, will replace links starting with
;             http[s]://sot.lmsal.com/data/sot with links starting with
;             http[s]://www.lmsal.com/solarsoft/hinode in the index.html
;             webpage that gets created (rarely used?)
;   link_list - switch that, if set, will insert previous/current/next links
;               into the list of html files (rarely used?)
;   refresh - switch that, if set, will force the output files to be (re)made,
;             otherwise those directories with summary webpages that already
;             exist will be skipped 
;   outdir - if set, routine writes out summary webpages to specified
;            directory structure (so be careful ... may overwrite things),
;            otherwise output directories are LMSAL production directories
;
; MODIFICATION HISTORY:
;   S.L.Freeland - c.2008 - created
;   M.DeRosa - 2021/12/20 - added this doc header, added better comments,
;                           added outdir keyword, removed no_link_list switch,
;                           and removed lots of other extraneous stuff
;   M.DeRosa - 2021/12/26 - for repeating scans, removed flash movie link
;                           (since flash is deprecated) and added
;                           browser-playable mp4
;   M.DeRosa - 2022/01/11 - fixed bug for repeating scan case without /refresh
;                           set (routine was setting movie name incorrectly)
;   R.Timmons - 2022/01/12 - added -nostdin to ffmpeg command for smoother
;                            batch running 
;   M.DeRosa - 2022/05/24 - now doesn't crash for two kinds of errors:
;                           when there are errors from sotsp_rasterize.pro,
;                           and when there is an incomplete level 2 map (i.e.,
;                           when the number of scan columns in the level 2
;                           map doesn't match the number of elements in
;                           the slit position array)
;   M.DeRosa - 2022/08/17 - fixed bug in IDL command section at bottom of the
;                           HTML page, without /refresh set (routine was
;                           putting incorrect ScanID in the IDL commands) 
;   M.DeRosa - 2022/11/26 - images on summary webpage now include missing scan
;                           columns, also the boilerplate text underneath the
;                           summary images has been updated to incidate this 
;   M.DeRosa - 2022/12/08 - summary HTML now includes level 2.1 output
;                           quantities in second row of large grid of images
;-

pro sotsp_level2dsummary, l2filesin, link_list=link_list, table=table, $
  debug=debug, qonly=qonly, sot2ssw=sot2ssw,refresh=refresh,outdir=outdir

;  check switches
debug=keyword_set(debug)
qonly=keyword_set(qonly)
sot2ssw=keyword_set(sot2ssw)
generate=(1-qonly) and (1-sot2ssw)
link_list=keyword_set(link_list) or (1-generate)

;  get ScanIDs and filenames of level 2 FITS files
spids=strextract(l2filesin,'SP3D/','/')
bad=where(strlen(spids) ne 15,bss)
if bss gt 0 then begin 
  box_message,['Poorly formed l2 file names?',spids(bad)]
  stop,'spids'
  return
endif
filenames=ssw_strsplit(l2filesin,'/',/tail,head=paths)

;  set up table directory
tdir=''
if keyword_set(table) then tdir='_'+strtrim(table,2) else table=0
tab=table

;  set up output directory
if keyword_set(outdir) then begin
  outpaths=outdir+ssw_strsplit(paths,'level2hao',/tail) + tdir
endif else begin
  outpaths=str_replace(paths,'level2hao','level2d') + tdir
  outpaths=str_replace(outpaths,'/archive','/archive1')
endelse
ups=all_vals(outpaths)
mdirs=outpaths
mnams='sotsp_level2_'+spids
hdocs=concat_dir(outpaths,mnams)+'.html'
if debug then stop,'hdoc'
for i=0,n_elements(ups)-1 do if 1-file_exist(ups(i)) then mk_dir,ups(i)
hindex=concat_dir(outpaths,'index.html')

;  use z buffer
set_plot,'z'

;  boilerplate for webpage
idlcmdss="<p><font color=orange size=+1> sotsp_stks2index,'" + spids + "',index,data,/LEVEL2,/SCALE </font> ; SP Team scaling applied</font><br>"

idlcmds="<br><font color=orange size=+1> sotsp_stks2index,'" + spids + "',index,data,/LEVEL2</font> ; transfer binary Extended FITS + read->index,data<br>"
idlcmdss="<br><font color=orange size=+1> sotsp_getdata,'" + spids + "',LEVEL=2</font>  ; binary Extended FITS -> local (no read) <br>"
idlcmdsss="<br><font color=orange size=+1> sotsp_getdata,'" + spids + "',/CDF, LEVEL=2</font> ; Merlin CDF Output -> local (no read) <br>"

l1urls='https://sot.lmsal.com/data/sot/level1d' +  $
   ssw_strsplit(paths,'level2hao',/tail) + '/index.html'

;  checks which html files already exist (and skips those)
if keyword_set(refresh) or qonly then l2files=l2filesin else begin
  needss=where(1-file_exist(hdocs), ncnt)
  if ncnt gt 0 then begin 
    l2files=l2filesin(needss)   
    hdocs=hdocs(needss)
    l1urls=l1urls(needss)
    mdirs=mdirs(needss)
    mnams=mnams(needss)
    hindex=hindex(needss)
    outpaths=outpaths(needss)
    idlcmds=idlcmds(needss)
    idlcmdss=idlcmdss(needss)
    idlcmdsss=idlcmdsss(needss)
  endif
  generate=generate and ncnt gt 0
endelse
if debug then stop,'ncnt,hdocs'

;  loop through the level 2 directories
if generate then begin  
  for i=0,n_elements(l2files) -1 do begin 
    delvarx,dd
    file_append,'done.dat',l2files(i),new=(i eq 0)
    next=get_fits_nextend(l2files(i))

    if next lt 42 then begin  ;  trap one type of error
      box_message,'Unexpected number of extensions, skipping...'
      file_append,'done.dat','BADONE'
      if debug then stop,'BADONE'
    endif else begin

      ;  read in data
      read_sotsp,l2files(i), ii,dd, /scale,/los_mag, scan_info=scan_info
      slitmap=sotsp_rasterize(scan_info,error=rastererror)  ;  quick error check

      ;  check if lev2.1 is available
      lev21file=str_replace(l2files(i),'level2hao','level2.1hao')
      lev21file=str_replace(lev21file,'.fits','_L2.1.fits')
      if file_test(lev21file) then lev21=1b else lev21=0b

      ;  insert missing slit positions and (if repeating scan) reform array
      case 1 of  ;  trap some additional errors
        n_elements(scan_info.slitpos) ne ii[0].naxis1: begin
          box_message,'Slit positions not equal to map columns, skipping...'
          file_append,'done.dat','BADONE'
          if debug then stop,'BADONE'
          end
        rastererror gt 0: begin
          box_message,'Error found in sotsp_rastserize, skipping...'
          file_append,'done.dat','BADONE'
          if debug then stop,'BADONE'
          end
        else: begin

          delvarx,oii,odd,status,error
          sotsp_reform,ii,dd,oii,odd, status=status, scan_info=scan_info, $
            error=error,slitmap=slitmap,debug=debug
          if error then box_message,'Constant SLITPOS but proceeding..'

          ;  create the large table of images
          thumbsize=([0,250])(1-status)
          ;; order=[31,32,33,30,34,rem_elem(indgen(30),23),35]
          order=[31,32,33,30,34,2,1,0,rem_elem(indgen(27),23-3)+3,35] 
          if lev21 then order=[order[0:4],[39,37,38,36,35],order[5:-2],23]
          labels=oii(order).ftype
          if lev21 then begin
            wh=strpos(labels,' (positive')
            for jj=0,n_elements(wh)-1 do $
              if wh[jj] gt 0 then labels[jj]=strmid(labels[jj],0,wh[jj])
          endif
          if (status or error) then odd=reform(transpose(odd,[0,3,1,2]), $
            [n_elements(slitmap),(size(dd,/dim))[1:2]])
          image2movie, cube_edit(odd,/data,ss=order), label=labels, $
            movie_dir=mdirs(i), $
            thumbsize=thumbsize, maxxy=250, corexy=status, $
            movie_name=mnams(i), table=table, html=html,/still, tperline=5

          ;  create movie frames for repeating scans
          if status then begin

            delvarx,oii,odd,status,error
            sotsp_reform,ii,dd,oii,odd, status=status, scan_info=scan_info, $
              only_maps=[31,33,0,1,2],/concat,error=error,slitmap=slitmap, $
              debug=debug
            dd=odd
            dd(*,0:20,*)=0  ;  make a black area at the bottom for labeling

            ;  get array of times, one for each raster
            sst=reform(slitmap(0,*))  ;  slitmap array should be 2-d
            whmissing=where(sst lt 0,nmissing)
            if nmissing gt 0 then begin  ;  in case first slitpos is missing
              for jj=0,nmissing-1 do begin
                whfirst=where(slitmap(*,whmissing[jj]) gt 0)
                sst[whmissing[jj]]=slitmap(whfirst[0],whmissing[jj])
              endfor
            endif
            ftimes=scan_info.times(sst)
            labels=anytim(ftimes,/ecs,/truncate) + ' ' + oii.ftype
            labels=str_replace(labels,'Magnetic Field',' B')
          
            ;  write out movie frames and generate movie webpages
            special_movie, ftimes, dd, /no_gifanimate, table=0, $
              movie_name=mnams(i)+'s', movie_dir=mdirs(i), thumbsize=250, $
              html=html2,label=labels, labstyle='/LC', maxxy=250,outsize=768, $
              controls='startstop,looprock,speed,step,setframe', $
              bottom_controls='enhance,step,toggle',active_zoom="true",rate='240'

            ;  create mp4 and edit html
            ffmpeg=ssw_bin('ffmpeg',found=ffmpegfound,/warning)
            if ffmpegfound then begin
              jsfile=str_replace(hdocs[i],'.html','s_j.html')
              mp4file=str_replace(hdocs[i],'.html','.mp4')
              spawn,'grep url_path '+jsfile+ $
                    ' | grep urls | awk -F''"'' ''{print $2}''',giflist
              openw,lun,outpaths[i]+'/giflist.txt',/get
              printf,lun,'file '''+outpaths[i]+giflist+''''
              free_lun,lun
              ffmpegcmd=ffmpeg+' -nostdin -y -r 10 -f concat -safe 0 -i '+outpaths[i]+'/giflist.txt -vcodec libx264 -pix_fmt yuv420p -r 10 '+mp4file
              box_message,'making mp4: '+ffmpegcmd
              spawn,ffmpegcmd
              if file_test(mp4file) then begin
                mp4size=file_size(mp4file,/string,/auto)
                htmlbuffer=html2[3]
                atag1=strpos(htmlbuffer,'<A')
                atag2=strpos(htmlbuffer,'</A>')
                html2[3]=strjoin([strmid(htmlbuffer,0,atag1), $
                  '<A HREF="'+mnams[i]+'.mp4">MP4 '+mp4size+'</A>',$
                  strmid(htmlbuffer,atag2+4)])
                html2=[html2,'<table border=5 cellpadding=5 cellspacing=3><tr align=center><td><video controls loop><source src="'+mnams[i]+'.mp4" type="video/mp4">Your browser does not support the video tag.</video></td></table>']
              endif
            endif else box_message,'ffmpeg not found, so no mp4 is being made'
          
          endif

          ;  write out webpage using the pieces created above
          html_doc,hdocs(i),/header
          file_append,hdocs(i),hinode_credits(/url,/sot)
          file_append,hdocs(i),str2html(l1urls(i),link='SP Level1 Summary corresponding to <em>this</em> Level2/Inversion')
          file_append,hdocs(i),html                ; thumbnail table
          if status then file_append,hdocs(i),html2 ; "movie" 
          file_append,hdocs(i),rd_tfile('$SSW/hinode/sot/idl/sp/util_lmsal/sotsp_l2.txt')
          file_append,hdocs(i), $
            [idlcmds(i),'-or-', idlcmdss(i),'-or-', idlcmdsss(i)]
          file_append,hdocs(i),['<p>',hinode_credits(/url)]
          html_doc,hdocs(i),/trailer

          ; make composite thumbnails/icons (for monthly index/summary webpages)
          sotsp_level2index2icons,mdirs(i) +'/'

          end
      endcase
    endelse
  endfor
endif

;  apply /sot2ssw or /link_list switches
sshd=where(file_exist(hdocs),hdcnt)
hdocs=hdocs(sshd)
hindex=hindex(sshd)
if sot2ssw then begin 
  box_message,'sot2ssw,hdocs'
  for i=0,hdcnt-1 do begin 
    dat=rd_tfile(hdocs(i))
    ss=where(strpos(dat,'level1d') ne -1,sscnt)
    if sscnt gt 0 then begin
      dat(ss)=str_replace(dat(ss),'sot.lmsal.com/data/sot', $
                          'www.lmsal.com/solarsoft/hinode')
      file_append,hdocs(i),dat,/new
      if i le 1 then stop,hdocs(i)
    endif
  endfor
endif else begin
  if link_list then begin 
    box_message,'link_list'
    set_logenv,'path_http','/net/kokuten/archive/hinode'
    set_logenv,'top_http','http://sot.lmsal.com/data'
    if debug then stop,'http_names,hdocs'
    html_linklist,hdocs,insert='sundiv.gif',/absolute,/init
  endif
endelse

;  write out index.html file(s)
for i=0,hdcnt-1 do file_append,hindex(i),rd_tfile(hdocs(i)),/new

if debug then stop,'before return'

end