;+------------------------------------------------------------------------
; This package of IDL functions facilitates data processing by applying 
; error checking and data normalization procedures to the structure that
; is returned by read_myCDF.  
;-------------------------------------------------------------------------

PRO update_determinants, desc, i, v, depend_list

    ; Get a list of fields for the variable data structure.
    tagnames = TAG_NAMES (v)

    FOR n = 0, N_ELEMENTS (depend_list) - 1 DO BEGIN

        ; Check if the ith attribute in the depend_list is defined for this
        ; variable.
        field = (WHERE (tagnames eq depend_list [n])) [0]

        IF  field ne -1 THEN BEGIN 

            ; And if it has a value.
            target_name = STRCOMPRESS (v.(field), /REMOVE_ALL)

            IF  STRLEN (target_name) gt 0 THEN BEGIN
            
                ; Search through each of the variables in the stucture for one 
                ; That matches the target name.
                target = (WHERE (STRMATCH (desc [*].varname, target_name) eq 1)) [0]

                ; If we couldn't find the determinant varaible, then just move on.
                IF  target eq -1 THEN BREAK

                ; Set the determinant flag (May already have been done).
                desc [target].determinant = 1

                ; Add the index of the variable we checking to the array
                ; of variables the target modifies.  
                IF  desc [target].modify eq !NULL THEN BEGIN

                    desc [target].modify = PTR_NEW ([i])

                ENDIF ELSE BEGIN

                    *desc [target].modify = [*desc [target].modify, i]

                ENDELSE

            ENDIF 

        ENDIF

    ENDFOR

END

FUNCTION create_descriptors, s

    node = {VARNAME:     '',         $  ; Variable name.
            EPOCH:       0,          $  ; True if variable is Epoch.
            NRV:         0,          $  ; True is variable records are NRV
            SORTED:      0,          $  ; True after data is sorted (if needed)
            DETERMINANT: 0,          $  ; True if another vairalbe is dependant
                                        ; on its value. (I.E. DEPEND_0, etc.)
            MODIFY:      PTR_NEW ()  $  ; Array of variables that this variable
                                        ; is a COMPONENT_* or DEPEND_* of.
            }
    ; depend_list is the list of attributes that may specify a variable that
    ; has a dependant relationship with the target variable.
    depend_list = ['DEPEND_0', 'DEPEND_1', 'DEPEND_2', 'COMPONENT_0', 'COMPONENT_1']                     

    desc = REPLICATE (node, N_TAGS (s))

    ; Fill in the variable names first.
    FOR i = 0, N_TAGS (s) - 1 DO desc [i].varname = s.(i).varname

    ; Now complete the rest of the structure.
    FOR i = 0, N_TAGS (s) - 1 DO BEGIN
                                                                       
        desc [i].epoch = STREGEX (s.(i).cdftype,                      $
                                  'CDF_EPOCH(16)?|CDF_TIME_TT2000',   $
                                  /BOOLEAN,                           $
                                  /FOLD_CASE)

        desc [i].nrv = ~STREGEX (s.(i).cdfrecvary, 'VARY', /BOOLEAN, /FOLD_CASE)


        ; Call the update_determinats procedure to create a list of all the 
        ; variables that this variable depends on.
        update_determinants, desc, i, s.(i), depend_list
        
    ENDFOR 
          
    RETURN, desc

END

FUNCTION get_attribute, v, attribute, valid

   ; Find the position of the requested attribute, if it is present in the
   ; variable.
   index = WHERE  (STRUPCASE (TAG_NAMES (v)) eq STRUPCASE (attribute), flag)

   ; Check if the requested attribute exists in the strucure.
   IF  flag ne 0 THEN BEGIN 

       ; Get its value.
       value = STRCOMPRESS (v.(index), /REMOVE_ALL)

       ; Check that it has a proper value.  If so, were done.
       IF  STRLEN (value) gt 0 THEN BEGIN

           valid = 1

           RETURN, value

       ENDIF 

   ENDIF

   ; Requested attribute missing or does not have a value.
   valid = 0

   RETURN, 0

END

PRO put_data, v, d 

    ; Get a list of tag names for the this variable.
    tnames = STRUPCASE (TAG_NAMES (v))
   
    i = WHERE (tnames eq  'HANDLE', cnt)

    IF  cnt eq 0 THEN BEGIN

        i = WHERE (tnames eq 'DAT', cnt)

        IF cnt eq 0 THEN MESSAGE, 'No data found in variable data structure'

        ; Add the new data to the variable.
        v.DAT = d

    ENDIF ELSE BEGIN 

        ; Create a new handle for the data.
        HANDLE_VALUE, v.HANDLE, d, /SET

    ENDELSE       

END 

FUNCTION get_data, v

    ; Get a list of tag names for the this variable.
    tnames = STRUPCASE (TAG_NAMES (v))
   
    i = WHERE (tnames eq  'HANDLE', cnt)

    IF  cnt eq 0 THEN BEGIN

        i = WHERE (tnames eq 'DAT', cnt)

        IF cnt eq 0 THEN MESSAGE, 'No data found in variable data structure'

        ; Get the data.
        val = v.DAT 

    ENDIF ELSE BEGIN 

        ; Get the data.
        HANDLE_VALUE, v.HANDLE, val

    ENDELSE   
   
    ; Were done.
    RETURN, val
 

END


FUNCTION traverse_depends, n, mono, valid, s, desc

FORWARD_FUNCTION traverse_depends
       
    ; Check if the data for this variable has already been processed.
    IF desc [n].sorted eq 1 THEN RETURN, 0

    PRINT, 'Updating data for ', desc [n].varname

    ; count the number of variables we modify.
    cnt = 1

    ; Get the data for for this variable.
    vdata = get_data (s.(n))

    ; Reformat it so that matches the corrected time order.

    ; Check the dimensionality of the data.  How we proceed depends on the
    ; the data structure.
    CASE SIZE (vdata, /N_DIMENSIONS) OF

       1: vdata = vdata [mono [valid]]

       2: vdata = vdata [*, mono [valid]]

       3: vdata = vdata [*, *, mono [valid]]

       4: vdata = vdata [*, *, *, mono [valid]]
  
       ELSE : BEGIN

                 MESSAGE, 'Can not remove time discontinuities from data' + $
                          ' with more than 4 dimensions.'
              END
    ENDCASE
 

    ; Put it back in to the data structure.
    put_data, s.(n), vdata

    ; Set the sorted flag.
    desc [n].sorted = 1

    ; Check if n is has any other variables that depend on it.
    IF  desc [n].determinant eq 0 THEN RETURN, cnt

    ; Find the number of variables that depend on this variables data.
    n_depend = N_ELEMENTS (*desc [n].modify)  

    FOR i = 0, n_depend - 1 DO BEGIN 
        
       next = (*desc [n].modify) [i]
       
       cnt = cnt + traverse_depends (next, mono, valid, s, desc) 

    ENDFOR

    RETURN, cnt

END

PRO sort_time_values, s, desc, VERBOSE=verbose

    ; Find all variables that are of type epoch.
    epochs = WHERE (desc [*].epoch eq 1, cnt)

    ; Loop to find and sort any time values for any variable that is of type 
    ; 'Epoch'.  In addition, any variable that whose values depdend on the 
    ; epoch, either directly or indirectly, will have their values sorted as
    ; as well.
    FOR n = 0, cnt - 1 DO BEGIN

        ; Get the data for for this variable.
        epoch = get_data (s.(epochs [n]))

        ; Number of epochos before screenin for invalid time values.
        nepoch = N_ELEMENTS (epoch)

        ; Get the FILLVAL if there is one.
        string = get_attribute  (s.(epochs [n]), 'FILLVAL', fv)
        
        ; If we do have a FILLVAL, then use it to extract the valid data.
        IF  fv THEN BEGIN 

            READS, string, fillval
             
            ;  Needs to be modified to handle NaN

            valid  = WHERE (epoch ne fillval, nvalid)

        ENDIF ELSE BEGIN 

            valid  = INDGEN (nepoch)
            nvalid = nepoch
        
        ENDELSE

        ; Sort the epochs so that they are monotonic increasing.
        mono = SORT (epoch [valid])

        ; Make sure evary time value is unique.  Duplicate time values will
        ; result in unrecoverable error.
        unique_epoch = UNIQ (mono)

        IF  N_ELEMENTS (unique_epoch) ne nvalid THEN BEGIN

            MESSAGE, 'Duplicate time values found.  Exiting.'

        ENDIF


        IF  KEYWORD_SET (verbose) THEN BEGIN 

            print, "Number of EPOCHS: ", nepoch
            print, "Number of valid EPOCHS: ", nvalid
            print, "Number of unique EPOCHS: ", N_ELEMENTS (unique_epoch)

        ENDIF 

        ; See if the epochs are correctly ordered and if they are all valid.
        ; If this is the case then we are done.
        IF (nvalid eq nepoch) && ARRAY_EQUAL (mono, INDGEN (nepoch)) THEN CONTINUE

        ; Update each variable that depends on this variable, starting with this
        ; variable.
        sink = traverse_depends (epochs [n], mono, valid, s, desc)

        ; stop
               
    ENDFOR

END

; Main wrapper function.
FUNCTION rectify_data, in, VERBOSE=verbose

    work = in

    desc = create_descriptors (work)
 
    sort_time_values, work, desc, VERBOSE=verbose

    RETURN, work

END

; Alternative interface.
PRO rectify_data, s, VERBOSE=verbose

    s = rectify_data (s, VERBOSE=verbose)

END