;+------------------------------------------------------------------------
; This package of IDL functions facilitates data processing by applying
; error checking and data normalization procedures to the structure that
; is returned by read_myCDF.
;-------------------------------------------------------------------------
PRO update_determinants, desc, i, v, depend_list
; Get a list of fields for the variable data structure.
tagnames = TAG_NAMES (v)
FOR n = 0, N_ELEMENTS (depend_list) - 1 DO BEGIN
; Check if the ith attribute in the depend_list is defined for this
; variable.
field = (WHERE (tagnames eq depend_list [n])) [0]
IF field ne -1 THEN BEGIN
; And if it has a value.
target_name = STRCOMPRESS (v.(field), /REMOVE_ALL)
IF STRLEN (target_name) gt 0 THEN BEGIN
; Search through each of the variables in the stucture for one
; That matches the target name.
target = (WHERE (STRMATCH (desc [*].varname, target_name) eq 1)) [0]
; If we couldn't find the determinant varaible, then just move on.
IF target eq -1 THEN BREAK
; Set the determinant flag (May already have been done).
desc [target].determinant = 1
; Add the index of the variable we checking to the array
; of variables the target modifies.
IF desc [target].modify eq !NULL THEN BEGIN
desc [target].modify = PTR_NEW ([i])
ENDIF ELSE BEGIN
*desc [target].modify = [*desc [target].modify, i]
ENDELSE
ENDIF
ENDIF
ENDFOR
END
FUNCTION create_descriptors, s
node = {VARNAME: '', $ ; Variable name.
EPOCH: 0, $ ; True if variable is Epoch.
NRV: 0, $ ; True is variable records are NRV
SORTED: 0, $ ; True after data is sorted (if needed)
DETERMINANT: 0, $ ; True if another vairalbe is dependant
; on its value. (I.E. DEPEND_0, etc.)
MODIFY: PTR_NEW () $ ; Array of variables that this variable
; is a COMPONENT_* or DEPEND_* of.
}
; depend_list is the list of attributes that may specify a variable that
; has a dependant relationship with the target variable.
depend_list = ['DEPEND_0', 'DEPEND_1', 'DEPEND_2', 'COMPONENT_0', 'COMPONENT_1']
desc = REPLICATE (node, N_TAGS (s))
; Fill in the variable names first.
FOR i = 0, N_TAGS (s) - 1 DO desc [i].varname = s.(i).varname
; Now complete the rest of the structure.
FOR i = 0, N_TAGS (s) - 1 DO BEGIN
desc [i].epoch = STREGEX (s.(i).cdftype, $
'CDF_EPOCH(16)?|CDF_TIME_TT2000', $
/BOOLEAN, $
/FOLD_CASE)
desc [i].nrv = ~STREGEX (s.(i).cdfrecvary, 'VARY', /BOOLEAN, /FOLD_CASE)
; Call the update_determinats procedure to create a list of all the
; variables that this variable depends on.
update_determinants, desc, i, s.(i), depend_list
ENDFOR
RETURN, desc
END
FUNCTION get_attribute, v, attribute, valid
; Find the position of the requested attribute, if it is present in the
; variable.
index = WHERE (STRUPCASE (TAG_NAMES (v)) eq STRUPCASE (attribute), flag)
; Check if the requested attribute exists in the strucure.
IF flag ne 0 THEN BEGIN
; Get its value.
value = STRCOMPRESS (v.(index), /REMOVE_ALL)
; Check that it has a proper value. If so, were done.
IF STRLEN (value) gt 0 THEN BEGIN
valid = 1
RETURN, value
ENDIF
ENDIF
; Requested attribute missing or does not have a value.
valid = 0
RETURN, 0
END
PRO put_data, v, d
; Get a list of tag names for the this variable.
tnames = STRUPCASE (TAG_NAMES (v))
i = WHERE (tnames eq 'HANDLE', cnt)
IF cnt eq 0 THEN BEGIN
i = WHERE (tnames eq 'DAT', cnt)
IF cnt eq 0 THEN MESSAGE, 'No data found in variable data structure'
; Add the new data to the variable.
v.DAT = d
ENDIF ELSE BEGIN
; Create a new handle for the data.
HANDLE_VALUE, v.HANDLE, d, /SET
ENDELSE
END
FUNCTION get_data, v
; Get a list of tag names for the this variable.
tnames = STRUPCASE (TAG_NAMES (v))
i = WHERE (tnames eq 'HANDLE', cnt)
IF cnt eq 0 THEN BEGIN
i = WHERE (tnames eq 'DAT', cnt)
IF cnt eq 0 THEN MESSAGE, 'No data found in variable data structure'
; Get the data.
val = v.DAT
ENDIF ELSE BEGIN
; Get the data.
HANDLE_VALUE, v.HANDLE, val
ENDELSE
; Were done.
RETURN, val
END
FUNCTION traverse_depends, n, mono, valid, s, desc
FORWARD_FUNCTION traverse_depends
; Check if the data for this variable has already been processed.
IF desc [n].sorted eq 1 THEN RETURN, 0
PRINT, 'Updating data for ', desc [n].varname
; count the number of variables we modify.
cnt = 1
; Get the data for for this variable.
vdata = get_data (s.(n))
; Reformat it so that matches the corrected time order.
; Check the dimensionality of the data. How we proceed depends on the
; the data structure.
CASE SIZE (vdata, /N_DIMENSIONS) OF
1: vdata = vdata [mono [valid]]
2: vdata = vdata [*, mono [valid]]
3: vdata = vdata [*, *, mono [valid]]
4: vdata = vdata [*, *, *, mono [valid]]
ELSE : BEGIN
MESSAGE, 'Can not remove time discontinuities from data' + $
' with more than 4 dimensions.'
END
ENDCASE
; Put it back in to the data structure.
put_data, s.(n), vdata
; Set the sorted flag.
desc [n].sorted = 1
; Check if n is has any other variables that depend on it.
IF desc [n].determinant eq 0 THEN RETURN, cnt
; Find the number of variables that depend on this variables data.
n_depend = N_ELEMENTS (*desc [n].modify)
FOR i = 0, n_depend - 1 DO BEGIN
next = (*desc [n].modify) [i]
cnt = cnt + traverse_depends (next, mono, valid, s, desc)
ENDFOR
RETURN, cnt
END
PRO sort_time_values, s, desc, VERBOSE=verbose
; Find all variables that are of type epoch.
epochs = WHERE (desc [*].epoch eq 1, cnt)
; Loop to find and sort any time values for any variable that is of type
; 'Epoch'. In addition, any variable that whose values depdend on the
; epoch, either directly or indirectly, will have their values sorted as
; as well.
FOR n = 0, cnt - 1 DO BEGIN
; Get the data for for this variable.
epoch = get_data (s.(epochs [n]))
; Number of epochos before screenin for invalid time values.
nepoch = N_ELEMENTS (epoch)
; Get the FILLVAL if there is one.
string = get_attribute (s.(epochs [n]), 'FILLVAL', fv)
; If we do have a FILLVAL, then use it to extract the valid data.
IF fv THEN BEGIN
READS, string, fillval
; Needs to be modified to handle NaN
valid = WHERE (epoch ne fillval, nvalid)
ENDIF ELSE BEGIN
valid = INDGEN (nepoch)
nvalid = nepoch
ENDELSE
; Sort the epochs so that they are monotonic increasing.
mono = SORT (epoch [valid])
; Make sure evary time value is unique. Duplicate time values will
; result in unrecoverable error.
unique_epoch = UNIQ (mono)
IF N_ELEMENTS (unique_epoch) ne nvalid THEN BEGIN
MESSAGE, 'Duplicate time values found. Exiting.'
ENDIF
IF KEYWORD_SET (verbose) THEN BEGIN
print, "Number of EPOCHS: ", nepoch
print, "Number of valid EPOCHS: ", nvalid
print, "Number of unique EPOCHS: ", N_ELEMENTS (unique_epoch)
ENDIF
; See if the epochs are correctly ordered and if they are all valid.
; If this is the case then we are done.
IF (nvalid eq nepoch) && ARRAY_EQUAL (mono, INDGEN (nepoch)) THEN CONTINUE
; Update each variable that depends on this variable, starting with this
; variable.
sink = traverse_depends (epochs [n], mono, valid, s, desc)
; stop
ENDFOR
END
; Main wrapper function.
FUNCTION rectify_data, in, VERBOSE=verbose
work = in
desc = create_descriptors (work)
sort_time_values, work, desc, VERBOSE=verbose
RETURN, work
END
; Alternative interface.
PRO rectify_data, s, VERBOSE=verbose
s = rectify_data (s, VERBOSE=verbose)
END