#!/data_systems/opt/bin/python3 # !/Users/wbarrett/opt/anaconda3/bin/python """ rsync_aim3_l2a_images_to_aim A script to use rsync to move updated l2a images from /aim3/user_data/france/ to /aim/data/cips Created August 2020 @author: Bill Barrett """ import os.path import time import re import traceback from aimpi_shell_execution import AimPiShellExecution from season_and_version_utils import SeasonAndVersionUtils # The user name for the all aim operations AIM_USER = 'aimpro' # The server name for the /aim3/user_data/france/ files AIM_SERVER = 'aimcips-gpu' # The base directory for the /aim3/user_data/france/ files REMOTE_BASE_DIR = '/aim3/user_data/france/' # The base directory on the local (AIM CIPS) system SERVER_BASE_DIR = '/aim/data/cips' # AIM PMC data is categorized by season # Older RAA data is currently housed by season SEASONS = ['north', 'south'] # The year AIM was launched FIRST_AIM_YEAR = 2007 # last year of reprocessing LAST_REPROCESSED_YEAR = 2019 # script run on AIM server to find current L2A ping files, # ignoring any error output (2>/dev/null) L2A_SEARCH_SCRIPT = '/aim/sds/common/scripts/get_level_2a_ping_file_dates.sh' + \ ' 2>/dev/null' # AIM CIPS L2A files should have the date in the following format # as part of the fie name AIM_DATE_RE = re.compile(r'.*(?P\d{4}\-[0-3]\d{2}).*') # Subdirectory for level 2a scene images LEVEL_VERSION_REVISION_PATH = 'level_2a/ver_01.10/rev_05/scene_images' class RsyncAim3L2A: """ Use rsync to move RAA data reprocessed by Jeff France from /ai,/user_data/france to /aim/data/cips """ def __enter__(self): """ Walk through the years, transferring the data from /aim3 to /aim """ for year in range(FIRST_AIM_YEAR, LAST_REPROCESSED_YEAR + 1): raa_server_base = os.path.join(SERVER_BASE_DIR, 'raa_{}'.format(year)) # Last month of the year is 12 for month in range(1,13): # Are there files available in the remote directory remote_path = os.path.join(REMOTE_BASE_DIR, str(year), '{:02d}'.format(month)) date_list = RsyncAim3L2A.get_file_dates_from_directory(remote_path) if date_list is None: continue # If there are files available, try to figure out where they should go for year_day_of_year in date_list: server_path = os.path.join(raa_server_base, '{:02d}'.format(month), LEVEL_VERSION_REVISION_PATH) server_path_found = False if (RsyncAim3L2A.server_directory_exists(server_path)): print('{0} {1}'.format(year_day_of_year, server_path)) server_path_found = True else: season = SeasonAndVersionUtils.get_season(year_day_of_year) server_path = os.path.join(SERVER_BASE_DIR, season, 'raa', LEVEL_VERSION_REVISION_PATH) if RsyncAim3L2A.server_directory_exists(server_path): if RsyncAim3L2A.is_file_date_in_directory(year_day_of_year, server_path): print('{0} {1}'.format(year_day_of_year, server_path)) server_path_found = True # PMC seasons can be tricky if not server_path_found: for previous in [True, False]: alternate_season = RsyncAim3L2A.get_previous_season(season, previous) server_path = os.path.join(SERVER_BASE_DIR, alternate_season, 'raa', LEVEL_VERSION_REVISION_PATH) if RsyncAim3L2A.server_directory_exists(server_path): if RsyncAim3L2A.is_file_date_in_directory(year_day_of_year, server_path): print('{0} {1}'.format(year_day_of_year, server_path)) server_path_found = True break if not server_path_found: print('no server path for {}'.format(year_day_of_year)) else: # rsync fails completely from the Python subprocess interface # scp only transfers a limited number of files from the Python # subprocess interface before it fails. So break things into # small steps for scene in range(1,32): scene_number = '{:02d}'.format(scene) command = 'scp -p ' + \ '{0}/cips_raa_2a_orbit_*{1}_v01.10_r05_scene{2}_*.png aimpro@aimcips:{3}/'.\ format(remote_path, year_day_of_year, scene_number, server_path) print(time.strftime("%Y-%m-%d %H:%M:%S")) print(command) result_code, output = AimPiShellExecution.execute_ssh_command(AIM_USER, AIM_SERVER, command) output_list = output.split('\n') print(time.strftime("%Y-%m-%d %H:%M:%S") + ' result_code: {0}'.format(result_code)) permission_denied = 0 for line in output_list: if line.rstrip(): print(line) if 'Permission denied' in line: permission_denied += 1 if permission_denied >= 1: break # break # break @staticmethod def get_previous_season(season, previous): """ Given a PMC season, find either the previous or the next season Parameters ---------- season: str a valid PMC season like 'north_2007' previous: boolean if True, get the previous season, if False the next Returns ------- : str the previous or next PMC seaosn """ season_list = season.split('_') ns = season_list[0] year = int(season_list[1]) if ns == 'north': other_season = 'south' else: other_season ='north' if previous: if other_season == 'south': year -= 1 else: if other_season == 'north': year += 1 return '{0}_{1}'.format(other_season, year) @staticmethod def server_directory_exists(server_path): """ Check to see if a given server directory exists Parameters ---------- server_path: str path for raa year directory Returns ------- : boolean True if directory exists, false otherwise """ command = 'ls {} 2>/dev/null | wc -l'.format(server_path) _, output = AimPiShellExecution.execute_ssh_command(AIM_USER , AIM_SERVER, command) file_count = int(output.rstrip()) return file_count > 0 @staticmethod def is_file_date_in_directory(year_day_of_year, server_path): """ Is the requested date from the file in the directory Parameters ---------- year_day_of_year : str something like '2007-144' server_path: str path for the remote directory Returns ------- : boolean True if the file is in the remote directory """ command = 'ls {0}/cips_raa_2a_orbit_*{1}*.png 2>/dev/null | wc -l'.format(server_path, year_day_of_year) result_code, output = AimPiShellExecution.execute_ssh_command(AIM_USER , AIM_SERVER, command) output_list = output.split('\n') if result_code == 0: return int(output_list[0]) > 0 else: return False @staticmethod def process_seasonal_directory(season, year): """ See what files are present in a given seasonal directory Parameters ---------- season : str PMC season, either 'north' or 'south' year : str four digit year as string """ remote_path = os.path.join(SERVER_BASE_DIR, '{0}_{1}'.format(season, year), 'raa/level_2a/ver_01.10/rev_05/scene_images') RsyncAim3L2A.get_file_dates_from_directory(remote_path) @staticmethod def process_raa_year_directory(self, server_path): """ See what files are present in an raa year directory Parameters ---------- server_path: str path for year on server """ for month in range(1,13): remote_path = os.path.join(server_path, '{:02d}'.format(month), 'level_2a/ver_01.10/rev_05/scene_images') self.get_file_dates_from_directory(remote_path) @staticmethod def get_file_dates_from_directory(remote_path): """ Get the available dates from an raa directory Parameters ---------- remote_path: str path to search for raa files Returns ------- : list of str or None a list of the available dates or None if none are available """ command = '{0} {1}'.format(L2A_SEARCH_SCRIPT, remote_path) print(command) result_code, output = AimPiShellExecution.execute_ssh_command(AIM_USER , AIM_SERVER, command) if result_code != 0: output = output.rstrip() if result_code == 1 and len(output) == 0: print('No files found') else: print('result: {0}, output: {1}'.format(result_code, output)) return None else: split_output = output.split('\n') # Skip any blank lines split_output = [x for x in split_output if x.rstrip()] return split_output def __exit__(self, exception_type, exception_value, traceback_info): """ Necessary if __enter__ is defined. Prints message on error exit. True for a normal exit, False if the exception_type is not None """ if exception_type is not None: print(time.strftime("%Y-%m-%d %H:%M:%S") + ' type: {0}, value: {1}'.format(exception_type, exception_value)) traceback.print_exception(exception_type, exception_value, traceback_info) return False # Comment to pass exception through return True if __name__ == '__main__': """ Make the class executable """ with RsyncAim3L2A() as AIM3_TO_AIM: pass