@InterfaceAudience.Private @InterfaceStability.Unstable public class JobHistoryUtils extends Object
| Modifier and Type | Field and Description | 
|---|---|
| static String | CONF_FILE_NAME_SUFFIXSuffix for configuration files. | 
| static org.apache.hadoop.fs.permission.FsPermission | HISTORY_DONE_DIR_PERMISSIONPermissions for the history done dir and derivatives. | 
| static org.apache.hadoop.fs.permission.FsPermission | HISTORY_DONE_DIR_UMASKUmask for the done dir and derivatives. | 
| static org.apache.hadoop.fs.permission.FsPermission | HISTORY_DONE_FILE_PERMISSION | 
| static org.apache.hadoop.fs.permission.FsPermission | HISTORY_INTERMEDIATE_DONE_DIR_PERMISSIONSPermissions for the intermediate done directory. | 
| static org.apache.hadoop.fs.permission.FsPermission | HISTORY_INTERMEDIATE_FILE_PERMISSIONS | 
| static org.apache.hadoop.fs.permission.FsPermission | HISTORY_STAGING_DIR_PERMISSIONSPermissions for the history staging dir while JobInProgress. | 
| static org.apache.hadoop.fs.permission.FsPermission | HISTORY_STAGING_USER_DIR_PERMISSIONSPermissions for the user directory under the staging directory. | 
| static String | JOB_HISTORY_FILE_EXTENSIONJob History File extension. | 
| static int | SERIAL_NUMBER_DIRECTORY_DIGITS | 
| static String | SUMMARY_FILE_NAME_SUFFIXSuffix for summary files. | 
| static Pattern | TIMESTAMP_DIR_PATTERN | 
| static String | TIMESTAMP_DIR_REGEX | 
| static int | VERSION | 
| Constructor and Description | 
|---|
| JobHistoryUtils() | 
| Modifier and Type | Method and Description | 
|---|---|
| static String | doneSubdirsBeforeSerialTail() | 
| static org.apache.hadoop.fs.PathFilter | getConfFileFilter()Gets a PathFilter which would match configuration files. | 
| static String | getConfiguredHistoryIntermediateDoneDirPrefix(org.apache.hadoop.conf.Configuration conf)Gets the configured directory prefix for intermediate done history files. | 
| static org.apache.hadoop.fs.permission.FsPermission | getConfiguredHistoryIntermediateUserDoneDirPermissions(org.apache.hadoop.conf.Configuration conf)Gets the configured directory permissions for the user directories in the
 directory of the intermediate done history files. | 
| static String | getConfiguredHistoryServerDoneDirPrefix(org.apache.hadoop.conf.Configuration conf)Gets the configured directory prefix for Done history files. | 
| static String | getConfiguredHistoryStagingDirPrefix(org.apache.hadoop.conf.Configuration conf,
                                    String jobId)Gets the configured directory prefix for In Progress history files. | 
| static List<org.apache.hadoop.fs.FileStatus> | getHistoryDirsForCleaning(org.apache.hadoop.fs.FileContext fc,
                         org.apache.hadoop.fs.Path root,
                         long cutoff)Looks for the dirs to clean. | 
| static org.apache.hadoop.fs.PathFilter | getHistoryFileFilter()Gets a PathFilter which would match job history file names. | 
| static String | getHistoryIntermediateDoneDirForUser(org.apache.hadoop.conf.Configuration conf)Gets the user directory for intermediate done history files. | 
| static String | getIntermediateConfFileName(JobId jobId)Get the done configuration file name for a job. | 
| static String | getIntermediateSummaryFileName(JobId jobId)Get the done summary file name for a job. | 
| static org.apache.hadoop.mapreduce.JobID | getJobIDFromHistoryFilePath(String pathString)Returns the jobId from a job history file name. | 
| static org.apache.hadoop.fs.Path | getPreviousJobHistoryPath(org.apache.hadoop.conf.Configuration conf,
                         org.apache.hadoop.yarn.api.records.ApplicationAttemptId applicationAttemptId) | 
| static org.apache.hadoop.fs.Path | getStagingConfFile(org.apache.hadoop.fs.Path logDir,
                  JobId jobId,
                  int attempt)Gets the conf file path for jobs in progress. | 
| static org.apache.hadoop.fs.Path | getStagingJobHistoryFile(org.apache.hadoop.fs.Path dir,
                        JobId jobId,
                        int attempt)Get the job history file path for non Done history files. | 
| static org.apache.hadoop.fs.Path | getStagingJobHistoryFile(org.apache.hadoop.fs.Path dir,
                        String jobId,
                        int attempt)Get the job history file path for non Done history files. | 
| static String | getTimestampPartFromPath(String path)Extracts the timstamp component from the path. | 
| static String | historyLogSubdirectory(JobId id,
                      String timestampComponent,
                      String serialNumberFormat)Gets the history subdirectory based on the jobId, timestamp and serial number format. | 
| static boolean | isValidJobHistoryFileName(String pathString)Checks whether the provided path string is a valid job history file. | 
| static int | jobSerialNumber(JobId id)Computes a serial number used as part of directory naming for the given jobId. | 
| static List<org.apache.hadoop.fs.FileStatus> | localGlobber(org.apache.hadoop.fs.FileContext fc,
            org.apache.hadoop.fs.Path root,
            String tail) | 
| static List<org.apache.hadoop.fs.FileStatus> | localGlobber(org.apache.hadoop.fs.FileContext fc,
            org.apache.hadoop.fs.Path root,
            String tail,
            org.apache.hadoop.fs.PathFilter filter) | 
| static List<org.apache.hadoop.fs.FileStatus> | localGlobber(org.apache.hadoop.fs.FileContext fc,
            org.apache.hadoop.fs.Path root,
            String tail,
            org.apache.hadoop.fs.PathFilter filter,
            AtomicBoolean hasFlatFiles) | 
| static String | serialNumberDirectoryComponent(JobId id,
                              String serialNumberFormat)Gets the serial number part of the path based on the jobId and serialNumber format. | 
| static boolean | shouldCreateNonUserDirectory(org.apache.hadoop.conf.Configuration conf) | 
| static String | timestampDirectoryComponent(long millisecondTime)Gets the timestamp component based on millisecond time. | 
public static final org.apache.hadoop.fs.permission.FsPermission HISTORY_STAGING_DIR_PERMISSIONS
public static final org.apache.hadoop.fs.permission.FsPermission HISTORY_STAGING_USER_DIR_PERMISSIONS
public static final org.apache.hadoop.fs.permission.FsPermission HISTORY_DONE_DIR_PERMISSION
public static final org.apache.hadoop.fs.permission.FsPermission HISTORY_DONE_FILE_PERMISSION
public static final org.apache.hadoop.fs.permission.FsPermission HISTORY_DONE_DIR_UMASK
public static final org.apache.hadoop.fs.permission.FsPermission HISTORY_INTERMEDIATE_DONE_DIR_PERMISSIONS
public static final org.apache.hadoop.fs.permission.FsPermission HISTORY_INTERMEDIATE_FILE_PERMISSIONS
public static final String CONF_FILE_NAME_SUFFIX
public static final String SUMMARY_FILE_NAME_SUFFIX
public static final String JOB_HISTORY_FILE_EXTENSION
public static final int VERSION
public static final int SERIAL_NUMBER_DIRECTORY_DIGITS
public static final String TIMESTAMP_DIR_REGEX
public static final Pattern TIMESTAMP_DIR_PATTERN
public static boolean isValidJobHistoryFileName(String pathString)
pathString - the path to be checked.public static org.apache.hadoop.mapreduce.JobID getJobIDFromHistoryFilePath(String pathString) throws IOException
pathString - the path string.IOException - if the filename format is invalid.public static org.apache.hadoop.fs.PathFilter getConfFileFilter()
PathFilter for matching conf files.public static org.apache.hadoop.fs.PathFilter getHistoryFileFilter()
PathFilter matching job history files.public static String getConfiguredHistoryStagingDirPrefix(org.apache.hadoop.conf.Configuration conf, String jobId) throws IOException
conf - the configuration for hte jobjobId - the id of the job the history file is for.IOExceptionpublic static String getConfiguredHistoryIntermediateDoneDirPrefix(org.apache.hadoop.conf.Configuration conf)
conf - public static org.apache.hadoop.fs.permission.FsPermission getConfiguredHistoryIntermediateUserDoneDirPermissions(org.apache.hadoop.conf.Configuration conf)
conf - The configuration objectpublic static String getConfiguredHistoryServerDoneDirPrefix(org.apache.hadoop.conf.Configuration conf)
conf - the configuration objectpublic static String getHistoryIntermediateDoneDirForUser(org.apache.hadoop.conf.Configuration conf) throws IOException
conf - the configuration objectIOExceptionpublic static boolean shouldCreateNonUserDirectory(org.apache.hadoop.conf.Configuration conf)
public static org.apache.hadoop.fs.Path getStagingJobHistoryFile(org.apache.hadoop.fs.Path dir,
                                                                 JobId jobId,
                                                                 int attempt)
public static org.apache.hadoop.fs.Path getStagingJobHistoryFile(org.apache.hadoop.fs.Path dir,
                                                                 String jobId,
                                                                 int attempt)
public static String getIntermediateConfFileName(JobId jobId)
jobId - the jobId.public static String getIntermediateSummaryFileName(JobId jobId)
jobId - the jobId.public static org.apache.hadoop.fs.Path getStagingConfFile(org.apache.hadoop.fs.Path logDir,
                                                           JobId jobId,
                                                           int attempt)
logDir - the log directory prefix.jobId - the jobId.attempt - attempt number for this job.public static String serialNumberDirectoryComponent(JobId id, String serialNumberFormat)
id - serialNumberFormat - public static String getTimestampPartFromPath(String path)
path - public static String historyLogSubdirectory(JobId id, String timestampComponent, String serialNumberFormat)
id - timestampComponent - serialNumberFormat - public static String timestampDirectoryComponent(long millisecondTime)
millisecondTime - public static String doneSubdirsBeforeSerialTail()
public static int jobSerialNumber(JobId id)
id - the jobId.public static List<org.apache.hadoop.fs.FileStatus> localGlobber(org.apache.hadoop.fs.FileContext fc, org.apache.hadoop.fs.Path root, String tail) throws IOException
IOExceptionpublic static List<org.apache.hadoop.fs.FileStatus> localGlobber(org.apache.hadoop.fs.FileContext fc, org.apache.hadoop.fs.Path root, String tail, org.apache.hadoop.fs.PathFilter filter) throws IOException
IOExceptionpublic static List<org.apache.hadoop.fs.FileStatus> localGlobber(org.apache.hadoop.fs.FileContext fc, org.apache.hadoop.fs.Path root, String tail, org.apache.hadoop.fs.PathFilter filter, AtomicBoolean hasFlatFiles) throws IOException
IOExceptionpublic static org.apache.hadoop.fs.Path getPreviousJobHistoryPath(org.apache.hadoop.conf.Configuration conf,
                                                                  org.apache.hadoop.yarn.api.records.ApplicationAttemptId applicationAttemptId)
                                                           throws IOException
IOExceptionpublic static List<org.apache.hadoop.fs.FileStatus> getHistoryDirsForCleaning(org.apache.hadoop.fs.FileContext fc, org.apache.hadoop.fs.Path root, long cutoff) throws IOException
fc - done dir FileContextroot - folder for completed jobscutoff - The cutoff for the max history ageIOExceptionCopyright © 2008–2023 Apache Software Foundation. All rights reserved.