    001/*
    002 * Licensed to the Apache Software Foundation (ASF) under one or more
    003 * contributor license agreements.  See the NOTICE file distributed with
    004 * this work for additional information regarding copyright ownership.
    005 * The ASF licenses this file to You under the Apache License, Version 2.0
    006 * (the "License"); you may not use this file except in compliance with
    007 * the License.  You may obtain a copy of the License at
    008 *
    009 *      https://www.apache.org/licenses/LICENSE-2.0
    010 *
    011 * Unless required by applicable law or agreed to in writing, software
    012 * distributed under the License is distributed on an "AS IS" BASIS,
    013 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    014 * See the License for the specific language governing permissions and
    015 * limitations under the License.
    016 */
    017
    018package org.apache.commons.net.ftp.parser;
    019
    020import java.text.ParseException;
    021import java.util.List;
    022
    023import org.apache.commons.net.ftp.Configurable;
    024import org.apache.commons.net.ftp.FTPClientConfig;
    025import org.apache.commons.net.ftp.FTPFile;
    026import org.apache.commons.net.ftp.FTPFileEntryParser;
    027
    028/**
    029 * Implements {@link FTPFileEntryParser} and {@link Configurable} for IBM zOS/MVS Systems.
    030 *
    031 * @see FTPFileEntryParser Usage instructions.
    032 */
    033public class MVSFTPEntryParser extends ConfigurableFTPFileEntryParserImpl {
    034
    035    static final int UNKNOWN_LIST_TYPE = -1;
    036    static final int FILE_LIST_TYPE = 0;
    037    static final int MEMBER_LIST_TYPE = 1;
    038    static final int UNIX_LIST_TYPE = 2;
    039    static final int JES_LEVEL_1_LIST_TYPE = 3;
    040    static final int JES_LEVEL_2_LIST_TYPE = 4;
    041
    042    /**
    043     * Dates are ignored for file lists, but are used for member lists where possible
    044     */
    045    static final String DEFAULT_DATE_FORMAT = "yyyy/MM/dd HH:mm"; // 2001/09/18
    046                                                                  // 13:52
    047
    048    /**
    049     * Matches these entries:
    050     *
    051     * <pre>
    052     *  Volume Unit    Referred Ext Used Recfm Lrecl BlkSz Dsorg Dsname
    053     *  B10142 3390   2006/03/20  2   31  F       80    80  PS   MDI.OKL.WORK
    054     * </pre>
    055     *
    056     * @see <a href= "https://www.ibm.com/support/knowledgecenter/zosbasics/com.ibm.zos.zconcepts/zconcepts_159.htm">Data set record formats</a>
    057     */
    058    static final String FILE_LIST_REGEX = "\\S+\\s+" + // volume
    059                                                       // ignored
    060            "\\S+\\s+" + // unit - ignored
    061            "\\S+\\s+" + // access date - ignored
    062            "\\S+\\s+" + // extents -ignored
    063            // If the values are too large, the fields may be merged (NET-639)
    064            "(?:\\S+\\s+)?" + // used - ignored
    065            "\\S+\\s+" + // recfm - ignored
    066            "\\S+\\s+" + // logical record length -ignored
    067            "\\S+\\s+" + // block size - ignored
    068            "(PS|PO|PO-E)\\s+" + // Dataset organization. Many exist
    069            // but only support: PS, PO, PO-E
    070            "(\\S+)\\s*"; // Dataset Name (file name)
    071
    072    /**
    073     * Matches these entries:
    074     *
    075     * <pre>
    076     *   Name      VV.MM   Created       Changed      Size  Init   Mod   Id
    077     *   TBSHELF   01.03 2002/09/12 2002/10/11 09:37    11    11     0 KIL001
    078     * </pre>
    079     */
    080    static final String MEMBER_LIST_REGEX = "(\\S+)\\s+" + // name
    081            "\\S+\\s+" + // version, modification (ignored)
    082            "\\S+\\s+" + // create date (ignored)
    083            "(\\S+)\\s+" + // modification date
    084            "(\\S+)\\s+" + // modification time
    085            "\\S+\\s+" + // size in lines (ignored)
    086            "\\S+\\s+" + // size in lines at creation(ignored)
    087            "\\S+\\s+" + // lines modified (ignored)
    088            "\\S+\\s*"; // id of user who modified (ignored)
    089
    090    /**
    091     * Matches these entries, note: no header:
    092     *
    093     * <pre>
    094     *   IBMUSER1  JOB01906  OUTPUT    3 Spool Files
    095     *   012345678901234567890123456789012345678901234
    096     *             1         2         3         4
    097     * </pre>
    098     */
    099    static final String JES_LEVEL_1_LIST_REGEX = "(\\S+)\\s+" + // job name ignored
    100            "(\\S+)\\s+" + // job number
    101            "(\\S+)\\s+" + // job status (OUTPUT,INPUT,ACTIVE)
    102            "(\\S+)\\s+" + // number of spool files
    103            "(\\S+)\\s+" + // Text "Spool" ignored
    104            "(\\S+)\\s*" // Text "Files" ignored
    105    ;
    106
    107    /**
    108     * JES INTERFACE LEVEL 2 parser Matches these entries:
    109     *
    110     * <pre>
    111     * JOBNAME  JOBID    OWNER    STATUS CLASS
    112     * IBMUSER1 JOB01906 IBMUSER  OUTPUT A        RC=0000 3 spool files
    113     * IBMUSER  TSU01830 IBMUSER  OUTPUT TSU      ABEND=522 3 spool files
    114     * </pre>
    115     *
    116     * Sample output from FTP session:
    117     *
    118     * <pre>
    119     * ftp> quote site filetype=jes
    120     * 200 SITE command was accepted
    121     * ftp> ls
    122     * 200 Port request OK.
    123     * 125 List started OK for JESJOBNAME=IBMUSER*, JESSTATUS=ALL and JESOWNER=IBMUSER
    124     * JOBNAME  JOBID    OWNER    STATUS CLASS
    125     * IBMUSER1 JOB01906 IBMUSER  OUTPUT A        RC=0000 3 spool files
    126     * IBMUSER  TSU01830 IBMUSER  OUTPUT TSU      ABEND=522 3 spool files
    127     * 250 List completed successfully.
    128     * ftp> ls job01906
    129     * 200 Port request OK.
    130     * 125 List started OK for JESJOBNAME=IBMUSER*, JESSTATUS=ALL and JESOWNER=IBMUSER
    131     * JOBNAME  JOBID    OWNER    STATUS CLASS
    132     * IBMUSER1 JOB01906 IBMUSER  OUTPUT A        RC=0000
    133     * --------
    134     * ID  STEPNAME PROCSTEP C DDNAME   BYTE-COUNT
    135     * 001 JES2              A JESMSGLG       858
    136     * 002 JES2              A JESJCL         128
    137     * 003 JES2              A JESYSMSG       443
    138     * 3 spool files
    139     * 250 List completed successfully.
    140     * </pre>
    141     */
    142
    143    static final String JES_LEVEL_2_LIST_REGEX = "(\\S+)\\s+" + // job name ignored
    144            "(\\S+)\\s+" + // job number
    145            "(\\S+)\\s+" + // owner ignored
    146            "(\\S+)\\s+" + // job status (OUTPUT,INPUT,ACTIVE) ignored
    147            "(\\S+)\\s+" + // job class ignored
    148            "(\\S+).*" // rest ignored
    149    ;
    150
    151    private int isType = UNKNOWN_LIST_TYPE;
    152
    153    /**
    154     * Fallback parser for Unix-style listings
    155     */
    156    private UnixFTPEntryParser unixFTPEntryParser;
    157
    158    /*
    159     * --------------------------------------------------------------------- Very brief and incomplete description of the zOS/MVS-file system. (Note: "zOS" is
    160     * the operating system on the mainframe, and is the new name for MVS)
    161     *
    162     * The file system on the mainframe does not have hierarchical structure as for example the Unix file system. For a more comprehensive description,
    163     * please refer to the IBM manuals
    164     *
    165     * @LINK: https://publibfp.boulder.ibm.com/cgi-bin/bookmgr/BOOKS/dgt2d440/CONTENTS
    166     *
    167     *
    168     * Dataset names =============
    169     *
    170     * A dataset name consist of a number of qualifiers separated by '.', each qualifier can be at most 8 characters, and the total length of a dataset can be
    171     * max 44 characters including the dots.
    172     *
    173     *
    174     * Dataset organization ====================
    175     *
    176     * A dataset represents a piece of storage allocated on one or more disks. The structure of the storage is described with the field dataset organization
    177     * (DSORG). There are a number of dataset organizations, but only two are usable for FTP transfer.
    178     *
    179     * DSORG: PS: sequential, or flat file PO: partitioned dataset PO-E: extended partitioned dataset
    180     *
    181     * The PS file is just a flat file, as you would find it on the Unix file system.
    182     *
    183     * The PO and PO-E files, can be compared to a single level directory structure. A PO file consist of a number of dataset members, or files if you will. It
    184     * is possible to CD into the file, and to retrieve the individual members.
    185     *
    186     *
    187     * Dataset record format =====================
    188     *
    189     * The physical layout of the dataset is described on the dataset itself. There are a number of record formats (RECFM), but just a few is relevant for the
    190     * FTP transfer.
    191     *
    192     * Any one beginning with either F or V can safely be used by FTP transfer. All others should only be used with great care. F means a fixed number of
    193     * records per allocated storage, and V means a variable number of records.
    194     *
    195     *
    196     * Other notes ===========
    197     *
    198     * The file system supports automatically backup and retrieval of datasets. If a file is backed up, the ftp LIST command will return: ARCIVE Not Direct
    199     * Access Device KJ.IOP998.ERROR.PL.UNITTEST
    200     *
    201     *
    202     * Implementation notes ====================
    203     *
    204     * Only datasets that have dsorg PS, PO or PO-E and have recfm beginning with F or V or U, is fully parsed.
    205     *
    206     * The following fields in FTPFile is used: FTPFile.Rawlisting: Always set. FTPFile.Type: DIRECTORY_TYPE or FILE_TYPE or UNKNOWN FTPFile.Name: name
    207     * FTPFile.Timestamp: change time or null
    208     *
    209     *
    210     *
    211     * Additional information ======================
    212     *
    213     * The MVS ftp server supports a number of features via the FTP interface. The features are controlled with the FTP command quote site
    214     * filetype=<SEQ|JES|DB2> SEQ is the default and used for normal file transfer JES is used to interact with the Job Entry Subsystem (JES) similar to a job
    215     * scheduler DB2 is used to interact with a DB2 subsystem
    216     *
    217     * This parser supports SEQ and JES.
    218     */
    219
    220    /**
    221     * The sole constructor for a MVSFTPEntryParser object.
    222     */
    223    public MVSFTPEntryParser() {
    224        super(""); // note the regex is set in preParse.
    225        super.configure(null); // configure parser with default configurations
    226    }
    227
    228    @Override
    229    protected FTPClientConfig getDefaultConfiguration() {
    230        return new FTPClientConfig(FTPClientConfig.SYST_MVS, DEFAULT_DATE_FORMAT, null);
    231    }
    232
    233    /**
    234     * Parses entries representing a dataset list.
    235     * <pre>
    236     * Format of ZOS/MVS file list: 1 2 3 4 5 6 7 8 9 10
    237     * Volume Unit Referred Ext Used Recfm Lrecl BlkSz Dsorg Dsname
    238     * B10142 3390 2006/03/20 2 31 F 80 80 PS MDI.OKL.WORK
    239     * ARCIVE Not Direct Access Device KJ.IOP998.ERROR.PL.UNITTEST
    240     * B1N231 3390 2006/03/20 1 15 VB 256 27998 PO PLU
    241     * B1N231 3390 2006/03/20 1 15 VB 256 27998 PO-E PLB
    242     * Migrated                                                HLQ.DATASET.NAME
    243     * </pre>
    244     * <pre>
    245     * ----------------------------------- Group within Regex [1] Volume [2] Unit [3] Referred [4] Ext: number of extents [5] Used [6] Recfm: Record format [7]
    246     * Lrecl: Logical record length [8] BlkSz: Block size [9] Dsorg: Dataset organization. Many exists but only support: PS, PO, PO-E [10] Dsname: Dataset name
    247     * </pre>
    248     *
    249     * @param entry zosDirectoryEntry
    250     * @return null: entry was not parsed.
    251     */
    252    private FTPFile parseFileList(final String entry) {
    253        if (matches(entry)) {
    254            final FTPFile file = new FTPFile();
    255            file.setRawListing(entry);
    256            final String name = group(2);
    257            final String dsorg = group(1);
    258            file.setName(name);
    259
    260            // DSORG
    261            if ("PS".equals(dsorg)) {
    262                file.setType(FTPFile.FILE_TYPE);
    263            } else if ("PO".equals(dsorg) || "PO-E".equals(dsorg)) {
    264                // regex already ruled out anything other than PO or PO-E
    265                file.setType(FTPFile.DIRECTORY_TYPE);
    266            } else {
    267                return null;
    268            }
    269
    270            return file;
    271        }
    272
    273        final boolean migrated = entry.startsWith("Migrated");
    274        if (migrated || entry.startsWith("ARCIVE")) {
    275            // Type of file is unknown for migrated datasets
    276            final FTPFile file = new FTPFile();
    277            file.setRawListing(entry);
    278            file.setType(FTPFile.UNKNOWN_TYPE);
    279            file.setName(entry.split("\\s+")[migrated ? 1 : 5]);
    280            return file;
    281        }
    282
    283        return null;
    284    }
    285
    286    /**
    287     * Parses a line of a z/OS - MVS FTP server file listing and converts it into a usable format in the form of an {@code FTPFile} instance. If the
    288     * file listing line doesn't describe a file, then {@code null} is returned. Otherwise, a {@code FTPFile} instance representing the file is
    289     * returned.
    290     *
    291     * @param entry A line of text from the file listing
    292     * @return An FTPFile instance corresponding to the supplied entry
    293     */
    294    @Override
    295    public FTPFile parseFTPEntry(final String entry) {
    296        switch (isType) {
    297        case FILE_LIST_TYPE:
    298            return parseFileList(entry);
    299        case MEMBER_LIST_TYPE:
    300            return parseMemberList(entry);
    301        case UNIX_LIST_TYPE:
    302            return unixFTPEntryParser.parseFTPEntry(entry);
    303        case JES_LEVEL_1_LIST_TYPE:
    304            return parseJeslevel1List(entry);
    305        case JES_LEVEL_2_LIST_TYPE:
    306            return parseJeslevel2List(entry);
    307        default:
    308            break;
    309        }
    310
    311        return null;
    312    }
    313
    314    /**
    315     * Matches these entries, note: no header:
    316     *
    317     * <pre>
    318     * [1]      [2]      [3]   [4] [5]
    319     * IBMUSER1 JOB01906 OUTPUT 3 Spool Files
    320     * 012345678901234567890123456789012345678901234
    321     *           1         2         3         4
    322     * -------------------------------------------
    323     * Group in regex
    324     * [1] Job name
    325     * [2] Job number
    326     * [3] Job status (INPUT,ACTIVE,OUTPUT)
    327     * [4] Number of sysout files
    328     * [5] The string "Spool Files"
    329     * </pre>
    330     *
    331     * @param entry zosDirectoryEntry
    332     * @return null: entry was not parsed.
    333     */
    334    private FTPFile parseJeslevel1List(final String entry) {
    335        return parseJeslevelList(entry, 3);
    336    }
    337
    338    /**
    339     * Matches these entries:
    340     *
    341     * <pre>
    342     * [1]      [2]      [3]     [4]    [5]
    343     * JOBNAME  JOBID    OWNER   STATUS CLASS
    344     * IBMUSER1 JOB01906 IBMUSER OUTPUT A       RC=0000 3 spool files
    345     * IBMUSER  TSU01830 IBMUSER OUTPUT TSU     ABEND=522 3 spool files
    346     * 012345678901234567890123456789012345678901234
    347     *           1         2         3         4
    348     * -------------------------------------------
    349     * Group in regex
    350     * [1] Job name
    351     * [2] Job number
    352     * [3] Owner
    353     * [4] Job status (INPUT,ACTIVE,OUTPUT)
    354     * [5] Job Class
    355     * [6] The rest
    356     * </pre>
    357     *
    358     * @param entry zosDirectoryEntry
    359     * @return null: entry was not parsed.
    360     */
    361    private FTPFile parseJeslevel2List(final String entry) {
    362        return parseJeslevelList(entry, 4);
    363    }
    364
    365    private FTPFile parseJeslevelList(final String entry, final int matchNum) {
    366        if (matches(entry)) {
    367            final FTPFile file = new FTPFile();
    368            if (group(matchNum).equalsIgnoreCase("OUTPUT")) {
    369                file.setRawListing(entry);
    370                final String name = group(2); /* Job Number, used by GET */
    371                file.setName(name);
    372                file.setType(FTPFile.FILE_TYPE);
    373                return file;
    374            }
    375        }
    376        return null;
    377    }
    378
    379    /**
    380     * Parses entries within a partitioned dataset.
    381     *
    382     * Format of a memberlist within a PDS:
    383     *
    384     * <pre>
    385     *    0         1        2          3        4     5     6      7    8
    386     *   Name      VV.MM   Created       Changed      Size  Init   Mod   Id
    387     *   TBSHELF   01.03 2002/09/12 2002/10/11 09:37    11    11     0 KIL001
    388     *   TBTOOL    01.12 2002/09/12 2004/11/26 19:54    51    28     0 KIL001
    389     *
    390     * -------------------------------------------
    391     * [1] Name
    392     * [2] VV.MM: Version . modification
    393     * [3] Created: yyyy / MM / dd
    394     * [4,5] Changed: yyyy / MM / dd HH:mm
    395     * [6] Size: number of lines
    396     * [7] Init: number of lines when first created
    397     * [8] Mod: number of modified lines a last save
    398     * [9] Id: User id for last update
    399     * </pre>
    400     *
    401     * @param entry zosDirectoryEntry
    402     * @return null: entry was not parsed.
    403     */
    404    private FTPFile parseMemberList(final String entry) {
    405        final FTPFile file = new FTPFile();
    406        if (matches(entry)) {
    407            file.setRawListing(entry);
    408            final String name = group(1);
    409            final String datestr = group(2) + " " + group(3);
    410            file.setName(name);
    411            file.setType(FTPFile.FILE_TYPE);
    412            try {
    413                file.setTimestamp(super.parseTimestamp(datestr));
    414            } catch (final ParseException e) {
    415                // just ignore parsing errors.
    416                // TODO check this is ok
    417                // Drop thru to try simple parser
    418            }
    419            return file;
    420        }
    421        /*
    422         * Assigns the name to the first word of the entry. Only to be used from a safe context, for example from a memberlist, where the regex for some reason
    423         * fails. Then just assign the name field of FTPFile.
    424         */
    425        if (entry != null && !entry.trim().isEmpty()) {
    426            file.setRawListing(entry);
    427            final String name = entry.split(" ")[0];
    428            file.setName(name);
    429            file.setType(FTPFile.FILE_TYPE);
    430            return file;
    431        }
    432        return null;
    433    }
    434
    435    /**
    436     * Pre-parses is called as part of the interface. Per definition, it is called before the parsing takes place. Three kinds of lists are recognized:
    437     * <ul>
    438     *     <li>z/OS-MVS File lists,</li>
    439     *     <li>z/OS-MVS Member lists,</li>
    440     *     <li>Unix file lists.</li>
    441     * </ul>
    442     * @since 2.0
    443     */
    444    @Override
    445    public List<String> preParse(final List<String> orig) {
    446        // simply remove the header line. Composite logic will take care of the
    447        // two different types of
    448        // list in short order.
    449        if (orig != null && !orig.isEmpty()) {
    450            final String header = orig.get(0);
    451            if (header.contains("Volume") && header.contains("Dsname")) {
    452                setType(FILE_LIST_TYPE);
    453                super.setRegex(FILE_LIST_REGEX);
    454            } else if (header.contains("Name") && header.contains("Id")) {
    455                setType(MEMBER_LIST_TYPE);
    456                super.setRegex(MEMBER_LIST_REGEX);
    457            } else if (header.startsWith("total")) {
    458                setType(UNIX_LIST_TYPE);
    459                unixFTPEntryParser = new UnixFTPEntryParser();
    460            } else if (header.indexOf("Spool Files") >= 30) {
    461                setType(JES_LEVEL_1_LIST_TYPE);
    462                super.setRegex(JES_LEVEL_1_LIST_REGEX);
    463            } else if (header.startsWith("JOBNAME") && header.indexOf("JOBID") > 8) { // header contains JOBNAME JOBID OWNER // STATUS CLASS
    464                setType(JES_LEVEL_2_LIST_TYPE);
    465                super.setRegex(JES_LEVEL_2_LIST_REGEX);
    466            } else {
    467                setType(UNKNOWN_LIST_TYPE);
    468            }
    469            if (isType != JES_LEVEL_1_LIST_TYPE) { // remove header is necessary
    470                orig.remove(0);
    471            }
    472        }
    473        return orig;
    474    }
    475
    476    /**
    477     * Sets the type of listing being processed.
    478     *
    479     * @param type The listing type.
    480     */
    481    void setType(final int type) {
    482        isType = type;
    483    }
    484
    485}