    001/*
    002 * Licensed to the Apache Software Foundation (ASF) under one or more
    003 * contributor license agreements.  See the NOTICE file distributed with
    004 * this work for additional information regarding copyright ownership.
    005 * The ASF licenses this file to You under the Apache License, Version 2.0
    006 * (the "License"); you may not use this file except in compliance with
    007 * the License.  You may obtain a copy of the License at
    008 *
    009 *      http://www.apache.org/licenses/LICENSE-2.0
    010 *
    011 * Unless required by applicable law or agreed to in writing, software
    012 * distributed under the License is distributed on an "AS IS" BASIS,
    013 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    014 * See the License for the specific language governing permissions and
    015 * limitations under the License.
    016 */
    017
    018package org.apache.commons.net.ftp.parser;
    019
    020import java.text.ParseException;
    021import java.util.List;
    022
    023import org.apache.commons.net.ftp.FTPClientConfig;
    024import org.apache.commons.net.ftp.FTPFile;
    025
    026/**
    027 * Implementation of FTPFileEntryParser and FTPFileListParser for IBM zOS/MVS
    028 * Systems.
    029 *
    030 * @see org.apache.commons.net.ftp.FTPFileEntryParser FTPFileEntryParser (for
    031 *      usage instructions)
    032 */
    033public class MVSFTPEntryParser extends ConfigurableFTPFileEntryParserImpl {
    034
    035    static final int UNKNOWN_LIST_TYPE = -1;
    036    static final int FILE_LIST_TYPE = 0;
    037    static final int MEMBER_LIST_TYPE = 1;
    038    static final int UNIX_LIST_TYPE = 2;
    039    static final int JES_LEVEL_1_LIST_TYPE = 3;
    040    static final int JES_LEVEL_2_LIST_TYPE = 4;
    041
    042    private int isType = UNKNOWN_LIST_TYPE;
    043
    044    /**
    045     * Fallback parser for Unix-style listings
    046     */
    047    private UnixFTPEntryParser unixFTPEntryParser;
    048
    049    /**
    050     * Dates are ignored for file lists, but are used for member lists where
    051     * possible
    052     */
    053    static final String DEFAULT_DATE_FORMAT = "yyyy/MM/dd HH:mm"; // 2001/09/18
    054                                                                    // 13:52
    055
    056    /**
    057     * Matches these entries:
    058     *
    059     * <pre>
    060     *  Volume Unit    Referred Ext Used Recfm Lrecl BlkSz Dsorg Dsname
    061     *  B10142 3390   2006/03/20  2   31  F       80    80  PS   MDI.OKL.WORK
    062     * </pre>
    063     *
    064     * @see <a href=
    065     *      "https://www.ibm.com/support/knowledgecenter/zosbasics/com.ibm.zos.zconcepts/zconcepts_159.htm">Data
    066     *      set record formats</a>
    067     */
    068    static final String FILE_LIST_REGEX = "\\S+\\s+" + // volume
    069                                                                // ignored
    070            "\\S+\\s+" + // unit - ignored
    071            "\\S+\\s+" + // access date - ignored
    072            "\\S+\\s+" + // extents -ignored
    073            // If the values are too large, the fields may be merged (NET-639)
    074            "(?:\\S+\\s+)?" + // used - ignored
    075            "(?:F|FB|V|VB|U)\\s+" + // recfm - F[B], V[B], U
    076            "\\S+\\s+" + // logical record length -ignored
    077            "\\S+\\s+" + // block size - ignored
    078            "(PS|PO|PO-E)\\s+" + // Dataset organisation. Many exist
    079            // but only support: PS, PO, PO-E
    080            "(\\S+)\\s*"; // Dataset Name (file name)
    081
    082    /**
    083     * Matches these entries:
    084     * <pre>
    085     *   Name      VV.MM   Created       Changed      Size  Init   Mod   Id
    086     *   TBSHELF   01.03 2002/09/12 2002/10/11 09:37    11    11     0 KIL001
    087     * </pre>
    088     */
    089    static final String MEMBER_LIST_REGEX = "(\\S+)\\s+" + // name
    090            "\\S+\\s+" + // version, modification (ignored)
    091            "\\S+\\s+" + // create date (ignored)
    092            "(\\S+)\\s+" + // modification date
    093            "(\\S+)\\s+" + // modification time
    094            "\\S+\\s+" + // size in lines (ignored)
    095            "\\S+\\s+" + // size in lines at creation(ignored)
    096            "\\S+\\s+" + // lines modified (ignored)
    097            "\\S+\\s*"; // id of user who modified (ignored)
    098
    099    /**
    100     * Matches these entries, note: no header:
    101     * <pre>
    102     *   IBMUSER1  JOB01906  OUTPUT    3 Spool Files
    103     *   012345678901234567890123456789012345678901234
    104     *             1         2         3         4
    105     * </pre>
    106     */
    107    static final String JES_LEVEL_1_LIST_REGEX =
    108            "(\\S+)\\s+" + // job name ignored
    109            "(\\S+)\\s+" + // job number
    110            "(\\S+)\\s+" + // job status (OUTPUT,INPUT,ACTIVE)
    111            "(\\S+)\\s+" + // number of spool files
    112            "(\\S+)\\s+" + // Text "Spool" ignored
    113            "(\\S+)\\s*" // Text "Files" ignored
    114    ;
    115
    116    /**
    117     * JES INTERFACE LEVEL 2 parser
    118     * Matches these entries:
    119     * <pre>
    120     * JOBNAME  JOBID    OWNER    STATUS CLASS
    121     * IBMUSER1 JOB01906 IBMUSER  OUTPUT A        RC=0000 3 spool files
    122     * IBMUSER  TSU01830 IBMUSER  OUTPUT TSU      ABEND=522 3 spool files
    123     * </pre>
    124     * Sample output from FTP session:
    125     * <pre>
    126     * ftp> quote site filetype=jes
    127     * 200 SITE command was accepted
    128     * ftp> ls
    129     * 200 Port request OK.
    130     * 125 List started OK for JESJOBNAME=IBMUSER*, JESSTATUS=ALL and JESOWNER=IBMUSER
    131     * JOBNAME  JOBID    OWNER    STATUS CLASS
    132     * IBMUSER1 JOB01906 IBMUSER  OUTPUT A        RC=0000 3 spool files
    133     * IBMUSER  TSU01830 IBMUSER  OUTPUT TSU      ABEND=522 3 spool files
    134     * 250 List completed successfully.
    135     * ftp> ls job01906
    136     * 200 Port request OK.
    137     * 125 List started OK for JESJOBNAME=IBMUSER*, JESSTATUS=ALL and JESOWNER=IBMUSER
    138     * JOBNAME  JOBID    OWNER    STATUS CLASS
    139     * IBMUSER1 JOB01906 IBMUSER  OUTPUT A        RC=0000
    140     * --------
    141     * ID  STEPNAME PROCSTEP C DDNAME   BYTE-COUNT
    142     * 001 JES2              A JESMSGLG       858
    143     * 002 JES2              A JESJCL         128
    144     * 003 JES2              A JESYSMSG       443
    145     * 3 spool files
    146     * 250 List completed successfully.
    147     * </pre>
    148     */
    149
    150    static final String JES_LEVEL_2_LIST_REGEX =
    151            "(\\S+)\\s+" + // job name ignored
    152            "(\\S+)\\s+" + // job number
    153            "(\\S+)\\s+" + // owner ignored
    154            "(\\S+)\\s+" + // job status (OUTPUT,INPUT,ACTIVE) ignored
    155            "(\\S+)\\s+" + // job class ignored
    156            "(\\S+).*" // rest ignored
    157    ;
    158
    159    /*
    160     * ---------------------------------------------------------------------
    161     * Very brief and incomplete description of the zOS/MVS-file system. (Note:
    162     * "zOS" is the operating system on the mainframe, and is the new name for
    163     * MVS)
    164     *
    165     * The file system on the mainframe does not have hierarchal structure as for
    166     * example the unix file system. For a more comprehensive description, please
    167     * refer to the IBM manuals
    168     *
    169     * @LINK:
    170     * http://publibfp.boulder.ibm.com/cgi-bin/bookmgr/BOOKS/dgt2d440/CONTENTS
    171     *
    172     *
    173     * Dataset names =============
    174     *
    175     * A dataset name consist of a number of qualifiers separated by '.', each
    176     * qualifier can be at most 8 characters, and the total length of a dataset
    177     * can be max 44 characters including the dots.
    178     *
    179     *
    180     * Dataset organisation ====================
    181     *
    182     * A dataset represents a piece of storage allocated on one or more disks.
    183     * The structure of the storage is described with the field dataset
    184     * organinsation (DSORG). There are a number of dataset organisations, but
    185     * only two are usable for FTP transfer.
    186     *
    187     * DSORG: PS: sequential, or flat file PO: partitioned dataset PO-E:
    188     * extended partitioned dataset
    189     *
    190     * The PS file is just a flat file, as you would find it on the unix file
    191     * system.
    192     *
    193     * The PO and PO-E files, can be compared to a single level directory
    194     * structure. A PO file consist of a number of dataset members, or files if
    195     * you will. It is possible to CD into the file, and to retrieve the
    196     * individual members.
    197     *
    198     *
    199     * Dataset record format =====================
    200     *
    201     * The physical layout of the dataset is described on the dataset itself.
    202     * There are a number of record formats (RECFM), but just a few is relavant
    203     * for the FTP transfer.
    204     *
    205     * Any one beginning with either F or V can safely used by FTP transfer. All
    206     * others should only be used with great care.
    207     * F means a fixed number of records per
    208     * allocated storage, and V means a variable number of records.
    209     *
    210     *
    211     * Other notes ===========
    212     *
    213     * The file system supports automatically backup and retrieval of datasets.
    214     * If a file is backed up, the ftp LIST command will return: ARCIVE Not
    215     * Direct Access Device KJ.IOP998.ERROR.PL.UNITTEST
    216     *
    217     *
    218     * Implementation notes ====================
    219     *
    220     * Only datasets that have dsorg PS, PO or PO-E and have recfm beginning
    221     * with F or V or U, is fully parsed.
    222     *
    223     * The following fields in FTPFile is used: FTPFile.Rawlisting: Always set.
    224     * FTPFile.Type: DIRECTORY_TYPE or FILE_TYPE or UNKNOWN FTPFile.Name: name
    225     * FTPFile.Timestamp: change time or null
    226     *
    227     *
    228     *
    229     * Additional information ======================
    230     *
    231     * The MVS ftp server supports a number of features via the FTP interface.
    232     * The features are controlled with the FTP command quote site filetype=<SEQ|JES|DB2>
    233     * SEQ is the default and used for normal file transfer JES is used to
    234     * interact with the Job Entry Subsystem (JES) similar to a job scheduler
    235     * DB2 is used to interact with a DB2 subsystem
    236     *
    237     * This parser supports SEQ and JES.
    238     *
    239     *
    240     *
    241     *
    242     *
    243     *
    244     */
    245
    246    /**
    247     * The sole constructor for a MVSFTPEntryParser object.
    248     *
    249     */
    250    public MVSFTPEntryParser() {
    251        super(""); // note the regex is set in preParse.
    252        super.configure(null); // configure parser with default configurations
    253    }
    254
    255    /**
    256     * Parses a line of an z/OS - MVS FTP server file listing and converts it
    257     * into a usable format in the form of an <code> FTPFile </code> instance.
    258     * If the file listing line doesn't describe a file, then
    259     * <code> null </code> is returned. Otherwise a <code> FTPFile </code>
    260     * instance representing the file is returned.
    261     *
    262     * @param entry
    263     *            A line of text from the file listing
    264     * @return An FTPFile instance corresponding to the supplied entry
    265     */
    266    @Override
    267    public FTPFile parseFTPEntry(final String entry) {
    268        if (isType == FILE_LIST_TYPE) {
    269            return parseFileList(entry);
    270        } else if (isType == MEMBER_LIST_TYPE) {
    271            return parseMemberList(entry);
    272        } else if (isType == UNIX_LIST_TYPE) {
    273             return unixFTPEntryParser.parseFTPEntry(entry);
    274        } else if (isType == JES_LEVEL_1_LIST_TYPE) {
    275            return parseJeslevel1List(entry);
    276        } else if (isType == JES_LEVEL_2_LIST_TYPE) {
    277            return parseJeslevel2List(entry);
    278        }
    279
    280        return null;
    281    }
    282
    283    /**
    284     * Parse entries representing a dataset list. Only datasets with DSORG PS or
    285     * PO or PO-E and with RECFM F[B], V[B], U will be parsed.
    286     *
    287     * Format of ZOS/MVS file list: 1 2 3 4 5 6 7 8 9 10 Volume Unit Referred
    288     * Ext Used Recfm Lrecl BlkSz Dsorg Dsname B10142 3390 2006/03/20 2 31 F 80
    289     * 80 PS MDI.OKL.WORK ARCIVE Not Direct Access Device
    290     * KJ.IOP998.ERROR.PL.UNITTEST B1N231 3390 2006/03/20 1 15 VB 256 27998 PO
    291     * PLU B1N231 3390 2006/03/20 1 15 VB 256 27998 PO-E PLB
    292     *
    293     * ----------------------------------- Group within Regex [1] Volume [2]
    294     * Unit [3] Referred [4] Ext: number of extents [5] Used [6] Recfm: Record
    295     * format [7] Lrecl: Logical record length [8] BlkSz: Block size [9] Dsorg:
    296     * Dataset organisation. Many exists but only support: PS, PO, PO-E [10]
    297     * Dsname: Dataset name
    298     *
    299     * Note: When volume is ARCIVE, it means the dataset is stored somewhere in
    300     * a tape archive. These entries is currently not supported by this parser.
    301     * A null value is returned.
    302     *
    303     * @param entry zosDirectoryEntry
    304     * @return null: entry was not parsed.
    305     */
    306    private FTPFile parseFileList(final String entry) {
    307        if (matches(entry)) {
    308            final FTPFile file = new FTPFile();
    309            file.setRawListing(entry);
    310            final String name = group(2);
    311            final String dsorg = group(1);
    312            file.setName(name);
    313
    314            // DSORG
    315            if ("PS".equals(dsorg)) {
    316                file.setType(FTPFile.FILE_TYPE);
    317            }
    318            else if ("PO".equals(dsorg) || "PO-E".equals(dsorg)) {
    319                // regex already ruled out anything other than PO or PO-E
    320                file.setType(FTPFile.DIRECTORY_TYPE);
    321            }
    322            else {
    323                return null;
    324            }
    325
    326            return file;
    327        }
    328
    329        return null;
    330    }
    331
    332    /**
    333     * Parse entries within a partitioned dataset.
    334     *
    335     * Format of a memberlist within a PDS:
    336     * <pre>
    337     *    0         1        2          3        4     5     6      7    8
    338     *   Name      VV.MM   Created       Changed      Size  Init   Mod   Id
    339     *   TBSHELF   01.03 2002/09/12 2002/10/11 09:37    11    11     0 KIL001
    340     *   TBTOOL    01.12 2002/09/12 2004/11/26 19:54    51    28     0 KIL001
    341     *
    342     * -------------------------------------------
    343     * [1] Name
    344     * [2] VV.MM: Version . modification
    345     * [3] Created: yyyy / MM / dd
    346     * [4,5] Changed: yyyy / MM / dd HH:mm
    347     * [6] Size: number of lines
    348     * [7] Init: number of lines when first created
    349     * [8] Mod: number of modified lines a last save
    350     * [9] Id: User id for last update
    351     * </pre>
    352     *
    353     * @param entry zosDirectoryEntry
    354     * @return null: entry was not parsed.
    355     */
    356    private FTPFile parseMemberList(final String entry) {
    357        final FTPFile file = new FTPFile();
    358        if (matches(entry)) {
    359            file.setRawListing(entry);
    360            final String name = group(1);
    361            final String datestr = group(2) + " " + group(3);
    362            file.setName(name);
    363            file.setType(FTPFile.FILE_TYPE);
    364            try {
    365                file.setTimestamp(super.parseTimestamp(datestr));
    366            } catch (final ParseException e) {
    367                // just ignore parsing errors.
    368                // TODO check this is ok
    369                // Drop thru to try simple parser
    370            }
    371            return file;
    372        }
    373
    374        /*
    375         * Assigns the name to the first word of the entry. Only to be used from a
    376         * safe context, for example from a memberlist, where the regex for some
    377         * reason fails. Then just assign the name field of FTPFile.
    378         */
    379        if (entry != null && !entry.trim().isEmpty()) {
    380            file.setRawListing(entry);
    381            final String name = entry.split(" ")[0];
    382            file.setName(name);
    383            file.setType(FTPFile.FILE_TYPE);
    384            return file;
    385        }
    386        return null;
    387    }
    388
    389    /**
    390     * Matches these entries, note: no header:
    391     * <pre>
    392     * [1]      [2]      [3]   [4] [5]
    393     * IBMUSER1 JOB01906 OUTPUT 3 Spool Files
    394     * 012345678901234567890123456789012345678901234
    395     *           1         2         3         4
    396     * -------------------------------------------
    397     * Group in regex
    398     * [1] Job name
    399     * [2] Job number
    400     * [3] Job status (INPUT,ACTIVE,OUTPUT)
    401     * [4] Number of sysout files
    402     * [5] The string "Spool Files"
    403     *</pre>
    404     *
    405     * @param entry zosDirectoryEntry
    406     * @return null: entry was not parsed.
    407     */
    408    private FTPFile parseJeslevel1List(final String entry) {
    409        if (matches(entry)) {
    410            final FTPFile file = new FTPFile();
    411            if (group(3).equalsIgnoreCase("OUTPUT")) {
    412                file.setRawListing(entry);
    413                final String name = group(2); /* Job Number, used by GET */
    414                file.setName(name);
    415                file.setType(FTPFile.FILE_TYPE);
    416                return file;
    417            }
    418        }
    419
    420        return null;
    421    }
    422
    423    /**
    424     * Matches these entries:
    425     * <pre>
    426     * [1]      [2]      [3]     [4]    [5]
    427     * JOBNAME  JOBID    OWNER   STATUS CLASS
    428     * IBMUSER1 JOB01906 IBMUSER OUTPUT A       RC=0000 3 spool files
    429     * IBMUSER  TSU01830 IBMUSER OUTPUT TSU     ABEND=522 3 spool files
    430     * 012345678901234567890123456789012345678901234
    431     *           1         2         3         4
    432     * -------------------------------------------
    433     * Group in regex
    434     * [1] Job name
    435     * [2] Job number
    436     * [3] Owner
    437     * [4] Job status (INPUT,ACTIVE,OUTPUT)
    438     * [5] Job Class
    439     * [6] The rest
    440     * </pre>
    441     *
    442     * @param entry zosDirectoryEntry
    443     * @return null: entry was not parsed.
    444     */
    445    private FTPFile parseJeslevel2List(final String entry) {
    446        if (matches(entry)) {
    447            final FTPFile file = new FTPFile();
    448            if (group(4).equalsIgnoreCase("OUTPUT")) {
    449                file.setRawListing(entry);
    450                final String name = group(2); /* Job Number, used by GET */
    451                file.setName(name);
    452                file.setType(FTPFile.FILE_TYPE);
    453                return file;
    454            }
    455        }
    456
    457        return null;
    458    }
    459
    460    /**
    461     * preParse is called as part of the interface. Per definition is is called
    462     * before the parsing takes place.
    463     * Three kind of lists is recognize:
    464     * z/OS-MVS File lists
    465     * z/OS-MVS Member lists
    466     * unix file lists
    467     * @since 2.0
    468     */
    469    @Override
    470    public List<String> preParse(final List<String> orig) {
    471        // simply remove the header line. Composite logic will take care of the
    472        // two different types of
    473        // list in short order.
    474        if (orig != null && !orig.isEmpty()) {
    475            final String header = orig.get(0);
    476            if (header.indexOf("Volume") >= 0 && header.indexOf("Dsname") >= 0) {
    477                setType(FILE_LIST_TYPE);
    478                super.setRegex(FILE_LIST_REGEX);
    479            } else if (header.indexOf("Name") >= 0 && header.indexOf("Id") >= 0) {
    480                setType(MEMBER_LIST_TYPE);
    481                super.setRegex(MEMBER_LIST_REGEX);
    482            } else if (header.indexOf("total") == 0) {
    483                setType(UNIX_LIST_TYPE);
    484                unixFTPEntryParser = new UnixFTPEntryParser();
    485            } else if (header.indexOf("Spool Files") >= 30) {
    486                setType(JES_LEVEL_1_LIST_TYPE);
    487                super.setRegex(JES_LEVEL_1_LIST_REGEX);
    488            } else if (header.indexOf("JOBNAME") == 0
    489                    && header.indexOf("JOBID") > 8) {// header contains JOBNAME JOBID OWNER // STATUS CLASS
    490                setType(JES_LEVEL_2_LIST_TYPE);
    491                super.setRegex(JES_LEVEL_2_LIST_REGEX);
    492            } else {
    493                setType(UNKNOWN_LIST_TYPE);
    494            }
    495
    496            if (isType != JES_LEVEL_1_LIST_TYPE) { // remove header is necessary
    497                orig.remove(0);
    498            }
    499        }
    500
    501        return orig;
    502    }
    503
    504    /**
    505     * Explicitly set the type of listing being processed.
    506     * @param type The listing type.
    507     */
    508    void setType(final int type) {
    509        isType = type;
    510    }
    511
    512    /*
    513     * @return
    514     */
    515    @Override
    516    protected FTPClientConfig getDefaultConfiguration() {
    517        return new FTPClientConfig(FTPClientConfig.SYST_MVS,
    518                DEFAULT_DATE_FORMAT, null);
    519    }
    520
    521}