    001/*
    002 * Licensed to the Apache Software Foundation (ASF) under one or more
    003 * contributor license agreements.  See the NOTICE file distributed with
    004 * this work for additional information regarding copyright ownership.
    005 * The ASF licenses this file to You under the Apache License, Version 2.0
    006 * (the "License"); you may not use this file except in compliance with
    007 * the License.  You may obtain a copy of the License at
    008 *
    009 *      http://www.apache.org/licenses/LICENSE-2.0
    010 *
    011 * Unless required by applicable law or agreed to in writing, software
    012 * distributed under the License is distributed on an "AS IS" BASIS,
    013 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    014 * See the License for the specific language governing permissions and
    015 * limitations under the License.
    016 */
    017
    018package org.apache.commons.net.ftp.parser;
    019
    020import java.text.ParseException;
    021import java.util.List;
    022
    023import org.apache.commons.net.ftp.FTPClientConfig;
    024import org.apache.commons.net.ftp.FTPFile;
    025
    026/**
    027 * Implementation of FTPFileEntryParser and FTPFileListParser for IBM zOS/MVS Systems.
    028 *
    029 * @see org.apache.commons.net.ftp.FTPFileEntryParser FTPFileEntryParser (for usage instructions)
    030 */
    031public class MVSFTPEntryParser extends ConfigurableFTPFileEntryParserImpl {
    032
    033    static final int UNKNOWN_LIST_TYPE = -1;
    034    static final int FILE_LIST_TYPE = 0;
    035    static final int MEMBER_LIST_TYPE = 1;
    036    static final int UNIX_LIST_TYPE = 2;
    037    static final int JES_LEVEL_1_LIST_TYPE = 3;
    038    static final int JES_LEVEL_2_LIST_TYPE = 4;
    039
    040    /**
    041     * Dates are ignored for file lists, but are used for member lists where possible
    042     */
    043    static final String DEFAULT_DATE_FORMAT = "yyyy/MM/dd HH:mm"; // 2001/09/18
    044                                                                  // 13:52
    045
    046    /**
    047     * Matches these entries:
    048     *
    049     * <pre>
    050     *  Volume Unit    Referred Ext Used Recfm Lrecl BlkSz Dsorg Dsname
    051     *  B10142 3390   2006/03/20  2   31  F       80    80  PS   MDI.OKL.WORK
    052     * </pre>
    053     *
    054     * @see <a href= "https://www.ibm.com/support/knowledgecenter/zosbasics/com.ibm.zos.zconcepts/zconcepts_159.htm">Data set record formats</a>
    055     */
    056    static final String FILE_LIST_REGEX = "\\S+\\s+" + // volume
    057                                                       // ignored
    058            "\\S+\\s+" + // unit - ignored
    059            "\\S+\\s+" + // access date - ignored
    060            "\\S+\\s+" + // extents -ignored
    061            // If the values are too large, the fields may be merged (NET-639)
    062            "(?:\\S+\\s+)?" + // used - ignored
    063            "\\S+\\s+" + // recfm - ignored
    064            "\\S+\\s+" + // logical record length -ignored
    065            "\\S+\\s+" + // block size - ignored
    066            "(PS|PO|PO-E)\\s+" + // Dataset organisation. Many exist
    067            // but only support: PS, PO, PO-E
    068            "(\\S+)\\s*"; // Dataset Name (file name)
    069
    070    /**
    071     * Matches these entries:
    072     *
    073     * <pre>
    074     *   Name      VV.MM   Created       Changed      Size  Init   Mod   Id
    075     *   TBSHELF   01.03 2002/09/12 2002/10/11 09:37    11    11     0 KIL001
    076     * </pre>
    077     */
    078    static final String MEMBER_LIST_REGEX = "(\\S+)\\s+" + // name
    079            "\\S+\\s+" + // version, modification (ignored)
    080            "\\S+\\s+" + // create date (ignored)
    081            "(\\S+)\\s+" + // modification date
    082            "(\\S+)\\s+" + // modification time
    083            "\\S+\\s+" + // size in lines (ignored)
    084            "\\S+\\s+" + // size in lines at creation(ignored)
    085            "\\S+\\s+" + // lines modified (ignored)
    086            "\\S+\\s*"; // id of user who modified (ignored)
    087
    088    /**
    089     * Matches these entries, note: no header:
    090     *
    091     * <pre>
    092     *   IBMUSER1  JOB01906  OUTPUT    3 Spool Files
    093     *   012345678901234567890123456789012345678901234
    094     *             1         2         3         4
    095     * </pre>
    096     */
    097    static final String JES_LEVEL_1_LIST_REGEX = "(\\S+)\\s+" + // job name ignored
    098            "(\\S+)\\s+" + // job number
    099            "(\\S+)\\s+" + // job status (OUTPUT,INPUT,ACTIVE)
    100            "(\\S+)\\s+" + // number of spool files
    101            "(\\S+)\\s+" + // Text "Spool" ignored
    102            "(\\S+)\\s*" // Text "Files" ignored
    103    ;
    104
    105    /**
    106     * JES INTERFACE LEVEL 2 parser Matches these entries:
    107     *
    108     * <pre>
    109     * JOBNAME  JOBID    OWNER    STATUS CLASS
    110     * IBMUSER1 JOB01906 IBMUSER  OUTPUT A        RC=0000 3 spool files
    111     * IBMUSER  TSU01830 IBMUSER  OUTPUT TSU      ABEND=522 3 spool files
    112     * </pre>
    113     *
    114     * Sample output from FTP session:
    115     *
    116     * <pre>
    117     * ftp> quote site filetype=jes
    118     * 200 SITE command was accepted
    119     * ftp> ls
    120     * 200 Port request OK.
    121     * 125 List started OK for JESJOBNAME=IBMUSER*, JESSTATUS=ALL and JESOWNER=IBMUSER
    122     * JOBNAME  JOBID    OWNER    STATUS CLASS
    123     * IBMUSER1 JOB01906 IBMUSER  OUTPUT A        RC=0000 3 spool files
    124     * IBMUSER  TSU01830 IBMUSER  OUTPUT TSU      ABEND=522 3 spool files
    125     * 250 List completed successfully.
    126     * ftp> ls job01906
    127     * 200 Port request OK.
    128     * 125 List started OK for JESJOBNAME=IBMUSER*, JESSTATUS=ALL and JESOWNER=IBMUSER
    129     * JOBNAME  JOBID    OWNER    STATUS CLASS
    130     * IBMUSER1 JOB01906 IBMUSER  OUTPUT A        RC=0000
    131     * --------
    132     * ID  STEPNAME PROCSTEP C DDNAME   BYTE-COUNT
    133     * 001 JES2              A JESMSGLG       858
    134     * 002 JES2              A JESJCL         128
    135     * 003 JES2              A JESYSMSG       443
    136     * 3 spool files
    137     * 250 List completed successfully.
    138     * </pre>
    139     */
    140
    141    static final String JES_LEVEL_2_LIST_REGEX = "(\\S+)\\s+" + // job name ignored
    142            "(\\S+)\\s+" + // job number
    143            "(\\S+)\\s+" + // owner ignored
    144            "(\\S+)\\s+" + // job status (OUTPUT,INPUT,ACTIVE) ignored
    145            "(\\S+)\\s+" + // job class ignored
    146            "(\\S+).*" // rest ignored
    147    ;
    148
    149    private int isType = UNKNOWN_LIST_TYPE;
    150
    151    /**
    152     * Fallback parser for Unix-style listings
    153     */
    154    private UnixFTPEntryParser unixFTPEntryParser;
    155
    156    /*
    157     * --------------------------------------------------------------------- Very brief and incomplete description of the zOS/MVS-file system. (Note: "zOS" is
    158     * the operating system on the mainframe, and is the new name for MVS)
    159     *
    160     * The file system on the mainframe does not have hierarchical structure as for example the unix file system. For a more comprehensive description,
    161     * please refer to the IBM manuals
    162     *
    163     * @LINK: http://publibfp.boulder.ibm.com/cgi-bin/bookmgr/BOOKS/dgt2d440/CONTENTS
    164     *
    165     *
    166     * Dataset names =============
    167     *
    168     * A dataset name consist of a number of qualifiers separated by '.', each qualifier can be at most 8 characters, and the total length of a dataset can be
    169     * max 44 characters including the dots.
    170     *
    171     *
    172     * Dataset organisation ====================
    173     *
    174     * A dataset represents a piece of storage allocated on one or more disks. The structure of the storage is described with the field dataset organisation
    175     * (DSORG). There are a number of dataset organisations, but only two are usable for FTP transfer.
    176     *
    177     * DSORG: PS: sequential, or flat file PO: partitioned dataset PO-E: extended partitioned dataset
    178     *
    179     * The PS file is just a flat file, as you would find it on the unix file system.
    180     *
    181     * The PO and PO-E files, can be compared to a single level directory structure. A PO file consist of a number of dataset members, or files if you will. It
    182     * is possible to CD into the file, and to retrieve the individual members.
    183     *
    184     *
    185     * Dataset record format =====================
    186     *
    187     * The physical layout of the dataset is described on the dataset itself. There are a number of record formats (RECFM), but just a few is relevant for the
    188     * FTP transfer.
    189     *
    190     * Any one beginning with either F or V can safely be used by FTP transfer. All others should only be used with great care. F means a fixed number of
    191     * records per allocated storage, and V means a variable number of records.
    192     *
    193     *
    194     * Other notes ===========
    195     *
    196     * The file system supports automatically backup and retrieval of datasets. If a file is backed up, the ftp LIST command will return: ARCIVE Not Direct
    197     * Access Device KJ.IOP998.ERROR.PL.UNITTEST
    198     *
    199     *
    200     * Implementation notes ====================
    201     *
    202     * Only datasets that have dsorg PS, PO or PO-E and have recfm beginning with F or V or U, is fully parsed.
    203     *
    204     * The following fields in FTPFile is used: FTPFile.Rawlisting: Always set. FTPFile.Type: DIRECTORY_TYPE or FILE_TYPE or UNKNOWN FTPFile.Name: name
    205     * FTPFile.Timestamp: change time or null
    206     *
    207     *
    208     *
    209     * Additional information ======================
    210     *
    211     * The MVS ftp server supports a number of features via the FTP interface. The features are controlled with the FTP command quote site
    212     * filetype=<SEQ|JES|DB2> SEQ is the default and used for normal file transfer JES is used to interact with the Job Entry Subsystem (JES) similar to a job
    213     * scheduler DB2 is used to interact with a DB2 subsystem
    214     *
    215     * This parser supports SEQ and JES.
    216     */
    217
    218    /**
    219     * The sole constructor for a MVSFTPEntryParser object.
    220     */
    221    public MVSFTPEntryParser() {
    222        super(""); // note the regex is set in preParse.
    223        super.configure(null); // configure parser with default configurations
    224    }
    225
    226    /*
    227     * @return
    228     */
    229    @Override
    230    protected FTPClientConfig getDefaultConfiguration() {
    231        return new FTPClientConfig(FTPClientConfig.SYST_MVS, DEFAULT_DATE_FORMAT, null);
    232    }
    233
    234    /**
    235     * Parses entries representing a dataset list.
    236     * <pre>
    237     * Format of ZOS/MVS file list: 1 2 3 4 5 6 7 8 9 10
    238     * Volume Unit Referred Ext Used Recfm Lrecl BlkSz Dsorg Dsname
    239     * B10142 3390 2006/03/20 2 31 F 80 80 PS MDI.OKL.WORK
    240     * ARCIVE Not Direct Access Device KJ.IOP998.ERROR.PL.UNITTEST
    241     * B1N231 3390 2006/03/20 1 15 VB 256 27998 PO PLU
    242     * B1N231 3390 2006/03/20 1 15 VB 256 27998 PO-E PLB
    243     * Migrated                                                HLQ.DATASET.NAME
    244     * </pre>
    245     * <pre>
    246     * ----------------------------------- Group within Regex [1] Volume [2] Unit [3] Referred [4] Ext: number of extents [5] Used [6] Recfm: Record format [7]
    247     * Lrecl: Logical record length [8] BlkSz: Block size [9] Dsorg: Dataset organisation. Many exists but only support: PS, PO, PO-E [10] Dsname: Dataset name
    248     * </pre>
    249     *
    250     * @param entry zosDirectoryEntry
    251     * @return null: entry was not parsed.
    252     */
    253    private FTPFile parseFileList(final String entry) {
    254        if (matches(entry)) {
    255            final FTPFile file = new FTPFile();
    256            file.setRawListing(entry);
    257            final String name = group(2);
    258            final String dsorg = group(1);
    259            file.setName(name);
    260
    261            // DSORG
    262            if ("PS".equals(dsorg)) {
    263                file.setType(FTPFile.FILE_TYPE);
    264            } else if ("PO".equals(dsorg) || "PO-E".equals(dsorg)) {
    265                // regex already ruled out anything other than PO or PO-E
    266                file.setType(FTPFile.DIRECTORY_TYPE);
    267            } else {
    268                return null;
    269            }
    270
    271            return file;
    272        }
    273
    274        final boolean migrated = entry.startsWith("Migrated");
    275        if (migrated || entry.startsWith("ARCIVE")) {
    276            // Type of file is unknown for migrated datasets
    277            final FTPFile file = new FTPFile();
    278            file.setRawListing(entry);
    279            file.setType(FTPFile.UNKNOWN_TYPE);
    280            file.setName(entry.split("\\s+")[migrated ? 1 : 5]);
    281            return file;
    282        }
    283
    284        return null;
    285    }
    286
    287    /**
    288     * Parses a line of a z/OS - MVS FTP server file listing and converts it into a usable format in the form of an <code> FTPFile </code> instance. If the
    289     * file listing line doesn't describe a file, then <code> null </code> is returned. Otherwise, a <code> FTPFile </code> instance representing the file is
    290     * returned.
    291     *
    292     * @param entry A line of text from the file listing
    293     * @return An FTPFile instance corresponding to the supplied entry
    294     */
    295    @Override
    296    public FTPFile parseFTPEntry(final String entry) {
    297        switch (isType) {
    298        case FILE_LIST_TYPE:
    299            return parseFileList(entry);
    300        case MEMBER_LIST_TYPE:
    301            return parseMemberList(entry);
    302        case UNIX_LIST_TYPE:
    303            return unixFTPEntryParser.parseFTPEntry(entry);
    304        case JES_LEVEL_1_LIST_TYPE:
    305            return parseJeslevel1List(entry);
    306        case JES_LEVEL_2_LIST_TYPE:
    307            return parseJeslevel2List(entry);
    308        default:
    309            break;
    310        }
    311
    312        return null;
    313    }
    314
    315    /**
    316     * Matches these entries, note: no header:
    317     *
    318     * <pre>
    319     * [1]      [2]      [3]   [4] [5]
    320     * IBMUSER1 JOB01906 OUTPUT 3 Spool Files
    321     * 012345678901234567890123456789012345678901234
    322     *           1         2         3         4
    323     * -------------------------------------------
    324     * Group in regex
    325     * [1] Job name
    326     * [2] Job number
    327     * [3] Job status (INPUT,ACTIVE,OUTPUT)
    328     * [4] Number of sysout files
    329     * [5] The string "Spool Files"
    330     * </pre>
    331     *
    332     * @param entry zosDirectoryEntry
    333     * @return null: entry was not parsed.
    334     */
    335    private FTPFile parseJeslevel1List(final String entry) {
    336        return parseJeslevelList(entry, 3);
    337    }
    338
    339    /**
    340     * Matches these entries:
    341     *
    342     * <pre>
    343     * [1]      [2]      [3]     [4]    [5]
    344     * JOBNAME  JOBID    OWNER   STATUS CLASS
    345     * IBMUSER1 JOB01906 IBMUSER OUTPUT A       RC=0000 3 spool files
    346     * IBMUSER  TSU01830 IBMUSER OUTPUT TSU     ABEND=522 3 spool files
    347     * 012345678901234567890123456789012345678901234
    348     *           1         2         3         4
    349     * -------------------------------------------
    350     * Group in regex
    351     * [1] Job name
    352     * [2] Job number
    353     * [3] Owner
    354     * [4] Job status (INPUT,ACTIVE,OUTPUT)
    355     * [5] Job Class
    356     * [6] The rest
    357     * </pre>
    358     *
    359     * @param entry zosDirectoryEntry
    360     * @return null: entry was not parsed.
    361     */
    362    private FTPFile parseJeslevel2List(final String entry) {
    363        return parseJeslevelList(entry, 4);
    364    }
    365
    366    private FTPFile parseJeslevelList(final String entry, final int matchNum) {
    367        if (matches(entry)) {
    368            final FTPFile file = new FTPFile();
    369            if (group(matchNum).equalsIgnoreCase("OUTPUT")) {
    370                file.setRawListing(entry);
    371                final String name = group(2); /* Job Number, used by GET */
    372                file.setName(name);
    373                file.setType(FTPFile.FILE_TYPE);
    374                return file;
    375            }
    376        }
    377        return null;
    378    }
    379
    380    /**
    381     * Parses entries within a partitioned dataset.
    382     *
    383     * Format of a memberlist within a PDS:
    384     *
    385     * <pre>
    386     *    0         1        2          3        4     5     6      7    8
    387     *   Name      VV.MM   Created       Changed      Size  Init   Mod   Id
    388     *   TBSHELF   01.03 2002/09/12 2002/10/11 09:37    11    11     0 KIL001
    389     *   TBTOOL    01.12 2002/09/12 2004/11/26 19:54    51    28     0 KIL001
    390     *
    391     * -------------------------------------------
    392     * [1] Name
    393     * [2] VV.MM: Version . modification
    394     * [3] Created: yyyy / MM / dd
    395     * [4,5] Changed: yyyy / MM / dd HH:mm
    396     * [6] Size: number of lines
    397     * [7] Init: number of lines when first created
    398     * [8] Mod: number of modified lines a last save
    399     * [9] Id: User id for last update
    400     * </pre>
    401     *
    402     * @param entry zosDirectoryEntry
    403     * @return null: entry was not parsed.
    404     */
    405    private FTPFile parseMemberList(final String entry) {
    406        final FTPFile file = new FTPFile();
    407        if (matches(entry)) {
    408            file.setRawListing(entry);
    409            final String name = group(1);
    410            final String datestr = group(2) + " " + group(3);
    411            file.setName(name);
    412            file.setType(FTPFile.FILE_TYPE);
    413            try {
    414                file.setTimestamp(super.parseTimestamp(datestr));
    415            } catch (final ParseException e) {
    416                // just ignore parsing errors.
    417                // TODO check this is ok
    418                // Drop thru to try simple parser
    419            }
    420            return file;
    421        }
    422
    423        /*
    424         * Assigns the name to the first word of the entry. Only to be used from a safe context, for example from a memberlist, where the regex for some reason
    425         * fails. Then just assign the name field of FTPFile.
    426         */
    427        if (entry != null && !entry.trim().isEmpty()) {
    428            file.setRawListing(entry);
    429            final String name = entry.split(" ")[0];
    430            file.setName(name);
    431            file.setType(FTPFile.FILE_TYPE);
    432            return file;
    433        }
    434        return null;
    435    }
    436
    437    /**
    438     * Pre-parses is called as part of the interface. Per definition, it is called before the parsing takes place. Three kinds of lists are recognized:
    439     * <ul>
    440     *     <li>z/OS-MVS File lists,</li>
    441     *     <li>z/OS-MVS Member lists,</li>
    442     *     <li>unix file lists.</li>
    443     * </ul>
    444     * @since 2.0
    445     */
    446    @Override
    447    public List<String> preParse(final List<String> orig) {
    448        // simply remove the header line. Composite logic will take care of the
    449        // two different types of
    450        // list in short order.
    451        if (orig != null && !orig.isEmpty()) {
    452            final String header = orig.get(0);
    453            if (header.contains("Volume") && header.contains("Dsname")) {
    454                setType(FILE_LIST_TYPE);
    455                super.setRegex(FILE_LIST_REGEX);
    456            } else if (header.contains("Name") && header.contains("Id")) {
    457                setType(MEMBER_LIST_TYPE);
    458                super.setRegex(MEMBER_LIST_REGEX);
    459            } else if (header.indexOf("total") == 0) {
    460                setType(UNIX_LIST_TYPE);
    461                unixFTPEntryParser = new UnixFTPEntryParser();
    462            } else if (header.indexOf("Spool Files") >= 30) {
    463                setType(JES_LEVEL_1_LIST_TYPE);
    464                super.setRegex(JES_LEVEL_1_LIST_REGEX);
    465            } else if (header.indexOf("JOBNAME") == 0 && header.indexOf("JOBID") > 8) {// header contains JOBNAME JOBID OWNER // STATUS CLASS
    466                setType(JES_LEVEL_2_LIST_TYPE);
    467                super.setRegex(JES_LEVEL_2_LIST_REGEX);
    468            } else {
    469                setType(UNKNOWN_LIST_TYPE);
    470            }
    471
    472            if (isType != JES_LEVEL_1_LIST_TYPE) { // remove header is necessary
    473                orig.remove(0);
    474            }
    475        }
    476
    477        return orig;
    478    }
    479
    480    /**
    481     * Sets the type of listing being processed.
    482     *
    483     * @param type The listing type.
    484     */
    485    void setType(final int type) {
    486        isType = type;
    487    }
    488
    489}