View Javadoc

    1   /*
    2    * Licensed to the Apache Software Foundation (ASF) under one or more
    3    * contributor license agreements.  See the NOTICE file distributed with
    4    * this work for additional information regarding copyright ownership.
    5    * The ASF licenses this file to You under the Apache License, Version 2.0
    6    * (the "License"); you may not use this file except in compliance with
    7    * the License.  You may obtain a copy of the License at
    8    *
    9    *  http://www.apache.org/licenses/LICENSE-2.0
    10   *
    11   * Unless required by applicable law or agreed to in writing, software
    12   * distributed under the License is distributed on an "AS IS" BASIS,
    13   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    14   * See the License for the specific language governing permissions and
    15   * limitations under the License.
    16   */
    17
    18  package org.apache.any23.extractor.rdf;
    19
    20  import java.io.IOException;
    21  import java.io.InputStream;
    22  import java.io.PushbackInputStream;
    23  import java.util.Arrays;
    24
    25  /**
    26   * This class uses several strategies to fix common JSON syntax errors, including:
    27   * <ol>
    28   * <li>Remove CDATA markers</li>
    29   * <li>Remove YAML and C-style comments</li>
    30   * <li>Allow single-quoted strings</li>
    31   * <li>Ignore duplicated commas between elements of objects and arrays</li>
    32   * <li>Remove trailing commas from objects and arrays</li>
    33   * <li>Insert omitted commas after objects and arrays</li>
    34   * <li>Ignore all unicode whitespace characters (assumes UTF-8 encoding)</li>
    35   * <li>Treat semi-colons as commas</li>
    36   * </ol>
    37   *
    38   * @author Hans Brende (hansbrende@apache.org)
    39   */
    40  class JsonCleaningInputStream extends InputStream {
    41
    42      private static final int EOL_COMMENT = 1;
    43      private static final int MULTILINE_COMMENT = 2;
    44
    45      private static final int NEEDS_COMMA = -1;
    46      private static final int NEEDS_COMMA_AND_NEWLINE = 1;
    47
    48      private boolean inEscape;
    49      private boolean inCDATA;
    50      private int needsComma;
    51      private int currentState;
    52
    53      private static final int MAX_BLANK_PUSHBACK = 128;
    54      private static final byte[] BLANK_PUSHBACK = new byte[MAX_BLANK_PUSHBACK];
    55
    56      static {
    57          Arrays.fill(BLANK_PUSHBACK, (byte) ' ');
    58          BLANK_PUSHBACK[0] = '\n';
    59      }
    60
    61      private final PushbackInputStream in;
    62
    63      JsonCleaningInputStream(InputStream in) {
    64          this.in = new PushbackInputStream(in, 256);
    65      }
    66
    67      private static void unread(PushbackInputStream in, int c) throws IOException {
    68          if (c != -1) {
    69              in.unread(c);
    70          }
    71      }
    72
    73      private static boolean isNextOrUnread(PushbackInputStream in, int... next) throws IOException {
    74          int i = -1;
    75          for (int test : next) {
    76              int c = in.read();
    77              if (c != test) {
    78                  unread(in, c);
    79                  while (i >= 0) {
    80                      in.unread(next[i--]);
    81                  }
    82                  return false;
    83              }
    84              i++;
    85          }
    86          return true;
    87      }
    88
    89      @Override
    90      public int read() throws IOException {
    91          PushbackInputStream in = this.in;
    92
    93          for (;;) {
    94              int c = in.read();
    95
    96              if (c == -1) {
    97                  return c;
    98              }
    99
    100             if (inCDATA) {
    101                 if (c == ']' && isNextOrUnread(in, ']', '>')) {
    102                     inCDATA = false;
    103                     continue;
    104                 }
    105             } else {
    106                 if (c == '<' && isNextOrUnread(in, '!', '[', 'C', 'D', 'A', 'T', 'A', '[')) {
    107                     inCDATA = true;
    108                     continue;
    109                 }
    110             }
    111
    112             int ctx = currentState;
    113             switch (ctx) {
    114             case 0:
    115                 break;
    116             case EOL_COMMENT:
    117                 if (c == '\r' || c == '\n') {
    118                     // end single-line comment
    119                     currentState = 0;
    120                     if (needsComma != 0) {
    121                         needsComma = NEEDS_COMMA_AND_NEWLINE;
    122                         continue;
    123                     }
    124                     return c;
    125                 }
    126                 continue;
    127             case MULTILINE_COMMENT:
    128                 if (c == '\r' || c == '\n') {
    129                     if (needsComma != 0) {
    130                         needsComma = NEEDS_COMMA_AND_NEWLINE;
    131                         continue;
    132                     }
    133                     return c;
    134                 } else if (c == '*' && isNextOrUnread(in, '/')) {
    135                     // end multiline comment
    136                     currentState = 0;
    137                 }
    138                 continue;
    139             default:
    140                 // we're in a quote
    141                 if (inEscape) {
    142                     // end escape
    143                     inEscape = false;
    144                 } else if (c == '\\') {
    145                     // begin escape
    146                     inEscape = true;
    147                 } else if (c == ctx) {
    148                     // end quote
    149                     currentState = 0;
    150                     return '"';
    151                 }
    152                 return c;
    153             }
    154
    155             // we're not in a quote or comment
    156
    157             $whitespace: {
    158                 switch (c) {
    159                 case '#':
    160                     currentState = EOL_COMMENT;
    161                     continue;
    162                 case '/':
    163                     int next = in.read();
    164                     if (next == '/') {
    165                         currentState = EOL_COMMENT;
    166                         continue;
    167                     } else if (next == '*') {
    168                         currentState = MULTILINE_COMMENT;
    169                         continue;
    170                     }
    171                     unread(in, next);
    172                     break;
    173                 case ',':
    174                 case ';':
    175                     // don't write out comma yet!
    176                     needsComma = NEEDS_COMMA;
    177                     continue;
    178                 case '}':
    179                 case ']':
    180                     // Only thing that can follow '}' or ']' is:
    181                     // '}' or ']' or ',' or EOF
    182                     needsComma = NEEDS_COMMA;
    183                     return c;
    184                 case '\r':
    185                 case '\n':
    186                     if (needsComma != 0) {
    187                         needsComma = NEEDS_COMMA_AND_NEWLINE;
    188                         continue;
    189                     }
    190                     return c;
    191                 // UTF-8 whitespace detection
    192                 case 0x09:
    193                 case 0x0b:
    194                 case 0x0c:
    195                 case 0x1c:
    196                 case 0x1d:
    197                 case 0x1e:
    198                 case 0x1f:
    199                 case 0x20:
    200                     break $whitespace;
    201                 case 0xc2:
    202                     if (isNextOrUnread(in, 0xa0)) {
    203                         break $whitespace;
    204                     }
    205                     break;
    206                 case 0xe1:
    207                     if (isNextOrUnread(in, 0x9a, 0x80) || isNextOrUnread(in, 0xa0, 0x8e)) {
    208                         break $whitespace;
    209                     }
    210                     break;
    211                 case 0xe2:
    212                     int c1 = in.read();
    213                     if (c1 == 0x80) {
    214                         int c2 = in.read();
    215                         // space separators
    216                         if (c2 >= 0x80 && c2 <= 0x8a || c2 == 0xaf
    217                         // line and paragraph separators
    218                                 || c2 == 0xa8 || c2 == 0xa9) {
    219                             break $whitespace;
    220                         }
    221                         unread(in, c2);
    222                         in.unread(0x80);
    223                     } else if (c1 == 0x81) {
    224                         int c2 = in.read();
    225                         if (c2 == 0x9f) {
    226                             break $whitespace;
    227                         }
    228                         unread(in, c2);
    229                         in.unread(0x81);
    230                     } else {
    231                         unread(in, c1);
    232                     }
    233                     break;
    234                 case 0xe3:
    235                     if (isNextOrUnread(in, 0x80, 0x80)) {
    236                         break $whitespace;
    237                     }
    238                     break;
    239                 default:
    240                     break;
    241                 }
    242
    243                 // here: character is not whitespace
    244
    245                 int nc = needsComma;
    246                 if (nc != 0) {
    247                     in.unread(c);
    248                     if (nc == NEEDS_COMMA) {
    249                         in.unread(' ');
    250                     } else {
    251                         in.unread(BLANK_PUSHBACK, 0, nc);
    252                     }
    253                     needsComma = 0;
    254                     return ',';
    255                 } else if (c == '"' || c == '\'') {
    256                     currentState = c;
    257                     return '"';
    258                 }
    259                 return c;
    260             } // end $whitespace
    261
    262             // here: character is whitespace
    263
    264             int nc = needsComma;
    265             if (nc != 0) {
    266                 if (nc != NEEDS_COMMA && nc != MAX_BLANK_PUSHBACK) {
    267                     needsComma = nc + 1;
    268                 }
    269                 continue;
    270             }
    271
    272             return ' ';
    273
    274         }
    275
    276     }
    277 }