View Javadoc

    1   /*
    2    * Licensed to the Apache Software Foundation (ASF) under one or more
    3    * contributor license agreements.  See the NOTICE file distributed with
    4    * this work for additional information regarding copyright ownership.
    5    * The ASF licenses this file to You under the Apache License, Version 2.0
    6    * (the "License"); you may not use this file except in compliance with
    7    * the License.  You may obtain a copy of the License at
    8    *
    9    *  http://www.apache.org/licenses/LICENSE-2.0
    10   *
    11   * Unless required by applicable law or agreed to in writing, software
    12   * distributed under the License is distributed on an "AS IS" BASIS,
    13   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    14   * See the License for the specific language governing permissions and
    15   * limitations under the License.
    16   */
    17
    18  package org.apache.any23.plugin.crawler;
    19
    20  import edu.uci.ics.crawler4j.crawler.Page;
    21  import org.apache.any23.Any23OnlineTestBase;
    22  import org.junit.Assert;
    23  import org.junit.Test;
    24  import org.slf4j.Logger;
    25  import org.slf4j.LoggerFactory;
    26
    27  import java.io.File;
    28  import java.net.URL;
    29  import java.util.HashSet;
    30  import java.util.Set;
    31  import java.util.Iterator;
    32
    33  /**
    34   * Test case for {@link SiteCrawler}.
    35   *
    36   * @author Michele Mostarda (mostarda@fbk.eu)
    37   */
    38  public class SiteCrawlerTest extends Any23OnlineTestBase {
    39
    40      public static final Logger logger = LoggerFactory.getLogger(SiteCrawlerTest.class);
    41
    42      /**
    43       * Tests the main crawler use case.
    44       *
    45       * @throws Exception if there is an error asserting test data
    46       */
    47      @Test
    48      public void testSiteCrawling() throws Exception {
    49          assumeOnlineAllowed();
    50
    51          File tmpFile = File.createTempFile("site-crawler-test", ".storage");
    52          tmpFile.delete();
    53
    54          final SiteCrawler controller = new SiteCrawler(tmpFile);
    55          controller.setMaxPages(100);
    56          logger.info("Crawler4j: Setting max num of pages to: " + controller.getMaxPages());
    57          controller.setPolitenessDelay(500);
    58          logger.info("Crawler4j: Setting Politeness delay to: " + controller.getPolitenessDelay() + "ms");
    59
    60          final Set<String> distinctPages = new HashSet<String>();
    61          controller.addListener(new CrawlerListener() {
    62              @Override
    63              public void visitedPage(Page page) {
    64                  distinctPages.add( page.getWebURL().getURL() );
    65                  Iterator<String> it = distinctPages.iterator();
    66                  while (it.hasNext()) {
    67                      logger.info("Crawler4j: Fetching page - " + it.next());
    68                  }
    69              }
    70          });
    71
    72          controller.start( new URL("http://any23.apache.org/"), false);
    73
    74          synchronized (this) {
    75              this.wait(15 * 1000);
    76          }
    77          controller.stop();
    78
    79          logger.info("Distinct pages: " + distinctPages.size());
    80          Assert.assertTrue("Expected some page crawled.", distinctPages.size() > 0);
    81      }
    82
    83  }