org.apache.nutch.crawl

Class CrawlDbReader

java.lang.Object

org.apache.hadoop.conf.Configured

org.apache.nutch.util.AbstractChecker

org.apache.nutch.crawl.CrawlDbReader

All Implemented Interfaces:

java.io.Closeable, java.lang.AutoCloseable, Configurable, Tool

public class CrawlDbReader
extends AbstractChecker
implements java.io.Closeable

Read utility for the CrawlDB.

Author:

Andrzej Bialecki

Nested Class Summary

Nested Classes

Modifier and Type	Class and Description
static class	CrawlDbReader.CrawlDatumCsvOutputFormat
static class	CrawlDbReader.CrawlDbDumpMapper
static class	CrawlDbReader.CrawlDbStatMapper
static class	CrawlDbReader.CrawlDbStatReducer
static class	CrawlDbReader.CrawlDbTopNMapper
static class	CrawlDbReader.CrawlDbTopNReducer

Field Summary

Fields

Modifier and Type	Field and Description
protected java.lang.String	crawlDb

Fields inherited from class org.apache.nutch.util.AbstractChecker

keepClientCnxOpen, stdin, tcpPort, usage

Constructor Summary

Constructors

Constructor and Description
CrawlDbReader()

Method Summary

Modifier and Type	Method and Description
void	close()
CrawlDatum	get(java.lang.String crawlDb, java.lang.String url, Configuration config)
static void	main(java.lang.String[] args)
protected int	process(java.lang.String line, java.lang.StringBuilder output)
void	processDumpJob(java.lang.String crawlDb, java.lang.String output, Configuration config, java.lang.String format, java.lang.String regex, java.lang.String status, java.lang.Integer retry, java.lang.String expr, java.lang.Float sample)
void	processStatJob(java.lang.String crawlDb, Configuration config, boolean sort)
void	processTopNJob(java.lang.String crawlDb, long topN, float min, java.lang.String output, Configuration config)
java.lang.Object	query(java.util.Map<java.lang.String,java.lang.String> args, Configuration conf, java.lang.String type, java.lang.String crawlId)
void	readUrl(java.lang.String crawlDb, java.lang.String url, Configuration config, java.lang.StringBuilder output)
int	run(java.lang.String[] args)

Methods inherited from class org.apache.nutch.util.AbstractChecker

getProtocolOutput, parseArgs, processSingle, processStdin, processTCP, run

Methods inherited from class org.apache.hadoop.conf.Configured

getConf, setConf

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Methods inherited from interface org.apache.hadoop.conf.Configurable

getConf, setConf

Field Detail

crawlDb

protected java.lang.String crawlDb

Constructor Detail

CrawlDbReader

public CrawlDbReader()

Method Detail

close

public void close()

Specified by:

close in interface java.io.Closeable

Specified by:

close in interface java.lang.AutoCloseable

processStatJob

public void processStatJob(java.lang.String crawlDb,
                           Configuration config,
                           boolean sort)
                    throws java.io.IOException,
                           java.lang.InterruptedException,
                           java.lang.ClassNotFoundException

Throws:

java.io.IOException

java.lang.InterruptedException

java.lang.ClassNotFoundException

get

public CrawlDatum get(java.lang.String crawlDb,
                      java.lang.String url,
                      Configuration config)
               throws java.io.IOException

Throws:

java.io.IOException

process

protected int process(java.lang.String line,
                      java.lang.StringBuilder output)
               throws java.lang.Exception

Specified by:

process in class AbstractChecker

Throws:

java.lang.Exception

readUrl

public void readUrl(java.lang.String crawlDb,
                    java.lang.String url,
                    Configuration config,
                    java.lang.StringBuilder output)
             throws java.io.IOException

Throws:

java.io.IOException

processDumpJob

public void processDumpJob(java.lang.String crawlDb,
                           java.lang.String output,
                           Configuration config,
                           java.lang.String format,
                           java.lang.String regex,
                           java.lang.String status,
                           java.lang.Integer retry,
                           java.lang.String expr,
                           java.lang.Float sample)
                    throws java.io.IOException,
                           java.lang.ClassNotFoundException,
                           java.lang.InterruptedException

Throws:

java.io.IOException

java.lang.ClassNotFoundException

java.lang.InterruptedException

processTopNJob

public void processTopNJob(java.lang.String crawlDb,
                           long topN,
                           float min,
                           java.lang.String output,
                           Configuration config)
                    throws java.io.IOException,
                           java.lang.ClassNotFoundException,
                           java.lang.InterruptedException

Throws:

java.io.IOException

java.lang.ClassNotFoundException

java.lang.InterruptedException

run

public int run(java.lang.String[] args)
        throws java.io.IOException,
               java.lang.InterruptedException,
               java.lang.ClassNotFoundException,
               java.lang.Exception

Specified by:

run in interface Tool

Throws:

java.io.IOException

java.lang.InterruptedException

java.lang.ClassNotFoundException

java.lang.Exception

main

public static void main(java.lang.String[] args)
                 throws java.lang.Exception

Throws:

java.lang.Exception

query

public java.lang.Object query(java.util.Map<java.lang.String,java.lang.String> args,
                              Configuration conf,
                              java.lang.String type,
                              java.lang.String crawlId)
                       throws java.lang.Exception

Throws:

java.lang.Exception