org.apache.nutch.indexer

Class IndexerMapReduce

java.lang.Object

org.apache.hadoop.conf.Configured

org.apache.nutch.indexer.IndexerMapReduce

All Implemented Interfaces:

java.io.Closeable, java.lang.AutoCloseable, Configurable, JobConfigurable, Mapper<Text,Writable,Text,NutchWritable>, Reducer<Text,NutchWritable,Text,NutchIndexAction>

public class IndexerMapReduce
extends Configured
implements Mapper<Text,Writable,Text,NutchWritable>, Reducer<Text,NutchWritable,Text,NutchIndexAction>

Field Summary

Fields

Modifier and Type	Field and Description
static java.lang.String	INDEXER_BINARY_AS_BASE64
static java.lang.String	INDEXER_DELETE
static java.lang.String	INDEXER_DELETE_ROBOTS_NOINDEX
static java.lang.String	INDEXER_DELETE_SKIPPED
static java.lang.String	INDEXER_PARAMS
static java.lang.String	INDEXER_SKIP_NOTMODIFIED
static java.lang.String	URL_FILTERING
static java.lang.String	URL_NORMALIZING

Constructor Summary

Constructors

Constructor and Description
IndexerMapReduce()

Method Summary

Modifier and Type	Method and Description
void	close()
void	configure(JobConf job)
static void	initMRJob(Path crawlDb, Path linkDb, java.util.Collection<Path> segments, JobConf job, boolean addBinaryContent)
void	map(Text key, Writable value, OutputCollector<Text,NutchWritable> output, Reporter reporter)
void	reduce(Text key, java.util.Iterator<NutchWritable> values, OutputCollector<Text,NutchIndexAction> output, Reporter reporter)

Methods inherited from class org.apache.hadoop.conf.Configured

getConf, setConf

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Field Detail

INDEXER_PARAMS

public static final java.lang.String INDEXER_PARAMS

See Also:

Constant Field Values

INDEXER_DELETE

public static final java.lang.String INDEXER_DELETE

See Also:

Constant Field Values

INDEXER_DELETE_ROBOTS_NOINDEX

public static final java.lang.String INDEXER_DELETE_ROBOTS_NOINDEX

See Also:

Constant Field Values

INDEXER_DELETE_SKIPPED

public static final java.lang.String INDEXER_DELETE_SKIPPED

See Also:

Constant Field Values

INDEXER_SKIP_NOTMODIFIED

public static final java.lang.String INDEXER_SKIP_NOTMODIFIED

See Also:

Constant Field Values

URL_FILTERING

public static final java.lang.String URL_FILTERING

See Also:

Constant Field Values

URL_NORMALIZING

public static final java.lang.String URL_NORMALIZING

See Also:

Constant Field Values

INDEXER_BINARY_AS_BASE64

public static final java.lang.String INDEXER_BINARY_AS_BASE64

See Also:

Constant Field Values

Constructor Detail

IndexerMapReduce

public IndexerMapReduce()

Method Detail

configure

public void configure(JobConf job)

Specified by:

configure in interface JobConfigurable

map

public void map(Text key,
                Writable value,
                OutputCollector<Text,NutchWritable> output,
                Reporter reporter)
         throws java.io.IOException

Specified by:

map in interface Mapper<Text,Writable,Text,NutchWritable>

Throws:

java.io.IOException

reduce

public void reduce(Text key,
                   java.util.Iterator<NutchWritable> values,
                   OutputCollector<Text,NutchIndexAction> output,
                   Reporter reporter)
            throws java.io.IOException

Specified by:

reduce in interface Reducer<Text,NutchWritable,Text,NutchIndexAction>

Throws:

java.io.IOException

close

public void close()
           throws java.io.IOException

Specified by:

close in interface java.io.Closeable

Specified by:

close in interface java.lang.AutoCloseable

Throws:

java.io.IOException

initMRJob

public static void initMRJob(Path crawlDb,
                             Path linkDb,
                             java.util.Collection<Path> segments,
                             JobConf job,
                             boolean addBinaryContent)