001 /**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements. See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership. The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License. You may obtain a copy of the License at
009 *
010 * http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018
019 package org.apache.hadoop.filecache;
020
021 import org.apache.hadoop.classification.InterfaceAudience;
022 import org.apache.hadoop.classification.InterfaceStability;
023 import org.apache.hadoop.conf.Configuration;
024 import org.apache.hadoop.fs.FileSystem;
025 import org.apache.hadoop.fs.Path;
026 import org.apache.hadoop.mapreduce.Job;
027
028 /**
029 * Distribute application-specific large, read-only files efficiently.
030 *
031 * <p><code>DistributedCache</code> is a facility provided by the Map-Reduce
032 * framework to cache files (text, archives, jars etc.) needed by applications.
033 * </p>
034 *
035 * <p>Applications specify the files, via urls (hdfs:// or http://) to be cached
036 * via the {@link org.apache.hadoop.mapred.JobConf}. The
037 * <code>DistributedCache</code> assumes that the files specified via urls are
038 * already present on the {@link FileSystem} at the path specified by the url
039 * and are accessible by every machine in the cluster.</p>
040 *
041 * <p>The framework will copy the necessary files on to the slave node before
042 * any tasks for the job are executed on that node. Its efficiency stems from
043 * the fact that the files are only copied once per job and the ability to
044 * cache archives which are un-archived on the slaves.</p>
045 *
046 * <p><code>DistributedCache</code> can be used to distribute simple, read-only
047 * data/text files and/or more complex types such as archives, jars etc.
048 * Archives (zip, tar and tgz/tar.gz files) are un-archived at the slave nodes.
049 * Jars may be optionally added to the classpath of the tasks, a rudimentary
050 * software distribution mechanism. Files have execution permissions.
051 * In older version of Hadoop Map/Reduce users could optionally ask for symlinks
052 * to be created in the working directory of the child task. In the current
053 * version symlinks are always created. If the URL does not have a fragment
054 * the name of the file or directory will be used. If multiple files or
055 * directories map to the same link name, the last one added, will be used. All
056 * others will not even be downloaded.</p>
057 *
058 * <p><code>DistributedCache</code> tracks modification timestamps of the cache
059 * files. Clearly the cache files should not be modified by the application
060 * or externally while the job is executing.</p>
061 *
062 * <p>Here is an illustrative example on how to use the
063 * <code>DistributedCache</code>:</p>
064 * <p><blockquote><pre>
065 * // Setting up the cache for the application
066 *
067 * 1. Copy the requisite files to the <code>FileSystem</code>:
068 *
069 * $ bin/hadoop fs -copyFromLocal lookup.dat /myapp/lookup.dat
070 * $ bin/hadoop fs -copyFromLocal map.zip /myapp/map.zip
071 * $ bin/hadoop fs -copyFromLocal mylib.jar /myapp/mylib.jar
072 * $ bin/hadoop fs -copyFromLocal mytar.tar /myapp/mytar.tar
073 * $ bin/hadoop fs -copyFromLocal mytgz.tgz /myapp/mytgz.tgz
074 * $ bin/hadoop fs -copyFromLocal mytargz.tar.gz /myapp/mytargz.tar.gz
075 *
076 * 2. Setup the application's <code>JobConf</code>:
077 *
078 * JobConf job = new JobConf();
079 * DistributedCache.addCacheFile(new URI("/myapp/lookup.dat#lookup.dat"),
080 * job);
081 * DistributedCache.addCacheArchive(new URI("/myapp/map.zip", job);
082 * DistributedCache.addFileToClassPath(new Path("/myapp/mylib.jar"), job);
083 * DistributedCache.addCacheArchive(new URI("/myapp/mytar.tar", job);
084 * DistributedCache.addCacheArchive(new URI("/myapp/mytgz.tgz", job);
085 * DistributedCache.addCacheArchive(new URI("/myapp/mytargz.tar.gz", job);
086 *
087 * 3. Use the cached files in the {@link org.apache.hadoop.mapred.Mapper}
088 * or {@link org.apache.hadoop.mapred.Reducer}:
089 *
090 * public static class MapClass extends MapReduceBase
091 * implements Mapper<K, V, K, V> {
092 *
093 * private Path[] localArchives;
094 * private Path[] localFiles;
095 *
096 * public void configure(JobConf job) {
097 * // Get the cached archives/files
098 * File f = new File("./map.zip/some/file/in/zip.txt");
099 * }
100 *
101 * public void map(K key, V value,
102 * OutputCollector<K, V> output, Reporter reporter)
103 * throws IOException {
104 * // Use data from the cached archives/files here
105 * // ...
106 * // ...
107 * output.collect(k, v);
108 * }
109 * }
110 *
111 * </pre></blockquote></p>
112 *
113 * It is also very common to use the DistributedCache by using
114 * {@link org.apache.hadoop.util.GenericOptionsParser}.
115 *
116 * This class includes methods that should be used by users
117 * (specifically those mentioned in the example above, as well
118 * as {@link DistributedCache#addArchiveToClassPath(Path, Configuration)}),
119 * as well as methods intended for use by the MapReduce framework
120 * (e.g., {@link org.apache.hadoop.mapred.JobClient}).
121 *
122 * @see org.apache.hadoop.mapred.JobConf
123 * @see org.apache.hadoop.mapred.JobClient
124 * @see org.apache.hadoop.mapreduce.Job
125 */
126 @InterfaceAudience.Public
127 @InterfaceStability.Stable
128 public class DistributedCache extends
129 org.apache.hadoop.mapreduce.filecache.DistributedCache {
130 //
131 }