HDFS API的简单使用(1)

package cn.framelife.hadoop;

import java.io.IOException;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;

/**
 * 上传本地文件到HDFS
 * @author 潘广伟
 *
 */
public class CopyFile {
	public static void main(String[] args) {

		Configuration configuration = new Configuration();

		//如果是远程集群中的hdfs，我们需要有以下两个配置文件
		configuration.addResource(new Path("core-site.xml"));
		configuration.addResource(new Path("hdfs-site.xml"));

		try {
			//根据配置获取到hdfs系统对象
			FileSystem hdfs = FileSystem.get(configuration);

			//源文件
			Path src = new Path("/home/benben/abc");

			//hdfs服务器中的路径
			Path dst = new Path("/user/");

			hdfs.copyFromLocalFile(src, dst);
			System.out.println("upload to"+configuration.get("fs.default.name"));

			//遍历hdfs下所有的文件
			FileStatus[] files = hdfs.listStatus(dst);
			for (FileStatus file : files) {
				System.out.println(file.getPath());
			}
		} catch (IOException e) {
			e.printStackTrace();
		}

	}
}

package cn.framelife.hadoop;

import java.io.IOException;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;

/**
 * 在HDFS中创建一个文件
 *
 * @author 潘广伟
 *
 */
public class CreateFile {

	public static void main(String[] args) {
		Configuration configuration = new Configuration();
		configuration.addResource(new Path("core-site.xml"));
		configuration.addResource(new Path("hdfs-site.xml"));

		try {
			FileSystem hdfs = FileSystem.get(configuration);

			byte[] buffer = "Hello Hadoop".getBytes();

			Path newPath = new Path("/user/hello1.txt");

			FSDataOutputStream outputStream = hdfs.create(newPath);

			outputStream.write(buffer, 0, buffer.length);

		} catch (IOException e) {
			e.printStackTrace();
		}
	}
}

package cn.framelife.hadoop;

import java.io.IOException;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;

/**
 * 删除HDFS文件
 * @author 潘广伟
 *
 */
public class DeleteFile {

	public static void main(String[] args) {
		Configuration configuration = new Configuration();
		configuration.addResource(new Path("core-site.xml"));
		configuration.addResource(new Path("hdfs-site.xml"));

		try {
			FileSystem hdfs = FileSystem.get(configuration);

			Path pathOfHDFS = new Path("/user/hello1.txt");

			//文件是否存在
			if(hdfs.exists(pathOfHDFS)){
				/*
				 * if path is a directory and set to true, the directory is deleted else throws an exception.
				 * In case of a file the recursive can be set to either true or false.
				*/
				hdfs.delete(pathOfHDFS, false);
			}

		} catch (IOException e) {
			e.printStackTrace();
		}
	}

}

package cn.framelife.hadoop;

import java.io.IOException;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;

public class Rename {

	/**
	 * 给HDFS中的文件重新命名
	 * @param 潘广伟
	 */
	public static void main(String[] args) {
		Configuration configuration = new Configuration();
		configuration.addResource(new Path("core-site.xml"));
		configuration.addResource(new Path("hdfs-site.xml"));

		try {
			FileSystem hdfs = FileSystem.get(configuration);

			Path pathOfHDFS = new Path("/user/abc.txt");
			Path newPathName = new Path("/user/abcd.txt");

			//成功就返回true
			boolean isRename = hdfs.rename(pathOfHDFS, newPathName);

			System.out.println(isRename);

		} catch (IOException e) {
			e.printStackTrace();
		}
	}

}

package cn.framelife.hadoop;

import java.io.IOException;
import java.util.Arrays;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.BlockLocation;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;

/**
 * 查看某个文件在HDFS集群的位置
 * @author 潘广伟
 *
 */
public class WhereIsFile {

	public static void main(String[] args) {
		Configuration configuration = new Configuration();
		configuration.addResource(new Path("core-site.xml"));
		configuration.addResource(new Path("hdfs-site.xml"));

		try {
			FileSystem hdfs = FileSystem.get(configuration);

			Path pathOfHDFS = new Path("/user/hello.txt");

			FileStatus fileStatus =  hdfs.getFileStatus(pathOfHDFS);
	        BlockLocation[] blockLocations = hdfs.getFileBlockLocations(fileStatus, 0, fileStatus.getLen());
	        for(BlockLocation block : blockLocations){
	        	System.out.println(Arrays.toString(block.getHosts())+ "\t" + Arrays.toString(block.getNames()));
	        }
		} catch (IOException e) {
			e.printStackTrace();
		}

	}

}

package cn.framelife.hadoop;

import java.io.IOException;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;

public class GetLastTime {

	/**
	 * 获取HDFS文件的最后修改时间
	 * @param 潘广伟
	 */
	public static void main(String[] args) {
		Configuration configuration = new Configuration();
		configuration.addResource(new Path("core-site.xml"));
		configuration.addResource(new Path("hdfs-site.xml"));

		try {
			FileSystem hdfs = FileSystem.get(configuration);

			Path pathOfHDFS = new Path("/user/abcd.txt");

			FileStatus file = hdfs.getFileStatus(pathOfHDFS);
			long time = file.getModificationTime();

			System.out.println(time);
		} catch (IOException e) {
			e.printStackTrace();
		}
	}

}

package cn.framelife.hadoop;

import java.io.IOException;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;

/**
 * 获取HDFS集群上所有节点名称
 *
 * @author 潘广伟
 *
 */
public class GetDataNodeName {
	public static void main(String[] args) {

		Configuration configuration = new Configuration();
		configuration.addResource(new Path("core-site.xml"));
		configuration.addResource(new Path("hdfs-site.xml"));

		DistributedFileSystem hdfs;
		try {
			hdfs = (DistributedFileSystem) FileSystem.get(configuration);
			DatanodeInfo[] dataNodeStats = hdfs.getDataNodeStats();

			for (DatanodeInfo dataNode : dataNodeStats) {
				System.out.println(dataNode.getHostName() + "\t" + dataNode.getName());
			}

		} catch (IOException e) {
			e.printStackTrace();
		}

	}
}