DBInputFormat从数据库读取数据到HDFS文件系统
发布日期：2021-11-15 02:11:39
浏览次数：2
分类：技术文章
本文共 10424 字，大约阅读时间需要 34 分钟。
public class MYSQL_TO_HDFS
{private final static Logger log = Logger.getLogger(KC86DATA_HDFS.class);/*private static final String out_path = "hdfs:// IP:9000/home/kc86_intput";private static final String driver = "oracle.jdbc.driver.OracleDriver";private static final String url = "jdbc:oracle:thin:@IP:1521:orcl";private static final String username = "CZ_ALL";private static final String password = "123456";*///连接数据库的参数private static final String out_path = "hdfs:// IP:9000/home/kc86_intput";private static final String driver = "com.mysql.jdbc.Driver";private static final String url = "jdbc:mysql://localhost:3306/zicontd";private static final String username = "root";private static final String password = "root"; /** * 读取数据库表中返回的实体类 * */ public static class KC86Entity implements Writable, DBWritable { private String AAZ217; private String AAC001; /** * 无参构造器 */ public KC86Entity() { super(); // TODO Auto-generated constructor stub } public KC86Entity(String aAZ217, String aAC001) { super(); AAZ217 = aAZ217; AAC001 = aAC001; } @Override public String toString() { return "KC86Entity [AAZ217=" + AAZ217 + ", AAC001=" + AAC001 + "]"; } @Override public void write(PreparedStatement statement) throws SQLException { // TODO Auto-generated method stub statement.setString(1, AAZ217); statement.setString(2, AAC001); } @Override public void readFields(ResultSet resultSet) throws SQLException { // TODO Auto-generated method stub AAZ217 = resultSet.getString(1); AAC001 = resultSet.getString(2); } @Override public void readFields(DataInput in) throws IOException { // TODO Auto-generated method stub AAZ217 =in.readUTF(); AAC001 =in.readUTF(); } } /** * Mapper函数,LongWritable, KC86Entity, NullWritable, KC86Entity为输入键、输入值;输出键，输出值 * */ static class MyDBInputFormatMRMapper extends Mapper { @Override protected void map(LongWritable key, KC86Entity value,Context context) throws IOException, InterruptedException { context.write(NullWritable.get(), value); } } /** * main的主函数 * @throws IOException * @throws InterruptedException * @throws ClassNotFoundException * */ public static void main(String[] args) throws IOException, ClassNotFoundException, InterruptedException { //当前运行环境中配置的HADOOP_USER_NAME属性值 String hadoop_user_name = System.getenv("HADOOP_USER_NAME"); System.setProperty("HADOOP_USER_NAME", hadoop_user_name); Configuration conf = new Configuration(); //在获取job对象之前，一定要先指定数据库的链接信息 DBConfiguration.configureDB(conf, driver, url, username, password); Job job = Job.getInstance(conf); String[] fieldNames = new String[] {"AAZ217","AAC001"}; DBInputFormat.setInput(job, KC86Entity.class, "KC86_test", null, null, fieldNames); job.setJarByClass(MYSQL_TO_HDFS .class); job.setMapperClass(MyDBInputFormatMRMapper.class); job.setOutputKeyClass(NullWritable.class); job.setOutputValueClass(KC86Entity.class); job.setNumReduceTasks(0); job.setInputFormatClass(DBInputFormat.class); //FileSystem fs = FileSystem.get(conf); FileSystem fs = FileSystem.get(URI.create(out_path), conf,"root"); Path p = new Path(out_path); if(fs.exists(p)){ fs.delete(p,true); //System.out.println("输出路径存在,已删除!"); log.info("输出路径存在,已删除!"); } FileOutputFormat.setOutputPath(job,p); System.exit(job.waitForCompletion(true) ? 0 : 1); } }
===============================================================
以下是maven的依赖pom
javax.activation activation 1.1 org.apache.directory.server apacheds-i18n 2.0.0-M15 org.apache.directory.server apacheds-kerberos-codec 2.0.0-M15 org.apache.directory.api api-asn1-api 1.0.0-M20 org.apache.directory.api api-util 1.0.0-M20 asm asm 3.2 org.apache.avro avro 1.7.4commons-beanutils commons-beanutils-core 1.8.0 commons-cli commons-cli 1.2 commons-codec commons-codec commons-collections commons-collections 3.2.2 org.apache.commons commons-compress 1.4.1 commons-configuration commons-configuration 1.6 commons-daemon commons-daemon 1.0.13 commons-digester commons-digester 1.8 commons-httpclient commons-httpclient 3.1 commons-io commons-io commons-lang commons-lang 2.6 commons-logging commons-logging 1.1.3 org.apache.commons commons-math3 3.1.1 commons-net commons-net 3.1 org.apache.curator curator-client 2.7.1 org.apache.curator curator-framework 2.7.1 org.apache.curator curator-recipes 2.7.1 com.google.code.gson gson 2.2.4 org.apache.hadoop hadoop-mapreduce-client-core 2.7.2 org.apache.hadoop hadoop-mapreduce-client-common 2.7.2 org.apache.hadoop hadoop-mapreduce-client-jobclient 2.7.2 com.google.guava guava 11.0.2 org.apache.hadoop hadoop-annotations 2.7.2 org.apache.hadoop hadoop-auth 2.7.2 org.apache.hadoop hadoop-common 2.7.2 org.apache.hadoop hadoop-hdfs 2.7.2 org.hamcrest hamcrest-core 1.3 org.apache.htrace htrace-core 3.1.0-incubating org.apache.httpcomponents httpclient 4.2.5 org.apache.httpcomponents httpcore 4.2.5 org.codehaus.jackson jackson-core-asl 1.9.13 org.codehaus.jackson jackson-jaxrs 1.9.13 org.codehaus.jackson jackson-mapper-asl 1.9.13 org.codehaus.jackson jackson-xc 1.9.13 com.jamesmurty.utils java-xmlbuilder 0.4 javax.xml.bind jaxb-api 2.2.2 com.sun.xml.bind jaxb-impl 2.2.3 com.sun.jersey jersey-core 1.9 com.sun.jersey jersey-json 1.9 com.sun.jersey jersey-server 1.9 net.java.dev.jets3t jets3t 0.9.0 org.codehaus.jettison jettison 1.1 org.mortbay.jetty jetty 6.1.26 org.mortbay.jetty jetty-util 6.1.26 com.jcraft jsch 0.1.42 javax.servlet.jsp jsp-api provided com.google.code.findbugs jsr305 3.0.0 junit junit test org.fusesource.leveldbjni leveldbjni-all 1.8 log4j log4j org.mockito mockito-all 1.8.5 test io.netty netty 3.6.2.Final io.netty netty-all 4.0.23.Final com.thoughtworks.paranamer paranamer 2.3 com.google.protobuf protobuf-java 2.5.0 javax.servlet servlet-api 2.5 provided org.slf4j slf4j-api org.slf4j slf4j-log4j12 test org.xerial.snappy snappy-java 1.0.4.1 javax.xml.stream stax-api 1.0-2 xerces xercesImpl 2.9.1 xml-apis xml-apis 1.3.04 xmlenc xmlenc 0.52 org.tukaani xz 1.0 org.apache.zookeeper zookeeper 3.4.6 pom com.oracle ojdbc6 mysql mysql-connector-java 5.1.44
转载地址：https://blog.csdn.net/qq_28358461/article/details/89443592 如侵犯您的版权，请留言回复原文章的地址，我们会给您删除此文章，给您带来不便请您谅解！
发表评论
最新留言
能坚持，总会有不一样的收获！
[***.219.124.196]2024年04月06日 22时02分40秒
关于作者
喝酒易醉，品茶养心，人生如梦，品茶悟道，何以解忧？唯有杜康！
-- 愿君每日到此一游！
推荐文章
Unity3D摄像机裁剪——NGUI篇
2019-04-27
lua深拷贝一个table
2019-04-27
app运行提示Unable to Initialize Unity Engine
2019-04-27
spring boot 与 Ant Design of Vue 实现修改按钮(十七)
2019-04-27
spring boot 与 Ant Design of Vue 实现删除按钮(十八)
2019-04-27
spring boot 与 Ant Design of Vue 实现新增角色(二十)
2019-04-27
spring boot 与 Ant Design of Vue 实现修改角色(二十一)
2019-04-27
spring boot 与 Ant Design of Vue 实现删除角色(补二十一)
2019-04-27
spring boot 与 Ant Design of Vue 实现左侧组织树(二十三)
2019-04-27
spring boot 与 Ant Design of Vue 实现新增组织(二十四)
2019-04-27
spring boot 与 Ant Design of Vue 实现修改组织(二十五)
2019-04-27
spring boot 与 Ant Design of Vue 实现删除组织(二十六)
2019-04-27
spring boot 与 Ant Design of Vue 实现新增用户(二十八)
2019-04-27
spring boot 与 Ant Design of Vue 实现修改用户(二十九)
2019-04-27
spring boot 与 Ant Design of Vue 实现删除用户(三十)
2019-04-27
白红宇的个人博客 - 记录点点滴滴的事 - 您是第 307988400 位访客
访问时间: 2024-04-26 06:26:41
访问IP: 3.145.8.42
Copyright © 2020 - 2023 blog.css8.cn 京ICP备2021015314号-1
手机版