java超快速文本去重复代码，java文本重复代码,import java.

java超快速文本去重复代码，java文本重复代码,import java.

import java.io.*;import java.util.HashSet;import java.util.Iterator;import java.util.Set;public class SpeedClear {    public static void main(String[] args) {        if(args.length==0){            print();            System.exit(1);        }        if(args.length!=2){            System.out.println("Format error...");            System.exit(1);        }        String pathname = args[0];        String newPath = args[1];        clear(pathname,newPath);     //调用去重复的方法...    }    /**     *      * @param pathname     *            源文件路径     * @param newPath     *            新的文件路径     * @throws Exception     */    public static void clear(String pathname, String newPath) {        System.out.println("Start... ");        try{    //懒的写Try..直接都包围起来吧....            File file = new File(pathname);        BufferedInputStream fis = new BufferedInputStream(new FileInputStream(file));BufferedReader buffer = new BufferedReader(new InputStreamReader(fis,"utf-8"),20*1024*1024);// 用5M的缓冲读取文本文件             //FileWriter fw  = new FileWriter(new File(newPath),true);  //去除后的文本            OutputStreamWriter out = new OutputStreamWriter(new FileOutputStream(new File(newPath)),"utf-8") ;            Set<String> set = new HashSet<String>();            String temp = ""; // 临时字符串            int x = 0;            while ((temp = buffer.readLine()) != null) { // 读文件,一行读一个                set.add(temp); // 存储到Set集合里面                if(x%30000==0){                    System.out.print("..") ;                }                x++;            }            fis.close();            buffer.close();   //关闭读取操作            //下面开始写文件            for (String xxser : set) {                out.write(xxser+"\r\n");            }            System.out.println("") ;            out.close();   //关闭写操作            System.out.println("size = " + set.size());            System.out.println("End...");        }catch(Exception e){            System.out.println("文件太大了,建议先100MB大小..") ;        }    }        public static void  print(){        System.out.println("*************************************************");        System.out.println("\t\tTo repeat \t\t");        System.out.println();        System.out.println("  format: java -Xmx1000m SpeedClear c:\\old.txt c:\\new.txt\t\t");        System.out.println();        System.out.println("\t\tAuthor:xxser    QQ:616100108");        System.out.println("*************************************************");    }}