/home/palakars/workspace-inprogress/code-in-progress/aitools3/aitools3-aq-invertedindex2-java/src/de/aitools/aq/invertedindex/usage/GoogleBooks.java

Go to the documentation of this file.

package de.aitools.aq.invertedindex.usage;

import java.io.BufferedReader;
import java.io.File;
import java.io.FilenameFilter;
import java.io.IOException;
import java.io.InputStreamReader;
import java.util.Calendar;
import java.util.zip.ZipEntry;
import java.util.zip.ZipException;
import java.util.zip.ZipFile;

import de.aitools.aq.check.A;
import de.aitools.aq.invertedindex.core.Configuration;
import de.aitools.aq.invertedindex.core.Configuration.KeySorting;
import de.aitools.aq.invertedindex.core.Configuration.ValueSorting;
import de.aitools.aq.invertedindex.core.Indexer;
import de.aitools.aq.value.pair.ShortInt;

public class GoogleBooks {
        
        public static final void index(File ngramDir, File indexDir)
        throws ZipException, IOException {
                A.CHECK(!ngramDir.equals(indexDir));
                
                Configuration config = new Configuration();
                config.setIndexDirectory(indexDir);
                config.setKeySorting(KeySorting.SORTED);
                config.setValueSorting(ValueSorting.DISABLED);
                Indexer<ShortInt> indexer = Indexer.open(ShortInt.class, config);
                
                final int minNgramLength = 1;
                final int maxNgramLength = 5;
                for (int i = minNgramLength; i <= maxNgramLength; ++i) {
                        final int curNgramLength = i; // I need that value as _final_
                        for (File file : ngramDir.listFiles(new FilenameFilter() {
                                @Override
                                public boolean accept(File dir, String name) {
                                        return name.contains(curNgramLength + "gram");
                                }})) {
                                String currentLine;
                                ShortInt curValue = new ShortInt();
                                ZipFile zf = new ZipFile(file);
                                ZipEntry ze = zf.getEntry(file.getName().replace(".zip", ""));
                                BufferedReader reader = new BufferedReader(
                                                new InputStreamReader(zf.getInputStream(ze)));
                                System.out.print(Calendar.getInstance().getTime());
                                System.out.println(" : Process " + ze.getName());
                                while ((currentLine = reader.readLine()) != null) {
                                        String[] curLineToken = currentLine.split("\t");
                                        if (curLineToken.length < 3 ||
                                                curLineToken[0].split(" ").length != curNgramLength)
                                                continue;
                                        curValue.set(
                                                        Short.parseShort(curLineToken[1]),
                                                        Integer.parseInt(curLineToken[2]));
                                        indexer.put(curLineToken[0], curValue);
                                }
                                zf.close();
                        }
                }
                indexer.index();
        }
        
        public static void main(String[] args) {
                // This is the directory of the original zipped n-gram files
                final File ngramDir = new File("/mnt/nfs/webis16/storage1/corpora/" +
                                "corpora-n-grams/corpus-google-books-ngram-viewer/" +
                                "googlebooks-eng-1M-20090715");
                // This is the directory of BigHashMap output
                final File indexDir = new File("/media/1TB/googlebooks-index");
                try {
                        GoogleBooks.index(ngramDir, indexDir);
                } catch (ZipException e) {
                        e.printStackTrace();
                } catch (IOException e) {
                        e.printStackTrace();
                }
        }

}

---