/home/palakars/workspace-inprogress/code-in-progress/aitools3/aitools3-aq-invertedindex-cpp/src/core/StorageBuilder.cpp

Go to the documentation of this file.

#include "StorageBuilder.hpp"
#include "MPHashFunction.hpp"
#include "Converter.hpp"
#include "Logging.hpp"
#include "System.hpp"

namespace aitools {
namespace invertedindex {

const std::string
StorageBuilder::data_file("data.");

const std::string
StorageBuilder::mphf_file("mphf");

const std::string
StorageBuilder::vocab_file("vocab");

const std::string
StorageBuilder::storage_file("storage");

const std::string
StorageBuilder::quantile_file("quantile");

StorageBuilder::StorageBuilder()
{}

StorageBuilder::~StorageBuilder()
{
        close();
}

bool
StorageBuilder::is_open() const
{
        return !directory_.empty();
}

const bfs::path&
StorageBuilder::directory() const
{
        return directory_;
}

void
StorageBuilder::close() throw (std::invalid_argument)
{
        if (!is_open()) return;

        // store mphf and vocabulary
        MPHashFunction mphf;
        mphf.create(vocabulary_);
        mphf.save(directory_ / mphf_file);
        vocabulary_.save(directory_ / vocab_file);
        writers_.back()->close();

        // store locations
        std::vector<Location> locations(mphf.size());
        LocationMap::const_iterator lend(locations_.end());
        for (LocationMap::const_iterator it(locations_.begin()); it != lend; ++it)
        {
                locations[mphf.hash(it->first)] = it->second;
        }
        FILE* storage(System::fopen(directory_ / storage_file, "wb"));
        System::fwrite(locations.data(), sizeof(Location), mphf.size(), storage);
        std::fclose(storage);

        // store quantiles
        if (!quantiles_.empty())
        {
                std::vector<Quantile> quantiles(mphf.size());
                QuantileMap::const_iterator qend(quantiles_.end());
                QuantileMap::const_iterator qit(quantiles_.begin());
                for (; qit != qend; ++qit)
                {
                        quantiles[mphf.hash(qit->first)] = qit->second;
                }
                FILE* quantile(System::fopen(directory_ / quantile_file, "wb"));
                System::fwrite(quantiles.data(), Quantile::size, mphf.size(), quantile);
                std::fclose(quantile);
        }
        directory_ = bfs::path();
}

void
StorageBuilder::open(const bfs::path& directory) throw (std::invalid_argument)
{
        if (is_open()) return;
        if (!bfs::exists(directory))
        {
                Exception::throw_invalid_argument("Does not exist", directory);
        }
        directory_ = directory;
}

void
StorageBuilder::put(const std::string& key, Iterator::SharedPointer iterator)
throw (std::runtime_error)
{
        if (!is_open())
        {
                Exception::throw_runtime_error("StorageBuilder is not open");
        }
        if (vocabulary_.contains(key))
        {
                Logging::log("Error: This key is not unique: " + key);
                Logging::log("Start indexing again with InputFormat = PSEUDO_INVERTED");
                Exception::throw_runtime_error("Invalid real inverted file");
        }
        if (writers_.empty())
        {
                writers_.push_back(Writer(new PostlistWriter));
                writers_.back()->open(directory_ / (data_file + "0"));
        }
        else if (writers_.back()->tell() + iterator->header().payload > max_file_size)
        {
                writers_.back()->close();
                std::string id(Converter::ui32_to_str(writers_.size()));
                writers_.push_back(Writer(new PostlistWriter));
                writers_.back()->open(directory_ / (data_file + id));
        }
        Location location(writers_.size()-1, writers_.back()->tell());
        locations_.insert(std::make_pair(key, location));
        vocabulary_.insert(key, iterator->header().value_count);
        writers_.back()->write(iterator);
}

void
StorageBuilder::put(const std::string& key, const Quantile& quantile)
throw (std::runtime_error)
{
        if (!is_open())
        {
                Exception::throw_runtime_error("StorageBuilder is not open");
        }
        if (!vocabulary_.contains(key))
        {
                Exception::throw_runtime_error("Unknown key for quantile");
        }
        quantiles_.insert(std::make_pair(key, quantile));
}

const Vocabulary&
StorageBuilder::vocabulary() const
{
        return vocabulary_;
}

} // namespace invertedindex
} // namespace aitools

---