include/core/big_hash_map.hpp

Go to the documentation of this file.

// big_hash_map.hpp -*- C++ -*-
// Copyright (C) 2011 Martin Trenkmann
#ifndef AITOOLS_BIGHASHMAP_CORE_BIG_HASH_MAP_HPP
#define AITOOLS_BIGHASHMAP_CORE_BIG_HASH_MAP_HPP

#include <boost/algorithm/string.hpp>

#include "core/raw_big_hash_map.hpp"
#include "util/memory.hpp"
#include "value/pair_traits.hpp"
#include "value/triple_traits.hpp"
#include "value/quadruple_traits.hpp"

namespace aitools {
namespace bighashmap {
namespace core {

template <typename T>
class big_hash_map : public raw_big_hash_map {

public:

        typedef cmph_map<T>                      map_type;
        typedef typename map_type::key_type      key_type;
        typedef typename map_type::value_type    value_type;
        typedef typename map_type::checksum_type checksum_type;

public:

        static const std::string index_file_name;

private: // disallow instantiation

        big_hash_map()
        :       raw_big_hash_map() {}

        big_hash_map(const std::vector<map_type*>& sub_maps)
        :       raw_big_hash_map(),
                maps_(sub_maps),
                size_(0)
        {
                for (auto it(maps_.begin()); it != maps_.end(); ++it)
                {
                        size_ += (*it)->size();
                }
        }

        static uint64_t
        table_size(const bfs::path& dir)
        {
                uint64_t size(0);
                if (bfs::exists(dir))
                {
                        const bfs::directory_iterator end;
                        for (bfs::directory_iterator it(dir); it != end; ++it)
                        {
                                if (!bfs::is_regular_file(it->path())) continue;
                                if (boost::ends_with(it->path().string(), "dat") ||
                                        boost::ends_with(it->path().string(), "mph"))
                                {
                                        size += bfs::file_size(it->path());
                                }
                        }
                }
                return size;
        }

public:

        ~big_hash_map()
        {
                for (auto it(maps_.begin()); it != maps_.end(); ++it)
                {
                        delete *it;
                }
        }

public:

        static const bfs::path
        build(const bfs::path& input_dir, const bfs::path& output_dir,
                                        algorithm algo = algorithm::bdz)
        {
                return builder<value_type>::build(input_dir, output_dir, algo);
        }

        static big_hash_map*
        open(const bfs::path& idx_file, util::memory_type mem)
        {
                bfs::ifstream ifs(idx_file);
                if (!ifs)
                {
                        util::throw_runtime_error("Cannot open", idx_file);
                }
                std::string sub_idx_file;
                std::vector<map_type*> sub_maps;
                const uint64_t payload(table_size(idx_file.parent_path()));
                if (payload > util::to_bytes(mem))
                {
                        while (std::getline(ifs, sub_idx_file))
                        {
                                sub_maps.push_back(external_map<T>::open(
                                                idx_file.parent_path() / sub_idx_file));
                        }
                }
                else
                {
                        while (std::getline(ifs, sub_idx_file))
                        {
                                sub_maps.push_back(internal_map<T>::open(
                                                idx_file.parent_path() / sub_idx_file));
                        }
                }
                ifs.close();
                return new big_hash_map(sub_maps);
        }

public:

        bool
        find(const key_type& key, value_type& value)
        {
                return maps_.empty() ? false
                                : maps_[util::hash32(key) % maps_.size()]->find(key, value);
        }

        bool
        find(const std::string& key, char* value_buffer)
        {
                value_type value;
                if (maps_[util::hash32(key) % maps_.size()]->find(key, value))
                {
                        value::value_traits<value_type>::copy_to(value, value_buffer);
                        return true;
                }
                return false;
        }

        uint64_t
        size() const
        {
                return size_;
        }

private:

        std::vector<map_type*> maps_;
        uint64_t               size_;

};

template <typename T>
const std::string
big_hash_map<T>::index_file_name(builder<T>::index_file_name);

} // namespace core
} // namespace bighashmap

using bighashmap::core::big_hash_map;

} // namespace aitools

#endif // AITOOLS_BIGHASHMAP_CORE_BIG_HASH_MAP_HPP

---