include/core/external_map.hpp

Go to the documentation of this file.

// external_map.hpp -*- C++ -*-
// Copyright (C) 2011 Martin Trenkmann
#ifndef AITOOLS_BIGHASHMAP_CORE_EXTERNAL_MAP_HPP
#define AITOOLS_BIGHASHMAP_CORE_EXTERNAL_MAP_HPP

#include <mutex>
#include <cmph.h>
#include <boost/filesystem/fstream.hpp>

#include "core/cmph_map.hpp"
#include "util/logging.hpp"
#include "util/checksum.hpp"
#include "util/systemio.hpp"
#include "value/pair.hpp"
#include "value/pair_traits.hpp"

namespace bfs = boost::filesystem;

namespace aitools {
namespace bighashmap {
namespace core {

template <typename T>
class external_map : public cmph_map<T> {

public:

        typedef cmph_map<T>                       base_type;
        typedef typename base_type::key_type      key_type;
        typedef typename base_type::value_type    value_type;
        typedef typename base_type::checksum_type checksum_type;

private:

        typedef value::pair<checksum_type, value_type> entry_type;
        typedef value::value_traits<entry_type>        entry_traits;

private: // disallow instantiation

        external_map()
        :       base_type() {}

        external_map(const bfs::path& mph_file, const bfs::path& dat_file)
        :       base_type(mph_file)
        {
                data_ = util::fopen(dat_file, "rb");
        }

public:

        ~external_map()
        {
                util::fclose(data_);
        }

public:

        static external_map*
        open(const bfs::path& idx_file)
        {
                bfs::ifstream ifs(idx_file);
                if (!ifs)
                {
                        util::throw_runtime_error("Cannot open", idx_file);
                }
                std::string mph_file;
                std::string dat_file;
                std::getline(ifs, mph_file);
                std::getline(ifs, dat_file);
                return new external_map(idx_file.parent_path() / mph_file,
                                                idx_file.parent_path() / dat_file);
        }

public:

        bool
        find(const key_type& key, value_type& value)
        {
                entry_type entry;
                const uint64_t offset(hash(key) * entry_traits::size_of(entry));

#ifdef THREAD_SAFE
                std::lock_guard<std::mutex> lg(mutex_);
#endif

                // Seek to table entry
                util::fseek(data_, offset, SEEK_SET);
                // Read checksum and value
                if (!entry_traits::read_from(entry, data_))
                {
                        util::log_error("Key", key);
                        util::log_error("File offset", offset);
                        util::throw_runtime_error("entry_traits::read_from failed");
                }
                // Check checksum
                const checksum_type checksum(util::hash<checksum_type>(key));
                if (entry.e1() != checksum)
                {
                        // This is no error, unknown keys normally cause a hash collision.
                        log_debug(__FILE__, __LINE__, "Key", key);
                        log_debug(__FILE__, __LINE__, "File offset", offset);
                        log_debug(__FILE__, __LINE__, "Actual checksum", checksum);
                        log_debug(__FILE__, __LINE__, "Expected checksum", entry.e1());
                        return false;
                }
                value = entry.e2();
                return true;
        }

private:

        FILE* data_;

#ifdef THREAD_SAFE
        std::mutex mutex_;
#endif

};

} // namespace core
} // namespace bighashmap
} // namespace aitools

#endif // AITOOLS_BIGHASHMAP_CORE_EXTERNAL_MAP_HPP

---