Version: 2.0.0

home

The disk I/O can be customized in libtorrent. In previous versions, the customization was at the level of each torrent. Now, the customization point is at the session level. All torrents added to a session will use the same disk I/O subsystem, as determined by the disk_io_constructor (in session_params).

This allows the disk subsystem to also customize threading and disk job management.

To customize the disk subsystem, implement disk_interface and provide a factory function to the session constructor (via session_params).

Example use:

struct temp_storage
{
  explicit temp_storage(lt::file_storage const& fs) : m_files(fs) {}

  lt::span<char const> readv(lt::piece_index_t const piece, int const offset, lt::storage_error& ec) const
  {
    auto const i = m_file_data.find(piece);
    if (i == m_file_data.end())
    {
      ec.operation = lt::operation_t::file_read;
      ec.ec = boost::asio::error::eof;
      return {};
    }
    if (int(i->second.size()) <= offset)
    {
      ec.operation = lt::operation_t::file_read;
      ec.ec = boost::asio::error::eof;
      return {};
    }
    return { i->second.data() + offset, int(i->second.size()) - offset };
  }
  void writev(lt::span<char const> const b, lt::piece_index_t const piece, int const offset)
  {
    auto& data = m_file_data[piece];
    if (data.empty())
    {
      // allocate the whole piece, otherwise we'll invalidate the pointers
      // we have returned back to libtorrent
      int const size = piece_size(piece);
      data.resize(std::size_t(size));
    }
    TORRENT_ASSERT(offset + b.size() <= int(data.size()));
    std::memcpy(data.data() + offset, b.data(), std::size_t(b.size()));
  }
  lt::sha1_hash hash(lt::piece_index_t const piece
    , lt::span<lt::sha256_hash> const block_hashes, lt::storage_error& ec) const
  {
    auto const i = m_file_data.find(piece);
    if (i == m_file_data.end())
    {
      ec.operation = lt::operation_t::file_read;
      ec.ec = boost::asio::error::eof;
      return {};
    }
    if (!block_hashes.empty())
    {
      int const piece_size2 = m_files.piece_size2(piece);
      int const blocks_in_piece2 = m_files.blocks_in_piece2(piece);
      char const* buf = i->second.data();
      std::int64_t offset = 0;
      for (int k = 0; k < blocks_in_piece2; ++k)
      {
        lt::hasher256 h2;
        std::ptrdiff_t const len2 = std::min(lt::default_block_size, int(piece_size2 - offset));
        h2.update({ buf, len2 });
        buf += len2;
        offset += len2;
        block_hashes[k] = h2.final();
      }
    }
    return lt::hasher(i->second).final();
  }
  lt::sha256_hash hash2(lt::piece_index_t const piece, int const offset, lt::storage_error& ec)
  {
    auto const i = m_file_data.find(piece);
    if (i == m_file_data.end())
    {
      ec.operation = lt::operation_t::file_read;
      ec.ec = boost::asio::error::eof;
      return {};
    }

    int const piece_size = m_files.piece_size2(piece);

    std::ptrdiff_t const len = std::min(lt::default_block_size, piece_size - offset);

    lt::span<char const> b = {i->second.data() + offset, len};
    return lt::hasher256(b).final();
  }

private:
  int piece_size(lt::piece_index_t piece) const
  {
    int const num_pieces = static_cast<int>((m_files.total_size() + m_files.piece_length() - 1) / m_files.piece_length());
    return static_cast<int>(piece) < num_pieces - 1
      ? m_files.piece_length() : static_cast<int>(m_files.total_size() - (num_pieces - 1) * m_files.piece_length());
  }

  lt::file_storage const& m_files;
  std::map<lt::piece_index_t, std::vector<char>> m_file_data;
};

lt::storage_index_t pop(std::vector<lt::storage_index_t>& q)
{
  TORRENT_ASSERT(!q.empty());
  lt::storage_index_t const ret = q.back();
  q.pop_back();
  return ret;
}

struct temp_disk_io final : lt::disk_interface
  , lt::buffer_allocator_interface
{
  explicit temp_disk_io(lt::io_context& ioc): m_ioc(ioc) {}

  void settings_updated() override {}

  lt::storage_holder new_torrent(lt::storage_params const& params
    , std::shared_ptr<void> const&) override
  {
    lt::storage_index_t const idx = m_free_slots.empty()
      ? m_torrents.end_index()
      : pop(m_free_slots);
      auto storage = std::make_unique<temp_storage>(params.files);
    if (idx == m_torrents.end_index()) m_torrents.emplace_back(std::move(storage));
    else m_torrents[idx] = std::move(storage);
    return lt::storage_holder(idx, *this);
  }

  void remove_torrent(lt::storage_index_t const idx) override
  {
    m_torrents[idx].reset();
    m_free_slots.push_back(idx);
  }

  void abort(bool) override {}

  void async_read(lt::storage_index_t storage, lt::peer_request const& r
    , std::function<void(lt::disk_buffer_holder block, lt::storage_error const& se)> handler
    , lt::disk_job_flags_t) override
  {
    // this buffer is owned by the storage. It will remain valid for as
    // long as the torrent remains in the session. We don't need any lifetime
    // management of it.
    lt::storage_error error;
    lt::span<char const> b = m_torrents[storage]->readv(r.piece, r.start, error);

    post(m_ioc, [handler, error, b, this]
      { handler(lt::disk_buffer_holder(*this, const_cast<char*>(b.data()), int(b.size())), error); });
  }

  bool async_write(lt::storage_index_t storage, lt::peer_request const& r
    , char const* buf, std::shared_ptr<lt::disk_observer>
    , std::function<void(lt::storage_error const&)> handler
    , lt::disk_job_flags_t) override
  {
    lt::span<char const> const b = { buf, r.length };

    m_torrents[storage]->writev(b, r.piece, r.start);

    post(m_ioc, [=]{ handler(lt::storage_error()); });
    return false;
  }

  void async_hash(lt::storage_index_t storage, lt::piece_index_t const piece
    , lt::span<lt::sha256_hash> block_hashes, lt::disk_job_flags_t
    , std::function<void(lt::piece_index_t, lt::sha1_hash const&, lt::storage_error const&)> handler) override
  {
    lt::storage_error error;
    lt::sha1_hash const hash = m_torrents[storage]->hash(piece, block_hashes, error);
    post(m_ioc, [=]{ handler(piece, hash, error); });
  }

  void async_hash2(lt::storage_index_t storage, lt::piece_index_t const piece
    , int const offset, lt::disk_job_flags_t
    , std::function<void(lt::piece_index_t, lt::sha256_hash const&, lt::storage_error const&)> handler) override
  {
    lt::storage_error error;
    lt::sha256_hash const hash = m_torrents[storage]->hash2(piece, offset, error);
    post(m_ioc, [=]{ handler(piece, hash, error); });
  }

  void async_move_storage(lt::storage_index_t, std::string p, lt::move_flags_t
    , std::function<void(lt::status_t, std::string const&, lt::storage_error const&)> handler) override
  {
    post(m_ioc, [=]{
      handler(lt::status_t::fatal_disk_error, p
        , lt::storage_error(lt::error_code(boost::system::errc::operation_not_supported, lt::system_category())));
    });
  }

  void async_release_files(lt::storage_index_t, std::function<void()>) override {}

  void async_delete_files(lt::storage_index_t, lt::remove_flags_t
    , std::function<void(lt::storage_error const&)> handler) override
  {
    post(m_ioc, [=]{ handler(lt::storage_error()); });
  }

  void async_check_files(lt::storage_index_t
    , lt::add_torrent_params const*
    , lt::aux::vector<std::string, lt::file_index_t>
    , std::function<void(lt::status_t, lt::storage_error const&)> handler) override
  {
    post(m_ioc, [=]{ handler(lt::status_t::no_error, lt::storage_error()); });
  }

  void async_rename_file(lt::storage_index_t
    , lt::file_index_t const idx
    , std::string const name
    , std::function<void(std::string const&, lt::file_index_t, lt::storage_error const&)> handler) override
  {
    post(m_ioc, [=]{ handler(name, idx, lt::storage_error()); });
  }

  void async_stop_torrent(lt::storage_index_t, std::function<void()> handler) override
  {
    post(m_ioc, handler);
  }

  void async_set_file_priority(lt::storage_index_t
    , lt::aux::vector<lt::download_priority_t, lt::file_index_t> prio
    , std::function<void(lt::storage_error const&
      , lt::aux::vector<lt::download_priority_t, lt::file_index_t>)> handler) override
  {
    post(m_ioc, [=]{
      handler(lt::storage_error(lt::error_code(
        boost::system::errc::operation_not_supported, lt::system_category())), std::move(prio));
    });
  }

  void async_clear_piece(lt::storage_index_t, lt::piece_index_t index
    , std::function<void(lt::piece_index_t)> handler) override
  {
    post(m_ioc, [=]{ handler(index); });
  }

  // implements buffer_allocator_interface
  void free_disk_buffer(char*) override
  {
    // never free any buffer. We only return buffers owned by the storage
    // object
  }

  void update_stats_counters(lt::counters&) const override {}

  std::vector<lt::open_file_state> get_status(lt::storage_index_t) const override
  { return {}; }

  void submit_jobs() override {}

private:

  lt::aux::vector<std::shared_ptr<temp_storage>, lt::storage_index_t> m_torrents;

  // slots that are unused in the m_torrents vector
  std::vector<lt::storage_index_t> m_free_slots;

  // callbacks are posted on this
  lt::io_context& m_ioc;
};

std::unique_ptr<lt::disk_interface> temp_disk_constructor(
  lt::io_context& ioc, lt::settings_interface const&, lt::counters&)
{
  return std::make_unique<temp_disk_io>(ioc);
}
[report issue]

settings_interface

Declared in "libtorrent/settings_pack.hpp"

the common interface to settings_pack and the internal representation of settings.

struct settings_interface
{
   virtual void set_int (int name, int val) = 0;
   virtual void set_str (int name, std::string val) = 0;
   virtual bool has_val (int name) const = 0;
   virtual void set_bool (int name, bool val) = 0;
   virtual std::string const& get_str (int name) const = 0;
   virtual int get_int (int name) const = 0;
   virtual bool get_bool (int name) const = 0;
};
[report issue]

open_file_state

Declared in "libtorrent/disk_interface.hpp"

this contains information about a file that's currently open by the libtorrent disk I/O subsystem. It's associated with a single torrent.

struct open_file_state
{
   file_index_t file_index;
   file_open_mode_t open_mode;
   time_point last_use;
};
[report issue]
file_index
the index of the file this entry refers to into the file_storage file list of this torrent. This starts indexing at 0.
[report issue]
open_mode

open_mode is a bitmask of the file flags this file is currently opened with. These are the flags used in the file::open() function. For possible flags, see file_open_mode_t.

Note that the read/write mode is not a bitmask. The two least significant bits are used to represent the read/write mode. Those bits can be masked out using the rw_mask constant.

[report issue]
last_use
a (high precision) timestamp of when the file was last used.
[report issue]

disk_interface

Declared in "libtorrent/disk_interface.hpp"

The disk_interface is the customization point for disk I/O in libtorrent. implement this interface and provide a factory function to the session constructor use custom disk I/O.

struct disk_interface
{
   virtual storage_holder new_torrent (storage_params const& p
      , std::shared_ptr<void> const& torrent) = 0;
   virtual void remove_torrent (storage_index_t) = 0;
   virtual void async_read (storage_index_t storage, peer_request const& r
      , std::function<void(disk_buffer_holder, storage_error const&)> handler
      , disk_job_flags_t flags = {}) = 0;
   virtual bool async_write (storage_index_t storage, peer_request const& r
      , char const* buf, std::shared_ptr<disk_observer> o
      , std::function<void(storage_error const&)> handler
      , disk_job_flags_t flags = {}) = 0;
   virtual void async_hash (storage_index_t storage, piece_index_t piece, span<sha256_hash> v2
      , disk_job_flags_t flags
      , std::function<void(piece_index_t, sha1_hash const&, storage_error const&)> handler) = 0;
   virtual void async_rename_file (storage_index_t storage
      , file_index_t index, std::string name
      , std::function<void(std::string const&, file_index_t, storage_error const&)> handler) = 0;
   virtual void async_stop_torrent (storage_index_t storage
      , std::function<void()> handler = std::function<void()>()) = 0;
   virtual void async_move_storage (storage_index_t storage, std::string p, move_flags_t flags
      , std::function<void(status_t, std::string const&, storage_error const&)> handler) = 0;
   virtual void async_delete_files (storage_index_t storage, remove_flags_t options
      , std::function<void(storage_error const&)> handler) = 0;
   virtual void async_release_files (storage_index_t storage
      , std::function<void()> handler = std::function<void()>()) = 0;
   virtual void async_check_files (storage_index_t storage
      , add_torrent_params const* resume_data
      , aux::vector<std::string, file_index_t> links
      , std::function<void(status_t, storage_error const&)> handler) = 0;
   virtual void async_set_file_priority (storage_index_t storage
      , aux::vector<download_priority_t, file_index_t> prio
      , std::function<void(storage_error const&
      , aux::vector<download_priority_t, file_index_t>)> handler) = 0;
   virtual void async_hash2 (storage_index_t storage, piece_index_t piece, int offset, disk_job_flags_t flags
      , std::function<void(piece_index_t, sha256_hash const&, storage_error const&)> handler) = 0;
   virtual void async_clear_piece (storage_index_t storage, piece_index_t index
      , std::function<void(piece_index_t)> handler) = 0;
   virtual void update_stats_counters (counters& c) const = 0;
   virtual std::vector<open_file_state> get_status (storage_index_t) const = 0;
   virtual void submit_jobs () = 0;
   virtual void settings_updated () = 0;
   virtual void abort (bool wait) = 0;
   virtual ~disk_interface ();

   static constexpr disk_job_flags_t force_copy  = 0_bit;
   static constexpr disk_job_flags_t sequential_access  = 3_bit;
   static constexpr disk_job_flags_t volatile_read  = 4_bit;
   static constexpr disk_job_flags_t v1_hash  = 5_bit;
};
[report issue]

async_hash()

virtual void async_hash (storage_index_t storage, piece_index_t piece, span<sha256_hash> v2
      , disk_job_flags_t flags
      , std::function<void(piece_index_t, sha1_hash const&, storage_error const&)> handler) = 0;

if v2 is non-empty it must be at least large enough to hold all v2 blocks in the piece

[report issue]

async_rename_file() async_stop_torrent() async_check_files() async_release_files() async_hash2() async_delete_files() async_set_file_priority() async_move_storage()

virtual void async_rename_file (storage_index_t storage
      , file_index_t index, std::string name
      , std::function<void(std::string const&, file_index_t, storage_error const&)> handler) = 0;
virtual void async_stop_torrent (storage_index_t storage
      , std::function<void()> handler = std::function<void()>()) = 0;
virtual void async_move_storage (storage_index_t storage, std::string p, move_flags_t flags
      , std::function<void(status_t, std::string const&, storage_error const&)> handler) = 0;
virtual void async_delete_files (storage_index_t storage, remove_flags_t options
      , std::function<void(storage_error const&)> handler) = 0;
virtual void async_release_files (storage_index_t storage
      , std::function<void()> handler = std::function<void()>()) = 0;
virtual void async_check_files (storage_index_t storage
      , add_torrent_params const* resume_data
      , aux::vector<std::string, file_index_t> links
      , std::function<void(status_t, storage_error const&)> handler) = 0;
virtual void async_set_file_priority (storage_index_t storage
      , aux::vector<download_priority_t, file_index_t> prio
      , std::function<void(storage_error const&
      , aux::vector<download_priority_t, file_index_t>)> handler) = 0;
virtual void async_hash2 (storage_index_t storage, piece_index_t piece, int offset, disk_job_flags_t flags
      , std::function<void(piece_index_t, sha256_hash const&, storage_error const&)> handler) = 0;

async_hash2 computes the v2 hash of a single block

[report issue]
force_copy
force making a copy of the cached block, rather than getting a reference to the block already in the cache.
[report issue]
sequential_access
hint that there may be more disk operations with sequential access to the file
[report issue]
volatile_read
don't keep the read block in cache
[report issue]
v1_hash
compute a v1 piece hash
[report issue]

storage_holder

Declared in "libtorrent/disk_interface.hpp"

a unique, owning, reference to the storage of a torrent in a disk io subsystem (class that implements disk_interface). This is held by the internal libtorrent torrent object to tie the storage object allocated for a torrent to the lifetime of the internal torrent object. When a torrent is removed from the session, this holder is destructed and will inform the disk object.

struct storage_holder
{
   storage_holder (storage_index_t idx, disk_interface& disk_io);
   storage_holder () = default;
   ~storage_holder ();
   explicit operator bool () const;
   operator storage_index_t () const;
   void reset ();
   storage_holder& operator= (storage_holder const&) = delete;
   storage_holder (storage_holder const&) = delete;
   storage_holder (storage_holder&& rhs) noexcept;
   storage_holder& operator= (storage_holder&& rhs) noexcept;
};
[report issue]

buffer_allocator_interface

Declared in "libtorrent/disk_buffer_holder.hpp"

the interface for freeing disk buffers, used by the disk_buffer_holder. when implementing disk_interface, this must also be implemented in order to return disk buffers back to libtorrent

struct buffer_allocator_interface
{
   virtual void free_disk_buffer (char* b) = 0;
};
[report issue]

disk_buffer_holder

Declared in "libtorrent/disk_buffer_holder.hpp"

The disk buffer holder acts like a unique_ptr that frees a disk buffer when it's destructed

If this buffer holder is moved-from, default constructed or reset, data() will return nullptr.

struct disk_buffer_holder
{
   disk_buffer_holder& operator= (disk_buffer_holder&&) & noexcept;
   disk_buffer_holder (disk_buffer_holder&&) noexcept;
   disk_buffer_holder (disk_buffer_holder const&) = delete;
   disk_buffer_holder& operator= (disk_buffer_holder const&) = delete;
   disk_buffer_holder (buffer_allocator_interface& alloc
      , char* buf, int sz) noexcept;
   disk_buffer_holder () noexcept = default;
   ~disk_buffer_holder ();
   char* data () const noexcept;
   void reset ();
   void swap (disk_buffer_holder& h) noexcept;
   bool is_mutable () const noexcept;
   explicit operator bool () const noexcept;
   std::ptrdiff_t size () const;
};
[report issue]

disk_buffer_holder()

disk_buffer_holder (buffer_allocator_interface& alloc
      , char* buf, int sz) noexcept;

construct a buffer holder that will free the held buffer using a disk buffer pool directly (there's only one disk_buffer_pool per session)

[report issue]

disk_buffer_holder()

disk_buffer_holder () noexcept = default;

default construct a holder that does not own any buffer

[report issue]

~disk_buffer_holder()

~disk_buffer_holder ();

frees disk buffer held by this object

[report issue]

data()

char* data () const noexcept;

return a pointer to the held buffer, if any. Otherwise returns nullptr.

[report issue]

reset()

void reset ();

free the held disk buffer, if any, and clear the holder. This sets the holder object to a default-constructed state

[report issue]

swap()

void swap (disk_buffer_holder& h) noexcept;

swap pointers of two disk buffer holders.

[report issue]

is_mutable()

bool is_mutable () const noexcept;

if this returns true, the buffer may not be modified in place

[report issue]

bool()

explicit operator bool () const noexcept;

implicitly convertible to true if the object is currently holding a buffer

[report issue]

disk_observer

Declared in "libtorrent/disk_observer.hpp"

struct disk_observer
{
   virtual void on_disk () = 0;
};
[report issue]

on_disk()

virtual void on_disk () = 0;

called when the disk cache size has dropped below the low watermark again and we can resume downloading from peers

[report issue]

file_open_mode_t

Declared in "libtorrent/disk_interface.hpp"

read_only
open the file for reading only
write_only
open the file for writing only
read_write
open the file for reading and writing
rw_mask
the mask for the bits determining read or write mode
sparse
open the file in sparse mode (if supported by the filesystem).
no_atime
don't update the access timestamps on the file (if supported by the operating system and filesystem). this generally improves disk performance.
random_access
open the file for random access. This disables read-ahead logic