One Hat Cyber Team

Edit File: atomic_timed_wait.h

// -*- C++ -*- header.

// Copyright (C) 2020-2021 Free Software Foundation, Inc.
//
// This file is part of the GNU ISO C++ Library.  This library is free
// software; you can redistribute it and/or modify it under the
// terms of the GNU General Public License as published by the
// Free Software Foundation; either version 3, or (at your option)
// any later version.

// This library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU General Public License for more details.

// Under Section 7 of GPL version 3, you are granted additional
// permissions described in the GCC Runtime Library Exception, version
// 3.1, as published by the Free Software Foundation.

// You should have received a copy of the GNU General Public License and
// a copy of the GCC Runtime Library Exception along with this program;
// see the files COPYING3 and COPYING.RUNTIME respectively.  If not, see
// <http://www.gnu.org/licenses/>.

/** @file bits/atomic_timed_wait.h
 *  This is an internal header file, included by other library headers.
 *  Do not attempt to use it directly. @headername{atomic}
 */

#ifndef _GLIBCXX_ATOMIC_TIMED_WAIT_H
#define _GLIBCXX_ATOMIC_TIMED_WAIT_H 1

#pragma GCC system_header

#include <bits/atomic_wait.h>

#if __cpp_lib_atomic_wait
#include <bits/functional_hash.h>
#include <bits/this_thread_sleep.h>

#include <chrono>

#ifdef _GLIBCXX_HAVE_LINUX_FUTEX
#include <exception> // std::terminate
#include <sys/time.h>
#endif

namespace std _GLIBCXX_VISIBILITY(default)
{
_GLIBCXX_BEGIN_NAMESPACE_VERSION

namespace __detail
  {
    using __wait_clock_t = chrono::steady_clock;

template<typename _Clock, typename _Dur>
      __wait_clock_t::time_point
      __to_wait_clock(const chrono::time_point<_Clock, _Dur>& __atime) noexcept
      {
	const typename _Clock::time_point __c_entry = _Clock::now();
	const __wait_clock_t::time_point __w_entry = __wait_clock_t::now();
	const auto __delta = __atime - __c_entry;
	using __w_dur = typename __wait_clock_t::duration;
	return __w_entry + chrono::ceil<__w_dur>(__delta);
      }

template<typename _Dur>
      __wait_clock_t::time_point
      __to_wait_clock(const chrono::time_point<__wait_clock_t,
					       _Dur>& __atime) noexcept
      {
	using __w_dur = typename __wait_clock_t::duration;
	return chrono::ceil<__w_dur>(__atime);
      }

#ifdef _GLIBCXX_HAVE_LINUX_FUTEX
#define _GLIBCXX_HAVE_PLATFORM_TIMED_WAIT
    // returns true if wait ended before timeout
    template<typename _Dur>
      bool
      __platform_wait_until_impl(const __platform_wait_t* __addr,
				 __platform_wait_t __old,
				 const chrono::time_point<__wait_clock_t, _Dur>&
				      __atime) noexcept
      {
	auto __s = chrono::time_point_cast<chrono::seconds>(__atime);
	auto __ns = chrono::duration_cast<chrono::nanoseconds>(__atime - __s);

struct timespec __rt =
	{
	  static_cast<std::time_t>(__s.time_since_epoch().count()),
	  static_cast<long>(__ns.count())
	};

auto __e = syscall (SYS_futex, __addr,
			    static_cast<int>(__futex_wait_flags::
						__wait_bitset_private),
			    __old, &__rt, nullptr,
			    static_cast<int>(__futex_wait_flags::
						__bitset_match_any));

if (__e)
	  {
	    if (errno == ETIMEDOUT)
	      return false;
	    if (errno != EINTR && errno != EAGAIN)
	      __throw_system_error(errno);
	  }
	return true;
      }

// returns true if wait ended before timeout
    template<typename _Clock, typename _Dur>
      bool
      __platform_wait_until(const __platform_wait_t* __addr, __platform_wait_t __old,
			    const chrono::time_point<_Clock, _Dur>& __atime)
      {
	if constexpr (is_same_v<__wait_clock_t, _Clock>)
	  {
	    return __platform_wait_until_impl(__addr, __old, __atime);
	  }
	else
	  {
	    if (!__platform_wait_until_impl(__addr, __old,
					    __to_wait_clock(__atime)))
	      {
		// We got a timeout when measured against __clock_t but
		// we need to check against the caller-supplied clock
		// to tell whether we should return a timeout.
		if (_Clock::now() < __atime)
		  return true;
	      }
	    return false;
	  }
      }
#else
// define _GLIBCXX_HAVE_PLATFORM_TIMED_WAIT and implement __platform_wait_until()
// if there is a more efficient primitive supported by the platform
// (e.g. __ulock_wait())which is better than pthread_cond_clockwait
#endif // ! PLATFORM_TIMED_WAIT

#ifdef _GLIBCXX_HAS_GTHREADS
    // Returns true if wait ended before timeout.
    // _Clock must be either steady_clock or system_clock.
    template<typename _Clock, typename _Dur>
      bool
      __cond_wait_until_impl(__condvar& __cv, mutex& __mx,
			     const chrono::time_point<_Clock, _Dur>& __atime)
      {
	static_assert(std::__is_one_of<_Clock, chrono::steady_clock,
					       chrono::system_clock>::value);

auto __s = chrono::time_point_cast<chrono::seconds>(__atime);
	auto __ns = chrono::duration_cast<chrono::nanoseconds>(__atime - __s);

__gthread_time_t __ts =
	  {
	    static_cast<std::time_t>(__s.time_since_epoch().count()),
	    static_cast<long>(__ns.count())
	  };

#ifdef _GLIBCXX_USE_PTHREAD_COND_CLOCKWAIT
	if constexpr (is_same_v<chrono::steady_clock, _Clock>)
	  __cv.wait_until(__mx, CLOCK_MONOTONIC, __ts);
	else
#endif
	  __cv.wait_until(__mx, __ts);
	return _Clock::now() < __atime;
      }

// returns true if wait ended before timeout
    template<typename _Clock, typename _Dur>
      bool
      __cond_wait_until(__condvar& __cv, mutex& __mx,
	  const chrono::time_point<_Clock, _Dur>& __atime)
      {
#ifdef _GLIBCXX_USE_PTHREAD_COND_CLOCKWAIT
	if constexpr (is_same_v<_Clock, chrono::steady_clock>)
	  return __detail::__cond_wait_until_impl(__cv, __mx, __atime);
	else
#endif
	if constexpr (is_same_v<_Clock, chrono::system_clock>)
	  return __detail::__cond_wait_until_impl(__cv, __mx, __atime);
	else
	  {
	    if (__cond_wait_until_impl(__cv, __mx,
				       __to_wait_clock(__atime)))
	      {
		// We got a timeout when measured against __clock_t but
		// we need to check against the caller-supplied clock
		// to tell whether we should return a timeout.
		if (_Clock::now() < __atime)
		  return true;
	      }
	    return false;
	  }
      }
#endif // _GLIBCXX_HAS_GTHREADS

struct __timed_waiter_pool : __waiter_pool_base
    {
      // returns true if wait ended before timeout
      template<typename _Clock, typename _Dur>
	bool
	_M_do_wait_until(__platform_wait_t* __addr, __platform_wait_t __old,
			 const chrono::time_point<_Clock, _Dur>& __atime)
	{
#ifdef _GLIBCXX_HAVE_PLATFORM_TIMED_WAIT
	  return __platform_wait_until(__addr, __old, __atime);
#else
	  __platform_wait_t __val;
	  __atomic_load(__addr, &__val, __ATOMIC_RELAXED);
	  if (__val == __old)
	    {
	      lock_guard<mutex> __l(_M_mtx);
	      return __cond_wait_until(_M_cv, _M_mtx, __atime);
	    }
	  else
	    return true;
#endif // _GLIBCXX_HAVE_PLATFORM_TIMED_WAIT
	}
    };

struct __timed_backoff_spin_policy
    {
      __wait_clock_t::time_point _M_deadline;
      __wait_clock_t::time_point _M_t0;

template<typename _Clock, typename _Dur>
	__timed_backoff_spin_policy(chrono::time_point<_Clock, _Dur>
				      __deadline = _Clock::time_point::max(),
				    chrono::time_point<_Clock, _Dur>
				      __t0 = _Clock::now()) noexcept
	  : _M_deadline(__to_wait_clock(__deadline))
	  , _M_t0(__to_wait_clock(__t0))
	{ }

bool
      operator()() const noexcept
      {
	using namespace literals::chrono_literals;
	auto __now = __wait_clock_t::now();
	if (_M_deadline <= __now)
	  return false;

// FIXME: this_thread::sleep_for not available #ifdef _GLIBCXX_NO_SLEEP

auto __elapsed = __now - _M_t0;
	if (__elapsed > 128ms)
	  {
	    this_thread::sleep_for(64ms);
	  }
	else if (__elapsed > 64us)
	  {
	    this_thread::sleep_for(__elapsed / 2);
	  }
	else if (__elapsed > 4us)
	  {
	    __thread_yield();
	  }
	else
	  return false;
	return true;
      }
    };

template<typename _EntersWait>
      struct __timed_waiter : __waiter_base<__timed_waiter_pool>
      {
	using __base_type = __waiter_base<__timed_waiter_pool>;

template<typename _Tp>
	  __timed_waiter(const _Tp* __addr) noexcept
	  : __base_type(__addr)
	{
	  if constexpr (_EntersWait::value)
	    _M_w._M_enter_wait();
	}

~__timed_waiter()
	{
	  if constexpr (_EntersWait::value)
	    _M_w._M_leave_wait();
	}

// returns true if wait ended before timeout
	template<typename _Tp, typename _ValFn,
		 typename _Clock, typename _Dur>
	  bool
	  _M_do_wait_until_v(_Tp __old, _ValFn __vfn,
			     const chrono::time_point<_Clock, _Dur>&
								__atime) noexcept
	  {
	    __platform_wait_t __val;
	    if (_M_do_spin(__old, std::move(__vfn), __val,
			   __timed_backoff_spin_policy(__atime)))
	      return true;
	    return __base_type::_M_w._M_do_wait_until(__base_type::_M_addr, __val, __atime);
	  }

// returns true if wait ended before timeout
	template<typename _Pred,
		 typename _Clock, typename _Dur>
	  bool
	  _M_do_wait_until(_Pred __pred, __platform_wait_t __val,
			  const chrono::time_point<_Clock, _Dur>&
							      __atime) noexcept
	  {
	    for (auto __now = _Clock::now(); __now < __atime;
		  __now = _Clock::now())
	      {
		if (__base_type::_M_w._M_do_wait_until(
		      __base_type::_M_addr, __val, __atime)
		    && __pred())
		  return true;

if (__base_type::_M_do_spin(__pred, __val,
			       __timed_backoff_spin_policy(__atime, __now)))
		  return true;
	      }
	    return false;
	  }

// returns true if wait ended before timeout
	template<typename _Pred,
		 typename _Clock, typename _Dur>
	  bool
	  _M_do_wait_until(_Pred __pred,
			   const chrono::time_point<_Clock, _Dur>&
								__atime) noexcept
	  {
	    __platform_wait_t __val;
	    if (__base_type::_M_do_spin(__pred, __val,
					__timed_backoff_spin_policy(__atime)))
	      return true;
	    return _M_do_wait_until(__pred, __val, __atime);
	  }

template<typename _Tp, typename _ValFn,
		 typename _Rep, typename _Period>
	  bool
	  _M_do_wait_for_v(_Tp __old, _ValFn __vfn,
			   const chrono::duration<_Rep, _Period>&
								__rtime) noexcept
	  {
	    __platform_wait_t __val;
	    if (_M_do_spin_v(__old, std::move(__vfn), __val))
	      return true;

if (!__rtime.count())
	      return false; // no rtime supplied, and spin did not acquire

auto __reltime = chrono::ceil<__wait_clock_t::duration>(__rtime);

return __base_type::_M_w._M_do_wait_until(
					  __base_type::_M_addr,
					  __val,
					  chrono::steady_clock::now() + __reltime);
	  }

template<typename _Pred,
		 typename _Rep, typename _Period>
	  bool
	  _M_do_wait_for(_Pred __pred,
			 const chrono::duration<_Rep, _Period>& __rtime) noexcept
	  {
	    __platform_wait_t __val;
	    if (__base_type::_M_do_spin(__pred, __val))
	      return true;

if (!__rtime.count())
	      return false; // no rtime supplied, and spin did not acquire

auto __reltime = chrono::ceil<__wait_clock_t::duration>(__rtime);

return _M_do_wait_until(__pred, __val,
				    chrono::steady_clock::now() + __reltime);
	  }
      };

using __enters_timed_wait = __timed_waiter<std::true_type>;
    using __bare_timed_wait = __timed_waiter<std::false_type>;
  } // namespace __detail

// returns true if wait ended before timeout
  template<typename _Tp, typename _ValFn,
	   typename _Clock, typename _Dur>
    bool
    __atomic_wait_address_until_v(const _Tp* __addr, _Tp&& __old, _ValFn&& __vfn,
			const chrono::time_point<_Clock, _Dur>&
			    __atime) noexcept
    {
      __detail::__enters_timed_wait __w{__addr};
      return __w._M_do_wait_until_v(__old, __vfn, __atime);
    }

template<typename _Tp, typename _Pred,
	   typename _Clock, typename _Dur>
    bool
    __atomic_wait_address_until(const _Tp* __addr, _Pred __pred,
				const chrono::time_point<_Clock, _Dur>&
							      __atime) noexcept
    {
      __detail::__enters_timed_wait __w{__addr};
      return __w._M_do_wait_until(__pred, __atime);
    }

template<typename _Pred,
	   typename _Clock, typename _Dur>
    bool
    __atomic_wait_address_until_bare(const __detail::__platform_wait_t* __addr,
				_Pred __pred,
				const chrono::time_point<_Clock, _Dur>&
							      __atime) noexcept
    {
      __detail::__bare_timed_wait __w{__addr};
      return __w._M_do_wait_until(__pred, __atime);
    }

template<typename _Tp, typename _ValFn,
	   typename _Rep, typename _Period>
    bool
    __atomic_wait_address_for_v(const _Tp* __addr, _Tp&& __old, _ValFn&& __vfn,
		      const chrono::duration<_Rep, _Period>& __rtime) noexcept
    {
      __detail::__enters_timed_wait __w{__addr};
      return __w._M_do_wait_for_v(__old, __vfn, __rtime);
    }

template<typename _Tp, typename _Pred,
	   typename _Rep, typename _Period>
    bool
    __atomic_wait_address_for(const _Tp* __addr, _Pred __pred,
		      const chrono::duration<_Rep, _Period>& __rtime) noexcept
    {

__detail::__enters_timed_wait __w{__addr};
      return __w._M_do_wait_for(__pred, __rtime);
    }

template<typename _Pred,
	   typename _Rep, typename _Period>
    bool
    __atomic_wait_address_for_bare(const __detail::__platform_wait_t* __addr,
			_Pred __pred,
			const chrono::duration<_Rep, _Period>& __rtime) noexcept
    {
      __detail::__bare_timed_wait __w{__addr};
      return __w._M_do_wait_for(__pred, __rtime);
    }
_GLIBCXX_END_NAMESPACE_VERSION
} // namespace std
#endif // __cpp_lib_atomic_wait
#endif // _GLIBCXX_ATOMIC_TIMED_WAIT_H