XrdPfcInfo.hh

Go to the documentation of this file.

#ifndef __XRDPFC_INFO_HH__
#define __XRDPFC_INFO_HH__
//----------------------------------------------------------------------------------
// Copyright (c) 2014 by Board of Trustees of the Leland Stanford, Jr., University
// Author: Alja Mrak-Tadel, Matevz  Tadel, Brian Bockelman
//----------------------------------------------------------------------------------
// XRootD is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// XRootD is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with XRootD.  If not, see <http://www.gnu.org/licenses/>.
//----------------------------------------------------------------------------------
 
#include <cstdio>
#include <ctime>
#include <assert.h>
#include <vector>
 
#include "XrdSys/XrdSysPthread.hh"
#include "XrdCl/XrdClConstants.hh"
#include "XrdCl/XrdClDefaultEnv.hh"
 
#include "XrdPfcTypes.hh"
 
class XrdOssDF;
class XrdCksCalc;
class XrdSysTrace;
 
namespace XrdPfc
{
class Stats;
 
//----------------------------------------------------------------------------
//----------------------------------------------------------------------------
 
class Info
{
public:
   struct Status {
      union {
         struct {
            int f_cksum_check : 3;   
 
            int _free_bits_ : 29;
         };
         unsigned int _raw_;
      };
      Status() : _raw_(0) {}
   };
 
   struct AStat
   {
      time_t    AttachTime;       
      time_t    DetachTime;       
      int       NumIos;           
      int       Duration;         
      int       NumMerged;        
      int       Reserved;         
      long long BytesHit;         
      long long BytesMissed;      
      long long BytesBypassed;    
 
      AStat() :
         AttachTime(0), DetachTime(0), NumIos(0), Duration(0), NumMerged(0), Reserved(0),
         BytesHit(0), BytesMissed(0), BytesBypassed(0)
      {}
 
      void MergeWith(const AStat &a);
   };
 
   struct Store
   {
      long long          m_buffer_size;            
      long long          m_file_size;              
      time_t             m_creationTime;           
      time_t             m_noCkSumTime;            
      size_t             m_accessCnt;              
      Status             m_status;                 
      int                m_astatSize;              
 
      Store () :
         m_buffer_size(0), m_file_size(0), m_creationTime(0), m_noCkSumTime(0),
         m_accessCnt(0),   m_astatSize(0)
      {}
   };
 
 
   //------------------------------------------------------------------------
   //------------------------------------------------------------------------
   Info(XrdSysTrace* trace, bool prefetchBuffer = false);
 
   //------------------------------------------------------------------------
   //------------------------------------------------------------------------
   ~Info();
 
   //---------------------------------------------------------------------
   //---------------------------------------------------------------------
   void SetBitWritten(int i);
 
   //---------------------------------------------------------------------
   //---------------------------------------------------------------------
   bool TestBitWritten(int i) const;
 
   //---------------------------------------------------------------------
   //---------------------------------------------------------------------
   bool TestBitPrefetch(int i) const;
 
   //---------------------------------------------------------------------
   //---------------------------------------------------------------------
   void SetBitPrefetch(int i);
 
   //---------------------------------------------------------------------
   //---------------------------------------------------------------------
   void SetBitSynced(int i);
 
   //---------------------------------------------------------------------
   //---------------------------------------------------------------------
   void SetAllBitsSynced();
 
   void SetBufferSizeFileSizeAndCreationTime(long long bs, long long fs);
 
   //---------------------------------------------------------------------
   //---------------------------------------------------------------------
   void ResizeBits();
 
   //---------------------------------------------------------------------
   //---------------------------------------------------------------------
   bool Read(XrdOssDF* fp, const char *dname, const char *fname = 0);
 
   //---------------------------------------------------------------------
   //---------------------------------------------------------------------
   bool Write(XrdOssDF* fp, const char *dname, const char *fname = 0);
 
   //---------------------------------------------------------------------
   //---------------------------------------------------------------------
   void CompactifyAccessRecords();
 
   //---------------------------------------------------------------------
   //---------------------------------------------------------------------
   void ResetAllAccessStats();
 
   //---------------------------------------------------------------------
   //---------------------------------------------------------------------
   void WriteIOStatAttach();
 
   //---------------------------------------------------------------------
   //---------------------------------------------------------------------
   void WriteIOStat(Stats& s);
 
  //---------------------------------------------------------------------
   //---------------------------------------------------------------------
   void WriteIOStatDetach(Stats& s);
 
   //---------------------------------------------------------------------
   //---------------------------------------------------------------------
   void WriteIOStatSingle(long long bytes_disk);
 
   //---------------------------------------------------------------------
   //---------------------------------------------------------------------
   void WriteIOStatSingle(long long bytes_disk, time_t att, time_t dtc);
 
   //---------------------------------------------------------------------
   //---------------------------------------------------------------------
   int CountBlocksNotWrittenInRng(int firstIdx, int lastIdx) const;
 
   //---------------------------------------------------------------------
   //---------------------------------------------------------------------
   int GetBitvecSizeInBytes() const;
 
   //---------------------------------------------------------------------
   //---------------------------------------------------------------------
   int GetNBlocks() const;
 
   //---------------------------------------------------------------------
   //---------------------------------------------------------------------
   long long GetFileSize() const;
 
   //---------------------------------------------------------------------
   //---------------------------------------------------------------------
   bool GetLatestDetachTime(time_t& t) const;
 
   //---------------------------------------------------------------------
   //---------------------------------------------------------------------
   const AStat* GetLastAccessStats() const;
 
   //---------------------------------------------------------------------
   //---------------------------------------------------------------------
   long long GetBufferSize() const;
 
   //---------------------------------------------------------------------
   //---------------------------------------------------------------------
   bool IsComplete() const;
 
   //---------------------------------------------------------------------
   //---------------------------------------------------------------------
   int GetNDownloadedBlocks() const;
 
   //---------------------------------------------------------------------
   //---------------------------------------------------------------------
   long long GetNDownloadedBytes() const;
 
   //---------------------------------------------------------------------
   //---------------------------------------------------------------------
   int GetLastDownloadedBlock() const;
 
   //---------------------------------------------------------------------
   //---------------------------------------------------------------------
   long long GetExpectedDataFileSize() const;
 
  //---------------------------------------------------------------------
   //---------------------------------------------------------------------
   void UpdateDownloadCompleteStatus();
 
   //---------------------------------------------------------------------
   //---------------------------------------------------------------------
   size_t GetAccessCnt() const { return m_store.m_accessCnt; }
 
   //---------------------------------------------------------------------
   //---------------------------------------------------------------------
   int GetVersion() { return m_version; }
 
   //---------------------------------------------------------------------
   //---------------------------------------------------------------------
   const Store&              RefStoredData() const { return m_store;  }
   const std::vector<AStat>& RefAStats()     const { return m_astats; }
 
   //---------------------------------------------------------------------
   //---------------------------------------------------------------------
   time_t GetCreationTime() const { return m_store.m_creationTime; }
 
   //---------------------------------------------------------------------
   //---------------------------------------------------------------------
   uint32_t CalcCksumStore();
   uint32_t CalcCksumSyncedAndAStats();
   void     CalcCksumMd5(unsigned char* buff, char* digest);
 
   CkSumCheck_e GetCkSumState()  const { return (CkSumCheck_e) m_store.m_status.f_cksum_check; }
   const char*  GetCkSumStateAsText() const;
 
   bool IsCkSumCache() const { return  m_store.m_status.f_cksum_check & CSChk_Cache; }
   bool IsCkSumNet()   const { return  m_store.m_status.f_cksum_check & CSChk_Net;   }
   bool IsCkSumAny()   const { return  m_store.m_status.f_cksum_check & CSChk_Both;  }
   bool IsCkSumBoth()  const { return (m_store.m_status.f_cksum_check & CSChk_Both) == CSChk_Both; }
 
   void SetCkSumState(CkSumCheck_e css)           { m_store.m_status.f_cksum_check  = css; }
   void DowngradeCkSumState(CkSumCheck_e css_ref) { m_store.m_status.f_cksum_check &= css_ref; }
   void ResetCkSumCache();
   void ResetCkSumNet();
 
   bool   HasNoCkSumTime() const { return m_store.m_noCkSumTime != 0; }
   time_t GetNoCkSumTime() const { return m_store.m_noCkSumTime; }
   time_t GetNoCkSumTimeForUVKeep() const { return m_store.m_noCkSumTime ? m_store.m_noCkSumTime : m_store.m_creationTime; }
   void   ResetNoCkSumTime() { m_store.m_noCkSumTime = 0; }
 
#ifdef XRDPFC_CKSUM_TEST
   static void TestCksumStuff();
#endif
 
   static const char*   m_traceID;          // has to be m_ (convention in TRACE macros)
   static const char*   s_infoExtension;
   static const size_t  s_infoExtensionLen;
   static       size_t  s_maxNumAccess;     // can be set from configuration
   static const int     s_defaultVersion;
 
   XrdSysTrace* GetTrace() const {return m_trace; }
 
protected:
   XrdSysTrace*   m_trace;
 
   Store          m_store;
   unsigned char *m_buff_synced;             
   unsigned char *m_buff_written;            
   unsigned char *m_buff_prefetch;           
   std::vector<AStat>  m_astats;             
 
   int  m_version;
   int  m_bitvecSizeInBits;                  
   int  m_missingBlocks;                     
   bool m_complete;                          
   bool m_hasPrefetchBuffer;                 
 
private:
   inline unsigned char cfiBIT(int n) const { return 1 << n; }
 
   // Reading functions for older cinfo file formats
   bool ReadV2(XrdOssDF* fp, off_t off, const char *dname, const char *fname);
   bool ReadV3(XrdOssDF* fp, off_t off, const char *dname, const char *fname);
 
   XrdCksCalc*   m_cksCalcMd5;
};
 
//------------------------------------------------------------------------------
 
inline bool Info::TestBitWritten(int i) const
{
   const int cn = i/8;
   assert(cn < GetBitvecSizeInBytes());
 
   const int off = i - cn*8;
   return (m_buff_written[cn] & cfiBIT(off)) != 0;
}
 
inline void Info::SetBitWritten(int i)
{
   const int cn = i/8;
   assert(cn < GetBitvecSizeInBytes());
 
   const int off = i - cn*8;
 
   m_buff_written[cn] |= cfiBIT(off);
 
   if (--m_missingBlocks == 0)
      m_complete = true;
}
 
inline void Info::SetBitPrefetch(int i)
{
   if (!m_buff_prefetch) return;
      
   const int cn = i/8;
   assert(cn < GetBitvecSizeInBytes());
 
   const int off = i - cn*8;
   m_buff_prefetch[cn] |= cfiBIT(off);
}
 
inline bool Info::TestBitPrefetch(int i) const
{
   if (!m_buff_prefetch) return false;
 
   const int cn = i/8;
   assert(cn < GetBitvecSizeInBytes());
 
   const int off = i - cn*8;
   return (m_buff_prefetch[cn] & cfiBIT(off)) != 0;
}
 
inline void Info::SetBitSynced(int i)
{
   const int cn = i/8;
   assert(cn < GetBitvecSizeInBytes());
 
   const int off = i - cn*8;
   m_buff_synced[cn] |= cfiBIT(off);
}
 
//------------------------------------------------------------------------------
 
inline int Info::GetNDownloadedBlocks() const
{
   int cntd = 0;
   for (int i = 0; i < m_bitvecSizeInBits; ++i)
      if (TestBitWritten(i)) cntd++;
 
   return cntd;
}
 
inline long long Info::GetNDownloadedBytes() const
{
   return m_store.m_buffer_size * GetNDownloadedBlocks();
}
 
inline int Info::GetLastDownloadedBlock() const
{
   for (int i = m_bitvecSizeInBits - 1; i >= 0; --i)
      if (TestBitWritten(i)) return i;
 
   return -1;
}
 
inline long long Info::GetExpectedDataFileSize() const
{
   int last_block = GetLastDownloadedBlock();
   if (last_block == m_bitvecSizeInBits - 1)
      return m_store.m_file_size;
   else
      return (last_block + 1) * m_store.m_buffer_size;
}
 
inline int Info::GetBitvecSizeInBytes() const
{
   if (m_bitvecSizeInBits)
      return ((m_bitvecSizeInBits - 1)/8 + 1);
   else
      return 0;
}
 
inline int Info::GetNBlocks() const
{
   return m_bitvecSizeInBits;
}
 
inline long long Info::GetFileSize() const
{
   return m_store.m_file_size;
}
 
inline bool Info::IsComplete() const
{
   return m_complete;
}
 
inline int Info::CountBlocksNotWrittenInRng(int firstIdx, int lastIdx) const
{
   // TODO rewrite to use full byte comparisons outside of edges ?
   // Also, it seems to be always called with firstIdx = 0, lastIdx = m_bitvecSizeInBits.
   int cnt = 0;
   for (int i = firstIdx; i < lastIdx; ++i)
      if (! TestBitWritten(i)) ++cnt;
 
   return cnt;
}
 
inline void Info::UpdateDownloadCompleteStatus()
{
   m_missingBlocks = CountBlocksNotWrittenInRng(0, m_bitvecSizeInBits);
   m_complete      = (m_missingBlocks == 0);
}
 
inline long long Info::GetBufferSize() const
{
   return m_store.m_buffer_size;
}
 
}
#endif