XRootD
Loading...
Searching...
No Matches
XrdCl::ZipArchive Class Reference

#include <XrdClZipArchive.hh>

+ Collaboration diagram for XrdCl::ZipArchive:

Public Member Functions

 ZipArchive (bool enablePlugIns=true)
 Constructor.
 
virtual ~ZipArchive ()
 Destructor.
 
XRootDStatus AppendFile (const std::string &fn, uint32_t crc32, uint32_t size, const void *buffer, ResponseHandler *handler, uint16_t timeout=0)
 
XRootDStatus CloseArchive (ResponseHandler *handler, uint16_t timeout=0)
 Create the central directory at the end of ZIP archive and close it.
 
XRootDStatus CloseFile ()
 
XRootDStatus GetCRC32 (const std::string &fn, uint32_t &cksum)
 
FileGetFile ()
 Get the underlying File object.
 
XRootDStatus GetOffset (const std::string &fn, uint64_t &offset)
 
bool GetProperty (const std::string &name, std::string &value)
 Get property on the underlying File object.
 
bool IsOpen ()
 
bool IsSecure ()
 Check if the underlying file is using an encrypted connection.
 
XRootDStatus List (DirectoryList *&list)
 
XRootDStatus OpenArchive (const std::string &url, OpenFlags::Flags flags, ResponseHandler *handler, uint16_t timeout=0)
 
XRootDStatus OpenFile (const std::string &fn, OpenFlags::Flags flags=OpenFlags::None, uint64_t size=0, uint32_t crc32=0)
 
XRootDStatus PgRead (uint64_t offset, uint32_t size, void *buffer, ResponseHandler *handler, uint16_t timeout=0)
 
XRootDStatus PgReadFrom (const std::string &fn, uint64_t offset, uint32_t size, void *buffer, ResponseHandler *handler, uint16_t timeout=0)
 
XRootDStatus Read (uint64_t offset, uint32_t size, void *buffer, ResponseHandler *handler, uint16_t timeout=0)
 
XRootDStatus ReadFrom (const std::string &fn, uint64_t offset, uint32_t size, void *buffer, ResponseHandler *handler, uint16_t timeout=0)
 
bool SetProperty (const std::string &name, const std::string &value)
 Set property on the underlying File object.
 
XRootDStatus Stat (const std::string &fn, StatInfo *&info)
 
XRootDStatus Stat (StatInfo *&info)
 
XRootDStatus UpdateMetadata (uint32_t crc32)
 
XRootDStatus Write (uint32_t size, const void *buffer, ResponseHandler *handler, uint16_t timeout=0)
 

Friends

class ::MicroTest
 
class ::XrdEcTests
 
template<typename RSP >
XRootDStatus ReadFromImpl (ZipArchive &, const std::string &, uint64_t, uint32_t, void *, ResponseHandler *, uint16_t)
 
template<bool >
class XrdEc::OpenOnlyImpl
 
class XrdEc::Reader
 
class XrdEc::StrmWriter
 

Detailed Description

Definition at line 60 of file XrdClZipArchive.hh.

Constructor & Destructor Documentation

◆ ZipArchive()

XrdCl::ZipArchive::ZipArchive ( bool  enablePlugIns = true)

Constructor.

Definition at line 208 of file XrdClZipArchive.cc.

208 : archive( enablePlugIns ),
209 archsize( 0 ),
210 cdexists( false ),
211 updated( false ),
212 cdoff( 0 ),
213 orgcdsz( 0 ),
214 orgcdcnt( 0 ),
215 openstage( None ),
216 ckpinit( false )
217 {
218 }

◆ ~ZipArchive()

XrdCl::ZipArchive::~ZipArchive ( )
virtual

Destructor.

Definition at line 223 of file XrdClZipArchive.cc.

224 {
225 }

Member Function Documentation

◆ AppendFile()

XRootDStatus XrdCl::ZipArchive::AppendFile ( const std::string &  fn,
uint32_t  crc32,
uint32_t  size,
const void *  buffer,
ResponseHandler handler,
uint16_t  timeout = 0 
)

Create a new file in the ZIP archive and append the data

Parameters
fn: the name of the new file to be created
crc32: the crc32 of the file
size: the size of the file
buffer: the buffer with the data
handler: user callback
timeout: operation timeout
Returns
: the status of the operation

Definition at line 877 of file XrdClZipArchive.cc.

883 {
884 Log *log = DefaultEnv::GetLog();
885 auto itr = cdmap.find( fn );
886 // check if the file already exists in the archive
887 if( itr != cdmap.end() )
888 {
889 log->Dump( ZipMsg, "[%p] Open failed: file exists %s, cannot append.",
890 this, fn.c_str() );
891 return XRootDStatus( stError, errInvalidOp );
892 }
893
894 log->Dump( ZipMsg, "[%p] Appending file: %s.", this, fn.c_str() );
895 //-------------------------------------------------------------------------
896 // Create Local File Header record
897 //-------------------------------------------------------------------------
898 lfh.reset( new LFH( fn, crc32, size, time( 0 ) ) );
899 //-------------------------------------------------------------------------
900 // And write it all
901 //-------------------------------------------------------------------------
902 return WriteImpl( size, buffer, handler, timeout );
903 }
static Log * GetLog()
Get default log.
const uint16_t stError
An error occurred that could potentially be retried.
const uint64_t ZipMsg
const uint16_t errInvalidOp
XrdSysError Log
Definition XrdConfig.cc:112

References XrdCl::Log::Dump(), XrdCl::errInvalidOp, XrdCl::DefaultEnv::GetLog(), XrdCl::stError, and XrdCl::ZipMsg.

+ Here is the call graph for this function:

◆ CloseArchive()

XRootDStatus XrdCl::ZipArchive::CloseArchive ( ResponseHandler handler,
uint16_t  timeout = 0 
)

Create the central directory at the end of ZIP archive and close it.

Parameters
handler: user callback
timeout: operation timeout
Returns
: the status of the operation

Definition at line 604 of file XrdClZipArchive.cc.

606 {
607 Log *log = DefaultEnv::GetLog();
608
609 //-------------------------------------------------------------------------
610 // If the file was updated, we need to write the Central Directory before
611 // closing the file.
612 //-------------------------------------------------------------------------
613 if( updated )
614 {
615 ChunkList chunks;
616 std::vector<std::shared_ptr<buffer_t>> wrtbufs;
617 for( auto &p : newfiles )
618 {
619 NewFile &nf = p.second;
620 if( !nf.overwrt ) continue;
621 uint32_t lfhlen = lfh->lfhSize;
622 auto lfhbuf = std::make_shared<buffer_t>();
623 lfhbuf->reserve( lfhlen );
624 nf.lfh->Serialize( *lfhbuf );
625 chunks.emplace_back( nf.offset, lfhbuf->size(), lfhbuf->data() );
626 wrtbufs.emplace_back( std::move( lfhbuf ) );
627 }
628
629 auto wrtbuff = std::make_shared<buffer_t>( GetCD() );
630 Pipeline p = XrdCl::Write( archive, cdoff,
631 wrtbuff->size(),
632 wrtbuff->data() );
633 wrtbufs.emplace_back( std::move( wrtbuff ) );
634
635 std::vector<ChunkList> listsvec;
636 XrdCl::Utils::SplitChunks( listsvec, chunks, 262144, 1024 );
637
638 for(auto itr = listsvec.rbegin(); itr != listsvec.rend(); ++itr)
639 {
640 p = XrdCl::VectorWrite( archive, *itr ) | p;
641 }
642 if( ckpinit )
643 p |= XrdCl::Checkpoint( archive, ChkPtCode::COMMIT );
644 p |= Close( archive ) >>
645 [=]( XRootDStatus &st )
646 {
647 if( st.IsOK() ) Clear();
648 else openstage = Error;
649 }
650 | XrdCl::Final( [=]( const XRootDStatus &st ) mutable
651 {
652 if( st.IsOK() )
653 log->Dump( ZipMsg, "[%p] Successfully closed ZIP archive "
654 "(CD written).", this );
655 else
656 log->Error( ZipMsg, "[%p] Failed to close ZIP archive: %s",
657 this, st.ToString().c_str() );
658 wrtbufs.clear();
659 if( handler ) handler->HandleResponse( make_status( st ), nullptr );
660 } );
661
662 Async( std::move( p ), timeout );
663 return XRootDStatus();
664 }
665
666 //-------------------------------------------------------------------------
667 // Otherwise, just close the ZIP archive
668 //-------------------------------------------------------------------------
669 Pipeline p = Close( archive ) >>
670 [=]( XRootDStatus &st )
671 {
672 if( st.IsOK() )
673 {
674 Clear();
675 log->Dump( ZipMsg, "[%p] Successfully closed "
676 "ZIP archive.", this );
677 }
678 else
679 {
680 openstage = Error;
681 log->Error( ZipMsg, "[%p] Failed to close ZIP archive:"
682 " %s", this, st.ToString().c_str() );
683 }
684 if( handler )
685 handler->HandleResponse( make_status( st ), nullptr );
686 };
687 Async( std::move( p ), timeout );
688 return XRootDStatus();
689 }
static void SplitChunks(std::vector< ChunkList > &listsvec, const ChunkList &chunks, const uint32_t maxcs, const size_t maxc)
Split chunks in a ChunkList into one or more ChunkLists.
VectorWriteImpl< false > VectorWrite(Ctx< File > file, Arg< ChunkList > chunks, uint16_t timeout=0)
Factory for creating VectorWriteImpl objects.
CheckpointImpl< false > Checkpoint(Ctx< File > file, Arg< ChkPtCode > code, uint16_t timeout=0)
Factory for creating ReadImpl objects.
CloseImpl< false > Close(Ctx< File > file, uint16_t timeout=0)
Factory for creating CloseImpl objects.
WriteImpl< false > Write(Ctx< File > file, Arg< uint64_t > offset, Arg< uint32_t > size, Arg< const void * > buffer, uint16_t timeout=0)
Factory for creating WriteImpl objects.
std::vector< ChunkInfo > ChunkList
List of chunks.
FinalOperation Final
std::future< XRootDStatus > Async(Pipeline pipeline, uint16_t timeout=0)

References XrdCl::Async(), XrdCl::Checkpoint(), XrdCl::Close(), XrdCl::COMMIT, XrdCl::Log::Dump(), XrdCl::Log::Error(), XrdCl::DefaultEnv::GetLog(), XrdCl::ResponseHandler::HandleResponse(), XrdCl::Status::IsOK(), XrdCl::Utils::SplitChunks(), XrdCl::Status::ToString(), XrdCl::VectorWrite(), XrdCl::Write(), and XrdCl::ZipMsg.

+ Here is the call graph for this function:

◆ CloseFile()

XRootDStatus XrdCl::ZipArchive::CloseFile ( )
inline

Close an open file within the ZIP archive

Returns
: the status of the operation

Definition at line 339 of file XrdClZipArchive.hh.

340 {
341 if( openstage != Done || openfn.empty() )
342 return XRootDStatus( stError, errInvalidOp,
343 0, "Archive not opened." );
344 openfn.clear();
345 lfh.reset();
346 return XRootDStatus();
347 }

References XrdCl::errInvalidOp, and XrdCl::stError.

◆ GetCRC32()

XRootDStatus XrdCl::ZipArchive::GetCRC32 ( const std::string &  fn,
uint32_t &  cksum 
)
inline

Get crc32 for a given file

Parameters
fn: file name
cksum: output parameter
Returns
: the status of the operation

Definition at line 278 of file XrdClZipArchive.hh.

279 { // make sure archive has been opened and CD has been parsed
280 if( openstage != Done )
281 return XRootDStatus( stError, errInvalidOp );
282 // make sure the file is part of the archive
283 auto cditr = cdmap.find( fn );
284 if( cditr == cdmap.end() )
285 return XRootDStatus( stError, errNotFound );
286 cksum = cdvec[cditr->second]->ZCRC32;
287 return XRootDStatus();
288 }
const uint16_t errNotFound

References XrdCl::errInvalidOp, XrdCl::errNotFound, and XrdCl::stError.

◆ GetFile()

File & XrdCl::ZipArchive::GetFile ( )
inline

Get the underlying File object.

Definition at line 390 of file XrdClZipArchive.hh.

391 {
392 return archive;
393 }

◆ GetOffset()

XRootDStatus XrdCl::ZipArchive::GetOffset ( const std::string &  fn,
uint64_t &  offset 
)
inline

Definition at line 290 of file XrdClZipArchive.hh.

290 {
291 if( openstage != XrdCl::ZipArchive::Done || !archive.IsOpen() )
293
294 auto cditr = cdmap.find( fn );
295 if( cditr == cdmap.end() )
297 XrdCl::errNotFound, "File not found." );
298
299 XrdCl::CDFH *cdfh = cdvec[cditr->second].get();
300
301 // check if the file is compressed, for now we only support uncompressed and inflate/deflate compression
302 if( cdfh->compressionMethod != 0 && cdfh->compressionMethod != Z_DEFLATED )
304 0, "The compression algorithm is not supported!" );
305
306 // Now the problem is that at the beginning of our
307 // file there is the Local-file-header, which size
308 // is not known because of the variable size 'extra'
309 // field, so we need to know the offset of the next
310 // record and shift it by the file size.
311 // The next record is either the next LFH (next file)
312 // or the start of the Central-directory.
313 uint64_t cdOffset = zip64eocd ? zip64eocd->cdOffset : eocd->cdOffset;
314 uint64_t nextRecordOffset = ( cditr->second + 1 < cdvec.size() ) ?
315 XrdCl::CDFH::GetOffset( *cdvec[cditr->second + 1] ) : cdOffset;
316 uint64_t filesize = cdfh->compressedSize;
317 if( filesize == std::numeric_limits<uint32_t>::max() && cdfh->extra )
318 filesize = cdfh->extra->compressedSize;
319 uint16_t descsize = cdfh->HasDataDescriptor() ?
320 XrdCl::DataDescriptor::GetSize( cdfh->IsZIP64() ) : 0;
321 offset = nextRecordOffset - filesize - descsize;
322 return XrdCl::XRootDStatus();
323 }
bool IsOpen() const
Check if the file is open.
Definition XrdClFile.cc:846
const uint16_t errNotSupported
std::unique_ptr< Extra > extra
uint16_t compressionMethod
bool HasDataDescriptor()
uint32_t compressedSize
bool IsZIP64() const

References XrdZip::CDFH::compressedSize, XrdZip::CDFH::compressionMethod, XrdCl::errInvalidOp, XrdCl::errNotFound, XrdCl::errNotSupported, XrdZip::CDFH::extra, XrdZip::CDFH::HasDataDescriptor(), XrdCl::File::IsOpen(), XrdZip::CDFH::IsZIP64(), and XrdCl::stError.

+ Here is the call graph for this function:

◆ GetProperty()

bool XrdCl::ZipArchive::GetProperty ( const std::string &  name,
std::string &  value 
)
inline

Get property on the underlying File object.

Definition at line 382 of file XrdClZipArchive.hh.

383 {
384 return archive.GetProperty( name, value );
385 }
bool GetProperty(const std::string &name, std::string &value) const
Definition XrdClFile.cc:878

References XrdCl::File::GetProperty().

+ Here is the call graph for this function:

◆ IsOpen()

bool XrdCl::ZipArchive::IsOpen ( )
inline
Returns
: true if ZIP archive has been successfully opened

Definition at line 358 of file XrdClZipArchive.hh.

359 {
360 return openstage == Done;
361 }

Referenced by XrdCl::ZipListHandler::HandleResponse().

+ Here is the caller graph for this function:

◆ IsSecure()

bool XrdCl::ZipArchive::IsSecure ( )
inline

Check if the underlying file is using an encrypted connection.

Definition at line 366 of file XrdClZipArchive.hh.

367 {
368 return archive.IsSecure();
369 }
bool IsSecure() const
Check if the file is using an encrypted connection.
Definition XrdClFile.cc:857

References XrdCl::File::IsSecure().

+ Here is the call graph for this function:

◆ List()

XRootDStatus XrdCl::ZipArchive::List ( DirectoryList *&  list)

List files in the ZIP archive

Returns
: the status of the operation

Definition at line 720 of file XrdClZipArchive.cc.

721 {
722 if( openstage != Done )
723 return XRootDStatus( stError, errInvalidOp,
724 0, "Archive not opened." );
725
726 std::string value;
727 archive.GetProperty( "LastURL", value );
728 URL url( value );
729
730 StatInfo *infoptr = 0;
731 XRootDStatus st = archive.Stat( false, infoptr );
732 std::unique_ptr<StatInfo> info( infoptr );
733
734 list = new DirectoryList();
735 list->SetParentName( url.GetPath() );
736
737 auto itr = cdvec.begin();
738 for( ; itr != cdvec.end() ; ++itr )
739 {
740 CDFH *cdfh = itr->get();
741 uint64_t uncompressedSize = cdfh->uncompressedSize;
742 if( uncompressedSize == std::numeric_limits<uint32_t>::max() && cdfh->extra )
743 uncompressedSize = cdfh->extra->uncompressedSize;
744 StatInfo *entry_info = make_stat( *info, uncompressedSize );
745 DirectoryList::ListEntry *entry =
746 new DirectoryList::ListEntry( url.GetHostId(), cdfh->filename, entry_info );
747 list->Add( entry );
748 }
749
750 return XRootDStatus();
751 }
XRootDStatus Stat(bool force, ResponseHandler *handler, uint16_t timeout=0) XRD_WARN_UNUSED_RESULT
Definition XrdClFile.cc:177

References XrdCl::DirectoryList::Add(), XrdCl::errInvalidOp, XrdZip::CDFH::extra, XrdZip::CDFH::filename, XrdCl::URL::GetHostId(), XrdCl::URL::GetPath(), XrdCl::File::GetProperty(), XrdCl::DirectoryList::SetParentName(), XrdCl::File::Stat(), XrdCl::stError, and XrdZip::CDFH::uncompressedSize.

Referenced by XrdCl::ZipListHandler::HandleResponse().

+ Here is the call graph for this function:
+ Here is the caller graph for this function:

◆ OpenArchive()

XRootDStatus XrdCl::ZipArchive::OpenArchive ( const std::string &  url,
OpenFlags::Flags  flags,
ResponseHandler handler,
uint16_t  timeout = 0 
)

Open ZIP Archive (and parse the Central Directory)

Parameters
url: the URL of the ZIP archive
flags: open flags to be used when openning the file
handler: user callback
timeout: operation timeout
Returns
: the status of the operation

Definition at line 266 of file XrdClZipArchive.cc.

270 {
271 Log *log = DefaultEnv::GetLog();
272 Fwd<uint32_t> rdsize; // number of bytes to be read
273 Fwd<uint64_t> rdoff; // offset for the read request
274 Fwd<void*> rdbuff; // buffer for data to be read
275 uint32_t maxrdsz = EOCD::maxCommentLength + EOCD::eocdBaseSize +
277
278 Pipeline open_archive = // open the archive
279 XrdCl::Open( archive, url, flags ) >>
280 [=]( XRootDStatus &status, StatInfo &info ) mutable
281 {
282 // check the status is OK
283 if( !status.IsOK() ) return;
284
285 archsize = info.GetSize();
286 // if it is an empty file (possibly a new file) there's nothing more to do
287 if( archsize == 0 )
288 {
289 cdexists = false;
290 openstage = Done;
291 log->Dump( ZipMsg, "[%p] Opened a ZIP archive (file empty).", this );
293 }
294 // prepare the arguments for the subsequent read
295 rdsize = ( archsize <= maxrdsz ? archsize : maxrdsz );
296 rdoff = archsize - *rdsize;
297 buffer.reset( new char[*rdsize] );
298 rdbuff = buffer.get();
299 openstage = HaveEocdBlk;
300 log->Dump( ZipMsg, "[%p] Opened a ZIP archive, reading "
301 "Central Directory at offset: %llu.", this, (unsigned long long) *rdoff );
302 }
303 // read the Central Directory (in several stages if necessary)
304 | XrdCl::Read( archive, rdoff, rdsize, rdbuff ) >>
305 [=]( XRootDStatus &status, ChunkInfo &chunk ) mutable
306 {
307 // check the status is OK
308 if( !status.IsOK() ) return;
309
310 const char *buff = reinterpret_cast<char*>( chunk.buffer );
311 while( true )
312 {
313 switch( openstage )
314 {
315 case HaveEocdBlk:
316 {
317 // Parse the EOCD record
318 const char *eocdBlock = EOCD::Find( buff, chunk.length );
319 if( !eocdBlock )
320 {
321 XRootDStatus error( stError, errDataError, 0,
322 "End-of-central-directory signature not found." );
323 Pipeline::Stop( error );
324 }
325 try{
326 eocd.reset( new EOCD( eocdBlock, chunk.length - uint32_t(eocdBlock - buff) ) );
327 log->Dump( ZipMsg, "[%p] EOCD record parsed: %s", this,
328 eocd->ToString().c_str() );
329 if(eocd->cdOffset > archsize || eocd->cdOffset + eocd->cdSize > archsize)
330 throw bad_data();
331 }
332 catch(const bad_data &ex){
333 XRootDStatus error( stError, errDataError, 0,
334 "End-of-central-directory signature corrupted." );
335 Pipeline::Stop( error );
336 }
337 // Do we have the whole archive?
338 if( chunk.length == archsize )
339 {
340 // If we managed to download the whole archive we don't need to
341 // worry about zip64, it is so small that standard EOCD will do
342 cdoff = eocd->cdOffset;
343 orgcdsz = eocd->cdSize;
344 orgcdcnt = eocd->nbCdRec;
345 buff = buff + cdoff;
346 openstage = HaveCdRecords;
347 continue;
348 }
349
350 // Let's see if it is ZIP64 (if yes, the EOCD will be preceded with ZIP64 EOCD locator)
351 const char *zip64EocdlBlock = eocdBlock - ZIP64_EOCDL::zip64EocdlSize;
352 // make sure there is enough data to assume there's a ZIP64 EOCD locator
353 if( zip64EocdlBlock > buffer.get() )
354 {
355 uint32_t signature = to<uint32_t>( zip64EocdlBlock );
356 if( signature == ZIP64_EOCDL::zip64EocdlSign )
357 {
358 buff = zip64EocdlBlock;
359 openstage = HaveZip64EocdlBlk;
360 continue;
361 }
362 }
363
364 // It's not ZIP64, we already know where the CD records are
365 // we need to read more data
366 cdoff = eocd->cdOffset;
367 orgcdsz = eocd->cdSize;
368 orgcdcnt = eocd->nbCdRec;
369 rdoff = eocd->cdOffset;
370 rdsize = eocd->cdSize;
371 buffer.reset( new char[*rdsize] );
372 rdbuff = buffer.get();
373 openstage = HaveCdRecords;
374 log->Dump( ZipMsg, "[%p] Reading additional data at offset: %llu.",
375 this, (unsigned long long) *rdoff );
376 Pipeline::Repeat(); break; // the break is really not needed ...
377 }
378
379 case HaveZip64EocdlBlk:
380 {
381 std::unique_ptr<ZIP64_EOCDL> eocdl( new ZIP64_EOCDL( buff ) );
382 log->Dump( ZipMsg, "[%p] EOCDL record parsed: %s",
383 this, eocdl->ToString().c_str() );
384
385 if( chunk.offset > eocdl->zip64EocdOffset )
386 {
387 // we need to read more data, adjust the read arguments
388 rdsize = archsize - eocdl->zip64EocdOffset;
389 rdoff = eocdl->zip64EocdOffset;
390 buffer.reset( new char[*rdsize] );
391 rdbuff = buffer.get();
392 openstage = HaveZip64EocdBlk;
393 log->Dump( ZipMsg, "[%p] Reading additional data at offset: %llu.",
394 this, (unsigned long long) *rdoff );
396 }
397
398 buff = buffer.get() + ( eocdl->zip64EocdOffset - chunk.offset );
399 openstage = HaveZip64EocdBlk;
400 continue;
401 }
402
403 case HaveZip64EocdBlk:
404 {
405 uint32_t signature = to<uint32_t>( buff );
406 if( signature != ZIP64_EOCD::zip64EocdSign )
407 {
408 XRootDStatus error( stError, errDataError, 0,
409 "ZIP64 End-of-central-directory signature not found." );
410 Pipeline::Stop( error );
411 }
412 zip64eocd.reset( new ZIP64_EOCD( buff ) );
413 log->Dump( ZipMsg, "[%p] ZIP64EOCD record parsed: %s",
414 this, zip64eocd->ToString().c_str() );
415
416 // now we can read the CD records, adjust the read arguments
417 cdoff = zip64eocd->cdOffset;
418 orgcdsz = zip64eocd->cdSize;
419 orgcdcnt = zip64eocd->nbCdRec;
420 rdoff = zip64eocd->cdOffset;
421 rdsize = zip64eocd->cdSize;
422 buffer.reset( new char[*rdsize] );
423 rdbuff = buffer.get();
424 openstage = HaveCdRecords;
425 log->Dump( ZipMsg, "[%p] Reading additional data at offset: %llu.",
426 this, (unsigned long long) *rdoff );
427 Pipeline::Repeat(); break; // the break is really not needed ...
428 }
429
430 case HaveCdRecords:
431 {
432 // make a copy of the original CDFH records
433 orgcdbuf.reserve( orgcdsz );
434 std::copy( buff, buff + orgcdsz, std::back_inserter( orgcdbuf ) );
435 try
436 {
437 if( zip64eocd )
438 std::tie( cdvec, cdmap ) = CDFH::Parse( buff, zip64eocd->cdSize, zip64eocd->nbCdRec );
439 else
440 std::tie( cdvec, cdmap ) = CDFH::Parse( buff, eocd->cdSize, eocd->nbCdRec );
441 log->Dump( ZipMsg, "[%p] CD records parsed.", this );
442 uint64_t sumCompSize = 0;
443 for (auto it = cdvec.begin(); it != cdvec.end(); it++)
444 {
445 sumCompSize += (*it)->IsZIP64() ? (*it)->extra->compressedSize : (*it)->compressedSize;
446 if ((*it)->offset > archsize || (*it)->offset + (*it)->compressedSize > archsize)
447 throw bad_data();
448 }
449 if (sumCompSize > archsize)
450 throw bad_data();
451 }
452 catch( const bad_data &ex )
453 {
454 XRootDStatus error( stError, errDataError, 0,
455 "ZIP Central Directory corrupted." );
456 Pipeline::Stop( error );
457 }
458 if( chunk.length != archsize ) buffer.reset();
459 openstage = Done;
460 cdexists = true;
461 break;
462 }
463
464 default: Pipeline::Stop( XRootDStatus( stError, errInvalidOp ) );
465 }
466
467 break;
468 }
469 }
470 | XrdCl::Final( [=]( const XRootDStatus &status )
471 { // finalize the pipeline by calling the user callback
472 if( status.IsOK() )
473 log->Debug( ZipMsg, "[%p] Opened a ZIP archive (%s): %s",
474 this, url.c_str(), status.ToString().c_str() );
475 else
476 log->Error( ZipMsg, "[%p] Failed to open a ZIP archive (%s): %s",
477 this, url.c_str(), status.ToString().c_str() );
478 if( handler )
479 handler->HandleResponse( make_status( status ), nullptr );
480 } );
481
482 Async( std::move( open_archive ), timeout );
483 return XRootDStatus();
484 }
static void Repeat()
Repeat current operation.
static void Stop(const XRootDStatus &status=XrdCl::XRootDStatus())
ReadImpl< false > Read(Ctx< File > file, Arg< uint64_t > offset, Arg< uint32_t > size, Arg< void * > buffer, uint16_t timeout=0)
Factory for creating ReadImpl objects.
const uint16_t errDataError
data is corrupted
OpenImpl< false > Open(Ctx< File > file, Arg< std::string > url, Arg< OpenFlags::Flags > flags, Arg< Access::Mode > mode=Access::None, uint16_t timeout=0)
Factory for creating ReadImpl objects.
static std::tuple< cdvec_t, cdmap_t > Parse(const char *buffer, uint32_t bufferSize, uint16_t nbCdRecords)
Definition XrdZipCDFH.hh:75
static const uint16_t eocdBaseSize
static const uint16_t maxCommentLength
static const char * Find(const char *buffer, uint64_t size)
Definition XrdZipEOCD.hh:41
static const uint32_t zip64EocdlSign
static const uint16_t zip64EocdlSize
static const uint32_t zip64EocdSign

References XrdCl::Async(), XrdCl::Log::Debug(), XrdCl::Log::Dump(), XrdZip::EOCD::eocdBaseSize, XrdCl::errDataError, XrdCl::errInvalidOp, XrdCl::Log::Error(), XrdZip::EOCD::Find(), XrdCl::DefaultEnv::GetLog(), XrdCl::ResponseHandler::HandleResponse(), XrdCl::Status::IsOK(), XrdZip::EOCD::maxCommentLength, XrdCl::Open(), XrdZip::CDFH::Parse(), XrdCl::Read(), XrdCl::Pipeline::Repeat(), XrdCl::stError, XrdCl::Pipeline::Stop(), XrdCl::Status::ToString(), XrdZip::ZIP64_EOCDL::zip64EocdlSign, XrdZip::ZIP64_EOCDL::zip64EocdlSize, XrdZip::ZIP64_EOCD::zip64EocdSign, and XrdCl::ZipMsg.

+ Here is the call graph for this function:

◆ OpenFile()

XRootDStatus XrdCl::ZipArchive::OpenFile ( const std::string &  fn,
OpenFlags::Flags  flags = OpenFlags::None,
uint64_t  size = 0,
uint32_t  crc32 = 0 
)

Open a file within the ZIP Archive

Parameters
fn: file name to be opened
flags: open flags (either 'Read' or 'New | Write')
size: file size (to be included in the LFH)
crc32: file crc32 (to be included in the LFH)
Returns
: the status of the operation

Definition at line 489 of file XrdClZipArchive.cc.

493 {
494 if( !openfn.empty() || openstage != Done || !archive.IsOpen() )
495 return XRootDStatus( stError, errInvalidOp );
496
497 Log *log = DefaultEnv::GetLog();
498 auto itr = cdmap.find( fn );
499 if( itr == cdmap.end() )
500 {
501 // the file does not exist in the archive so it only makes sense
502 // if our user is opening for append
503 if( flags & OpenFlags::New )
504 {
505 openfn = fn;
506 lfh.reset( new LFH( fn, crc32, size, time( 0 ) ) );
507 log->Dump( ZipMsg, "[%p] File %s opened for append.",
508 this, fn.c_str() );
509 return XRootDStatus();
510 }
511 log->Dump( ZipMsg, "[%p] Open failed: %s not in the ZIP archive.",
512 this, fn.c_str() );
513 return XRootDStatus( stError, errNotFound );
514 }
515
516 // the file name exist in the archive but our user wants to append
517 // a file with the same name
518 if( flags & OpenFlags::New )
519 {
520 log->Dump( ZipMsg, "[%p] Open failed: file exists %s, cannot append.",
521 this, fn.c_str() );
522
523 return XRootDStatus( stError, errInvalidOp, EEXIST, "The file already exists in the ZIP archive." );
524 }
525
526 openfn = fn;
527 log->Dump( ZipMsg, "[%p] File %s opened for reading.",
528 this, fn.c_str() );
529 return XRootDStatus();
530 }

References XrdCl::Log::Dump(), XrdCl::errInvalidOp, XrdCl::errNotFound, XrdCl::DefaultEnv::GetLog(), XrdCl::File::IsOpen(), XrdCl::OpenFlags::New, XrdCl::stError, and XrdCl::ZipMsg.

+ Here is the call graph for this function:

◆ PgRead()

XRootDStatus XrdCl::ZipArchive::PgRead ( uint64_t  offset,
uint32_t  size,
void *  buffer,
ResponseHandler handler,
uint16_t  timeout = 0 
)
inline

PgRead data from an open file

Parameters
offset: offset within the file to read at
size: number of bytes to be read
buffer: the buffer for the data
handler: user callback
timeout: operation timeout
Returns
: the status of the operation

Definition at line 143 of file XrdClZipArchive.hh.

148 {
149 if( openfn.empty() ) return XRootDStatus( stError, errInvalidOp );
150 return PgReadFrom( openfn, offset, size, buffer, handler, timeout );
151 }
XRootDStatus PgReadFrom(const std::string &fn, uint64_t offset, uint32_t size, void *buffer, ResponseHandler *handler, uint16_t timeout=0)

References XrdCl::errInvalidOp, PgReadFrom(), and XrdCl::stError.

+ Here is the call graph for this function:

◆ PgReadFrom()

XRootDStatus XrdCl::ZipArchive::PgReadFrom ( const std::string &  fn,
uint64_t  offset,
uint32_t  size,
void *  buffer,
ResponseHandler handler,
uint16_t  timeout = 0 
)

PgRead data from a given file

Parameters
fn: the name of the file from which we are going to read
offset: offset within the file to read at
size: number of bytes to be read
buffer: the buffer for the data
handler: user callback
timeout: operation timeout
Returns
: the status of the operation

Definition at line 707 of file XrdClZipArchive.cc.

713 {
714 return ReadFromImpl<PageInfo>( *this, fn, offset, size, buffer, handler, timeout );
715 }

Referenced by PgRead().

+ Here is the caller graph for this function:

◆ Read()

XRootDStatus XrdCl::ZipArchive::Read ( uint64_t  offset,
uint32_t  size,
void *  buffer,
ResponseHandler handler,
uint16_t  timeout = 0 
)
inline

Read data from an open file

Parameters
offset: offset within the file to read at
size: number of bytes to be read
buffer: the buffer for the data
handler: user callback
timeout: operation timeout
Returns
: the status of the operation

Definition at line 122 of file XrdClZipArchive.hh.

127 {
128 if( openfn.empty() ) return XRootDStatus( stError, errInvalidOp );
129 return ReadFrom( openfn, offset, size, buffer, handler, timeout );
130 }
XRootDStatus ReadFrom(const std::string &fn, uint64_t offset, uint32_t size, void *buffer, ResponseHandler *handler, uint16_t timeout=0)

References XrdCl::errInvalidOp, ReadFrom(), and XrdCl::stError.

+ Here is the call graph for this function:

◆ ReadFrom()

XRootDStatus XrdCl::ZipArchive::ReadFrom ( const std::string &  fn,
uint64_t  offset,
uint32_t  size,
void *  buffer,
ResponseHandler handler,
uint16_t  timeout = 0 
)

Read data from a given file

Parameters
fn: the name of the file from which we are going to read
offset: offset within the file to read at
size: number of bytes to be read
buffer: the buffer for the data
handler: user callback
timeout: operation timeout
Returns
: the status of the operation

Definition at line 694 of file XrdClZipArchive.cc.

700 {
701 return ReadFromImpl<ChunkInfo>( *this, fn, offset, size, buffer, handler, timeout );
702 }

Referenced by Read().

+ Here is the caller graph for this function:

◆ SetProperty()

bool XrdCl::ZipArchive::SetProperty ( const std::string &  name,
const std::string &  value 
)
inline

Set property on the underlying File object.

Definition at line 374 of file XrdClZipArchive.hh.

375 {
376 return archive.SetProperty( name, value );
377 }
bool SetProperty(const std::string &name, const std::string &value)
Definition XrdClFile.cc:867

References XrdCl::File::SetProperty().

+ Here is the call graph for this function:

◆ Stat() [1/2]

XRootDStatus XrdCl::ZipArchive::Stat ( const std::string &  fn,
StatInfo *&  info 
)
inline

Get stat info for given file

Parameters
fn: the name of the file
info: output parameter
Returns
: the status of the operation

Definition at line 242 of file XrdClZipArchive.hh.

243 { // make sure archive has been opened and CD has been parsed
244 if( openstage != Done )
245 return XRootDStatus( stError, errInvalidOp );
246 // make sure the file is part of the archive
247 auto cditr = cdmap.find( fn );
248 if( cditr == cdmap.end() )
249 return XRootDStatus( stError, errNotFound );
250 // create the result
251 info = make_stat( fn );
252 if (info)
253 return XRootDStatus();
254 else // have difficult to access the openned archive.
255 return XRootDStatus( stError, errNotFound );
256 }

References XrdCl::errInvalidOp, XrdCl::errNotFound, and XrdCl::stError.

◆ Stat() [2/2]

XRootDStatus XrdCl::ZipArchive::Stat ( StatInfo *&  info)
inline

Get stat info for an open file

Parameters
info: output parameter
Returns
: the status of the operation

Definition at line 264 of file XrdClZipArchive.hh.

265 {
266 if( openfn.empty() )
267 return XRootDStatus( stError, errInvalidOp );
268 return Stat( openfn, info );
269 }
struct stat Stat
Definition XrdCks.cc:49

References XrdCl::errInvalidOp, Stat, and XrdCl::stError.

◆ UpdateMetadata()

XRootDStatus XrdCl::ZipArchive::UpdateMetadata ( uint32_t  crc32)

Update the metadata of the currently open file

Parameters
crc32: the crc32 checksum
Returns
: the status of the operation

Definition at line 849 of file XrdClZipArchive.cc.

850 {
851 if( openstage != Done || openfn.empty() )
852 return XRootDStatus( stError, errInvalidOp, 0, "Archive not opened." );
853
854 //---------------------------------------------------------------------
855 // Firstly, update the crc32 in the central directory
856 //---------------------------------------------------------------------
857 auto itr = cdmap.find( openfn );
858 if( itr == cdmap.end() )
859 return XRootDStatus( stError, errInvalidOp );
860 cdvec[itr->second]->ZCRC32 = crc32;
861
862 //---------------------------------------------------------------------
863 // Secondly, update the crc32 in the LFH and mark it as needing
864 // overwriting
865 //---------------------------------------------------------------------
866 auto itr2 = newfiles.find( openfn );
867 if( itr2 == newfiles.end() )
868 return XRootDStatus( stError, errInvalidOp );
869 itr2->second.lfh->ZCRC32 = crc32;
870
871 return XRootDStatus();
872 }

References XrdCl::errInvalidOp, and XrdCl::stError.

◆ Write()

XRootDStatus XrdCl::ZipArchive::Write ( uint32_t  size,
const void *  buffer,
ResponseHandler handler,
uint16_t  timeout = 0 
)
inline

Append data to a new file

Parameters
size: number of bytes to be appended
buffer: the buffer with the data to be appended
handler: user callback
timeout: operation timeout
Returns
: the status of the operation

Definition at line 198 of file XrdClZipArchive.hh.

202 {
203 if( openstage != Done || openfn.empty() )
204 return XRootDStatus( stError, errInvalidOp, 0, "Archive not opened." );
205
206 return WriteImpl( size, buffer, handler, timeout );
207 }

References XrdCl::errInvalidOp, and XrdCl::stError.

Friends And Related Symbol Documentation

◆ ::MicroTest

friend class ::MicroTest
friend

Definition at line 66 of file XrdClZipArchive.hh.

◆ ::XrdEcTests

friend class ::XrdEcTests
friend

Definition at line 67 of file XrdClZipArchive.hh.

◆ ReadFromImpl

template<typename RSP >
XRootDStatus ReadFromImpl ( ZipArchive me,
const std::string &  fn,
uint64_t  relativeOffset,
uint32_t  size,
void *  usrbuff,
ResponseHandler usrHandler,
uint16_t  timeout 
)
friend

Definition at line 44 of file XrdClZipArchive.cc.

51 {
52 if( me.openstage != ZipArchive::Done || !me.archive.IsOpen() )
53 return XRootDStatus( stError, errInvalidOp );
54
55 Log *log = DefaultEnv::GetLog();
56
57 auto cditr = me.cdmap.find( fn );
58 if( cditr == me.cdmap.end() )
59 return XRootDStatus( stError, errNotFound,
60 errNotFound, "File not found." );
61
62 CDFH *cdfh = me.cdvec[cditr->second].get();
63
64 // check if the file is compressed, for now we only support uncompressed and inflate/deflate compression
65 if( cdfh->compressionMethod != 0 && cdfh->compressionMethod != Z_DEFLATED )
66 return XRootDStatus( stError, errNotSupported,
67 0, "The compression algorithm is not supported!" );
68
69 // Now the problem is that at the beginning of our
70 // file there is the Local-file-header, which size
71 // is not known because of the variable size 'extra'
72 // field, so we need to know the offset of the next
73 // record and shift it by the file size.
74 // The next record is either the next LFH (next file)
75 // or the start of the Central-directory.
76 uint64_t cdOffset = me.zip64eocd ? me.zip64eocd->cdOffset : me.eocd->cdOffset;
77 uint64_t nextRecordOffset = ( cditr->second + 1 < me.cdvec.size() ) ?
78 CDFH::GetOffset( *me.cdvec[cditr->second + 1] ) : cdOffset;
79 uint64_t filesize = cdfh->compressedSize;
80 if( filesize == std::numeric_limits<uint32_t>::max() && cdfh->extra )
81 filesize = cdfh->extra->compressedSize;
82 uint16_t descsize = cdfh->HasDataDescriptor() ?
83 DataDescriptor::GetSize( cdfh->IsZIP64() ) : 0;
84 uint64_t fileoff = nextRecordOffset - filesize - descsize;
85 uint64_t offset = fileoff + relativeOffset;
86 uint64_t uncompressedSize = cdfh->uncompressedSize;
87 if( uncompressedSize == std::numeric_limits<uint32_t>::max() && cdfh->extra )
88 uncompressedSize = cdfh->extra->uncompressedSize;
89 uint64_t sizeTillEnd = relativeOffset > uncompressedSize ?
90 0 : uncompressedSize - relativeOffset;
91 if( size > sizeTillEnd ) size = sizeTillEnd;
92
93 // if it is a compressed file use ZIP cache to read from the file
94 if( cdfh->compressionMethod == Z_DEFLATED )
95 {
96 log->Dump( ZipMsg, "[%p] Reading compressed data.", &me );
97 // check if respective ZIP cache exists
98 bool empty = me.zipcache.find( fn ) == me.zipcache.end();
99 // if the entry does not exist, it will be created using
100 // default constructor
101 ZipCache &cache = me.zipcache[fn];
102
103 if( relativeOffset > uncompressedSize )
104 {
105 // we are reading past the end of file,
106 // we can serve the request right away!
107 RSP *r = new RSP( relativeOffset, 0, usrbuff );
108 AnyObject *rsp = new AnyObject();
109 rsp->Set( r );
110 usrHandler->HandleResponse( new XRootDStatus(), rsp );
111 return XRootDStatus();
112 }
113
114 uint32_t sizereq = size;
115 if( relativeOffset + size > uncompressedSize )
116 sizereq = uncompressedSize - relativeOffset;
117 cache.QueueReq( relativeOffset, sizereq, usrbuff, usrHandler );
118
119 // if we have the whole ZIP archive we can populate the cache
120 // straight away
121 if( empty && me.buffer)
122 {
123 auto begin = me.buffer.get() + fileoff;
124 auto end = begin + filesize ;
125 buffer_t buff( begin, end );
126 cache.QueueRsp( XRootDStatus(), 0, std::move( buff ) );
127 return XRootDStatus();
128 }
129
130 // if we don't have the data we need to issue a remote read
131 if( !me.buffer )
132 {
133 if( relativeOffset > filesize ) return XRootDStatus(); // there's nothing to do,
134 // we already have all the data locally
135 uint32_t rdsize = size;
136 // check if this is the last read (we reached the end of
137 // file from user perspective)
138 if( relativeOffset + size >= uncompressedSize )
139 {
140 // if yes, make sure we readout all the compressed data
141 // Note: In a patological case the compressed size may
142 // be greater than the uncompressed size
143 rdsize = filesize > relativeOffset ?
144 filesize - relativeOffset :
145 0;
146 }
147 // make sure we are not reading past the end of
148 // compressed data
149 if( relativeOffset + size > filesize )
150 rdsize = filesize - relativeOffset;
151
152
153 // now read the data ...
154 auto rdbuff = std::make_shared<ZipCache::buffer_t>( rdsize );
155 Pipeline p = XrdCl::RdWithRsp<RSP>( me.archive, offset, rdbuff->size(), rdbuff->data() ) >>
156 [relativeOffset, rdbuff, &cache, &me]( XRootDStatus &st, RSP &rsp )
157 {
158 Log *log = DefaultEnv::GetLog();
159 log->Dump( ZipMsg, "[%p] Read %u bytes of remote data at offset %llu.",
160 &me, rsp.GetLength(), (unsigned long long) rsp.GetOffset() );
161 cache.QueueRsp( st, relativeOffset, std::move( *rdbuff ) );
162 };
163 Async( std::move( p ), timeout );
164 }
165
166 return XRootDStatus();
167 }
168
169 // check if we have the whole file in our local buffer
170 if( me.buffer || size == 0 )
171 {
172 if( size )
173 {
174 memcpy( usrbuff, me.buffer.get() + offset, size );
175 log->Dump( ZipMsg, "[%p] Serving read from local cache.", &me );
176 }
177
178 if( usrHandler )
179 {
180 XRootDStatus *st = ZipArchive::make_status();
181 RSP *rsp = new RSP( relativeOffset, size, usrbuff );
182 ZipArchive::Schedule( usrHandler, st, rsp );
183 }
184 return XRootDStatus();
185 }
186
187 Pipeline p = XrdCl::RdWithRsp<RSP>( me.archive, offset, size, usrbuff ) >>
188 [=, &me]( XRootDStatus &st, RSP &r )
189 {
190 log->Dump( ZipMsg, "[%p] Read %u bytes of remote data at "
191 "offset %llu.", &me, r.GetLength(), (unsigned long long) r.GetOffset() );
192 if( usrHandler )
193 {
194 XRootDStatus *status = ZipArchive::make_status( st );
195 RSP *rsp = nullptr;
196 if( st.IsOK() )
197 rsp = new RSP( relativeOffset, r.GetLength(), r.GetBuffer() );
198 usrHandler->HandleResponse( status, ZipArchive::PkgRsp( rsp ) );
199 }
200 };
201 Async( std::move( p ), timeout );
202 return XRootDStatus();
203 }
std::vector< char > buffer_t
static uint64_t GetOffset(const CDFH &cdfh)
static uint8_t GetSize(bool zip64)

◆ XrdEc::OpenOnlyImpl

template<bool >
friend class XrdEc::OpenOnlyImpl
friend

Definition at line 65 of file XrdClZipArchive.hh.

◆ XrdEc::Reader

friend class XrdEc::Reader
friend

Definition at line 63 of file XrdClZipArchive.hh.

◆ XrdEc::StrmWriter

friend class XrdEc::StrmWriter
friend

Definition at line 62 of file XrdClZipArchive.hh.


The documentation for this class was generated from the following files: