[ tor-browser ].git.dasho

7zUpdate.cpp (65892B)
      1 // 7zUpdate.cpp
      2 
      3 #include "StdAfx.h"
      4 
      5 #include "../../../../C/CpuArch.h"
      6 
      7 #include "../../../Common/Wildcard.h"
      8 
      9 #include "../../Common/CreateCoder.h"
     10 #include "../../Common/LimitedStreams.h"
     11 #include "../../Common/ProgressUtils.h"
     12 
     13 #include "../../Compress/CopyCoder.h"
     14 
     15 #include "../Common/ItemNameUtils.h"
     16 
     17 #include "7zDecode.h"
     18 #include "7zEncode.h"
     19 #include "7zFolderInStream.h"
     20 #include "7zHandler.h"
     21 #include "7zOut.h"
     22 #include "7zUpdate.h"
     23 
     24 namespace NArchive {
     25 namespace N7z {
     26 
     27 
     28 #define k_X86 k_BCJ
     29 
     30 struct CFilterMode
     31 {
     32  UInt32 Id;
     33  UInt32 Delta;
     34 
     35  CFilterMode(): Id(0), Delta(0) {}
     36 
     37  void SetDelta()
     38  {
     39    if (Id == k_IA64)
     40      Delta = 16;
     41    else if (Id == k_ARM || Id == k_PPC || Id == k_SPARC)
     42      Delta = 4;
     43    else if (Id == k_ARMT)
     44      Delta = 2;
     45    else
     46      Delta = 0;
     47  }
     48 };
     49 
     50 
     51 /* ---------- PE ---------- */
     52 
     53 #define MZ_SIG 0x5A4D
     54 
     55 #define PE_SIG 0x00004550
     56 #define PE_OptHeader_Magic_32 0x10B
     57 #define PE_OptHeader_Magic_64 0x20B
     58 #define PE_SectHeaderSize 40
     59 #define PE_SECT_EXECUTE 0x20000000
     60 
     61 static int Parse_EXE(const Byte *buf, size_t size, CFilterMode *filterMode)
     62 {
     63  if (size < 512 || GetUi16(buf) != MZ_SIG)
     64    return 0;
     65 
     66  const Byte *p;
     67  UInt32 peOffset, optHeaderSize, filterId;
     68 
     69  peOffset = GetUi32(buf + 0x3C);
     70  if (peOffset >= 0x1000 || peOffset + 512 > size || (peOffset & 7) != 0)
     71    return 0;
     72  p = buf + peOffset;
     73  if (GetUi32(p) != PE_SIG)
     74    return 0;
     75  p += 4;
     76  
     77  switch (GetUi16(p))
     78  {
     79    case 0x014C:
     80    case 0x8664:  filterId = k_X86; break;
     81    
     82    /*
     83    IMAGE_FILE_MACHINE_ARM   0x01C0  // ARM LE
     84    IMAGE_FILE_MACHINE_THUMB 0x01C2  // ARM Thumb / Thumb-2 LE
     85    IMAGE_FILE_MACHINE_ARMNT 0x01C4  // ARM Thumb-2, LE
     86    Note: We use ARM filter for 0x01C2. (WinCE 5 - 0x01C2) files mostly contain ARM code (not Thumb/Thumb-2).
     87    */
     88 
     89    case 0x01C0:                            // WinCE old
     90    case 0x01C2:  filterId = k_ARM; break;  // WinCE new
     91    case 0x01C4:  filterId = k_ARMT; break; // WinRT
     92 
     93    case 0x0200:  filterId = k_IA64; break;
     94    default:  return 0;
     95  }
     96 
     97  optHeaderSize = GetUi16(p + 16);
     98  if (optHeaderSize > (1 << 10))
     99    return 0;
    100 
    101  p += 20; /* headerSize */
    102 
    103  switch (GetUi16(p))
    104  {
    105    case PE_OptHeader_Magic_32:
    106    case PE_OptHeader_Magic_64:
    107      break;
    108    default:
    109      return 0;
    110  }
    111 
    112  filterMode->Id = filterId;
    113  return 1;
    114 }
    115 
    116 
    117 /* ---------- ELF ---------- */
    118 
    119 #define ELF_SIG 0x464C457F
    120 
    121 #define ELF_CLASS_32  1
    122 #define ELF_CLASS_64  2
    123 
    124 #define ELF_DATA_2LSB 1
    125 #define ELF_DATA_2MSB 2
    126 
    127 static UInt16 Get16(const Byte *p, Bool be) { if (be) return (UInt16)GetBe16(p); return (UInt16)GetUi16(p); }
    128 static UInt32 Get32(const Byte *p, Bool be) { if (be) return GetBe32(p); return GetUi32(p); }
    129 // static UInt64 Get64(const Byte *p, Bool be) { if (be) return GetBe64(p); return GetUi64(p); }
    130 
    131 static int Parse_ELF(const Byte *buf, size_t size, CFilterMode *filterMode)
    132 {
    133  Bool /* is32, */ be;
    134  UInt32 filterId;
    135 
    136  if (size < 512 || buf[6] != 1) /* ver */
    137    return 0;
    138 
    139  if (GetUi32(buf) != ELF_SIG)
    140    return 0;
    141 
    142  switch (buf[4])
    143  {
    144    case ELF_CLASS_32: /* is32 = True; */ break;
    145    case ELF_CLASS_64: /* is32 = False; */ break;
    146    default: return 0;
    147  }
    148 
    149  switch (buf[5])
    150  {
    151    case ELF_DATA_2LSB: be = False; break;
    152    case ELF_DATA_2MSB: be = True; break;
    153    default: return 0;
    154  }
    155 
    156  switch (Get16(buf + 0x12, be))
    157  {
    158    case 3:
    159    case 6:
    160    case 62: filterId = k_X86; break;
    161    case 2:
    162    case 18:
    163    case 43: filterId = k_SPARC; break;
    164    case 20:
    165    case 21: if (!be) return 0; filterId = k_PPC; break;
    166    case 40: if ( be) return 0; filterId = k_ARM; break;
    167    
    168    /* Some IA-64 ELF exacutable have size that is not aligned for 16 bytes.
    169       So we don't use IA-64 filter for IA-64 ELF */
    170    // case 50: if ( be) return 0; filterId = k_IA64; break;
    171 
    172    default: return 0;
    173  }
    174 
    175  filterMode->Id = filterId;
    176  return 1;
    177 }
    178 
    179 
    180 
    181 /* ---------- Mach-O ---------- */
    182 
    183 #define MACH_SIG_BE_32 0xCEFAEDFE
    184 #define MACH_SIG_BE_64 0xCFFAEDFE
    185 #define MACH_SIG_LE_32 0xFEEDFACE
    186 #define MACH_SIG_LE_64 0xFEEDFACF
    187 
    188 #define MACH_ARCH_ABI64 (1 << 24)
    189 #define MACH_MACHINE_386 7
    190 #define MACH_MACHINE_ARM 12
    191 #define MACH_MACHINE_SPARC 14
    192 #define MACH_MACHINE_PPC 18
    193 #define MACH_MACHINE_PPC64 (MACH_ARCH_ABI64 | MACH_MACHINE_PPC)
    194 #define MACH_MACHINE_AMD64 (MACH_ARCH_ABI64 | MACH_MACHINE_386)
    195 
    196 static unsigned Parse_MACH(const Byte *buf, size_t size, CFilterMode *filterMode)
    197 {
    198  UInt32 filterId, numCommands, commandsSize;
    199 
    200  if (size < 512)
    201    return 0;
    202 
    203  Bool /* mode64, */ be;
    204  switch (GetUi32(buf))
    205  {
    206    case MACH_SIG_BE_32: /* mode64 = False; */ be = True; break;
    207    case MACH_SIG_BE_64: /* mode64 = True;  */ be = True; break;
    208    case MACH_SIG_LE_32: /* mode64 = False; */ be = False; break;
    209    case MACH_SIG_LE_64: /* mode64 = True;  */ be = False; break;
    210    default: return 0;
    211  }
    212 
    213  switch (Get32(buf + 4, be))
    214  {
    215    case MACH_MACHINE_386:
    216    case MACH_MACHINE_AMD64: filterId = k_X86; break;
    217    case MACH_MACHINE_ARM:   if ( be) return 0; filterId = k_ARM; break;
    218    case MACH_MACHINE_SPARC: if (!be) return 0; filterId = k_SPARC; break;
    219    case MACH_MACHINE_PPC:
    220    case MACH_MACHINE_PPC64: if (!be) return 0; filterId = k_PPC; break;
    221    default: return 0;
    222  }
    223 
    224  numCommands = Get32(buf + 0x10, be);
    225  commandsSize = Get32(buf + 0x14, be);
    226 
    227  if (commandsSize > (1 << 24) || numCommands > (1 << 18))
    228    return 0;
    229 
    230  filterMode->Id = filterId;
    231  return 1;
    232 }
    233 
    234 
    235 /* ---------- WAV ---------- */
    236 
    237 #define WAV_SUBCHUNK_fmt  0x20746D66
    238 #define WAV_SUBCHUNK_data 0x61746164
    239 
    240 #define RIFF_SIG 0x46464952
    241 
    242 static Bool Parse_WAV(const Byte *buf, size_t size, CFilterMode *filterMode)
    243 {
    244  UInt32 subChunkSize, pos;
    245  if (size < 0x2C)
    246    return False;
    247 
    248  if (GetUi32(buf + 0) != RIFF_SIG ||
    249      GetUi32(buf + 8) != 0x45564157 || // WAVE
    250      GetUi32(buf + 0xC) != WAV_SUBCHUNK_fmt)
    251    return False;
    252  subChunkSize = GetUi32(buf + 0x10);
    253  /* [0x14 = format] = 1 (PCM) */
    254  if (subChunkSize < 0x10 || subChunkSize > 0x12 || GetUi16(buf + 0x14) != 1)
    255    return False;
    256  
    257  unsigned numChannels = GetUi16(buf + 0x16);
    258  unsigned bitsPerSample = GetUi16(buf + 0x22);
    259 
    260  if ((bitsPerSample & 0x7) != 0 || bitsPerSample >= 256 || numChannels >= 256)
    261    return False;
    262 
    263  pos = 0x14 + subChunkSize;
    264 
    265  const int kNumSubChunksTests = 10;
    266  // Do we need to scan more than 3 sub-chunks?
    267  for (int i = 0; i < kNumSubChunksTests; i++)
    268  {
    269    if (pos + 8 > size)
    270      return False;
    271    subChunkSize = GetUi32(buf + pos + 4);
    272    if (GetUi32(buf + pos) == WAV_SUBCHUNK_data)
    273    {
    274      unsigned delta = numChannels * (bitsPerSample >> 3);
    275      if (delta >= 256)
    276        return False;
    277      filterMode->Id = k_Delta;
    278      filterMode->Delta = delta;
    279      return True;
    280    }
    281    if (subChunkSize > (1 << 16))
    282      return False;
    283    pos += subChunkSize + 8;
    284  }
    285  return False;
    286 }
    287 
    288 static Bool ParseFile(const Byte *buf, size_t size, CFilterMode *filterMode)
    289 {
    290  filterMode->Id = 0;
    291  filterMode->Delta = 0;
    292 
    293  if (Parse_EXE(buf, size, filterMode)) return True;
    294  if (Parse_ELF(buf, size, filterMode)) return True;
    295  if (Parse_MACH(buf, size, filterMode)) return True;
    296  return Parse_WAV(buf, size, filterMode);
    297 }
    298 
    299 
    300 
    301 
    302 struct CFilterMode2: public CFilterMode
    303 {
    304  bool Encrypted;
    305  unsigned GroupIndex;
    306  
    307  CFilterMode2(): Encrypted(false) {}
    308 
    309  int Compare(const CFilterMode2 &m) const
    310  {
    311    if (!Encrypted)
    312    {
    313      if (m.Encrypted)
    314        return -1;
    315    }
    316    else if (!m.Encrypted)
    317      return 1;
    318    
    319    if (Id < m.Id) return -1;
    320    if (Id > m.Id) return 1;
    321 
    322    if (Delta < m.Delta) return -1;
    323    if (Delta > m.Delta) return 1;
    324 
    325    return 0;
    326  }
    327  
    328  bool operator ==(const CFilterMode2 &m) const
    329  {
    330    return Id == m.Id && Delta == m.Delta && Encrypted == m.Encrypted;
    331  }
    332 };
    333 
    334 static unsigned GetGroup(CRecordVector<CFilterMode2> &filters, const CFilterMode2 &m)
    335 {
    336  unsigned i;
    337  for (i = 0; i < filters.Size(); i++)
    338  {
    339    const CFilterMode2 &m2 = filters[i];
    340    if (m == m2)
    341      return i;
    342    /*
    343    if (m.Encrypted != m2.Encrypted)
    344    {
    345      if (!m.Encrypted)
    346        break;
    347      continue;
    348    }
    349    
    350    if (m.Id < m2.Id)  break;
    351    if (m.Id != m2.Id) continue;
    352 
    353    if (m.Delta < m2.Delta) break;
    354    if (m.Delta != m2.Delta) continue;
    355    */
    356  }
    357  // filters.Insert(i, m);
    358  // return i;
    359  return filters.Add(m);
    360 }
    361 
    362 static inline bool Is86Filter(CMethodId m)
    363 {
    364  return (m == k_BCJ || m == k_BCJ2);
    365 }
    366 
    367 static inline bool IsExeFilter(CMethodId m)
    368 {
    369  switch (m)
    370  {
    371    case k_BCJ:
    372    case k_BCJ2:
    373    case k_ARM:
    374    case k_ARMT:
    375    case k_PPC:
    376    case k_SPARC:
    377    case k_IA64:
    378      return true;
    379  }
    380  return false;
    381 }
    382 
    383 static unsigned Get_FilterGroup_for_Folder(
    384    CRecordVector<CFilterMode2> &filters, const CFolderEx &f, bool extractFilter)
    385 {
    386  CFilterMode2 m;
    387  m.Id = 0;
    388  m.Delta = 0;
    389  m.Encrypted = f.IsEncrypted();
    390 
    391  if (extractFilter)
    392  {
    393    const CCoderInfo &coder = f.Coders[f.UnpackCoder];
    394  
    395    if (coder.MethodID == k_Delta)
    396    {
    397      if (coder.Props.Size() == 1)
    398      {
    399        m.Delta = (unsigned)coder.Props[0] + 1;
    400        m.Id = k_Delta;
    401      }
    402    }
    403    else if (IsExeFilter(coder.MethodID))
    404    {
    405      m.Id = (UInt32)coder.MethodID;
    406      if (m.Id == k_BCJ2)
    407        m.Id = k_BCJ;
    408      m.SetDelta();
    409    }
    410  }
    411  
    412  return GetGroup(filters, m);
    413 }
    414 
    415 
    416 
    417 
    418 static HRESULT WriteRange(IInStream *inStream, ISequentialOutStream *outStream,
    419    UInt64 position, UInt64 size, ICompressProgressInfo *progress)
    420 {
    421  RINOK(inStream->Seek(position, STREAM_SEEK_SET, 0));
    422  CLimitedSequentialInStream *streamSpec = new CLimitedSequentialInStream;
    423  CMyComPtr<CLimitedSequentialInStream> inStreamLimited(streamSpec);
    424  streamSpec->SetStream(inStream);
    425  streamSpec->Init(size);
    426 
    427  NCompress::CCopyCoder *copyCoderSpec = new NCompress::CCopyCoder;
    428  CMyComPtr<ICompressCoder> copyCoder = copyCoderSpec;
    429  RINOK(copyCoder->Code(inStreamLimited, outStream, NULL, NULL, progress));
    430  return (copyCoderSpec->TotalSize == size ? S_OK : E_FAIL);
    431 }
    432 
    433 /*
    434 unsigned CUpdateItem::GetExtensionPos() const
    435 {
    436  int slashPos = Name.ReverseFind_PathSepar();
    437  int dotPos = Name.ReverseFind_Dot();
    438  if (dotPos <= slashPos)
    439    return Name.Len();
    440  return dotPos + 1;
    441 }
    442 
    443 UString CUpdateItem::GetExtension() const
    444 {
    445  return Name.Ptr(GetExtensionPos());
    446 }
    447 */
    448 
    449 #define RINOZ(x) { int __tt = (x); if (__tt != 0) return __tt; }
    450 
    451 #define RINOZ_COMP(a, b) RINOZ(MyCompare(a, b))
    452 
    453 /*
    454 static int CompareBuffers(const CByteBuffer &a1, const CByteBuffer &a2)
    455 {
    456  size_t c1 = a1.GetCapacity();
    457  size_t c2 = a2.GetCapacity();
    458  RINOZ_COMP(c1, c2);
    459  for (size_t i = 0; i < c1; i++)
    460    RINOZ_COMP(a1[i], a2[i]);
    461  return 0;
    462 }
    463 
    464 static int CompareCoders(const CCoderInfo &c1, const CCoderInfo &c2)
    465 {
    466  RINOZ_COMP(c1.NumInStreams, c2.NumInStreams);
    467  RINOZ_COMP(c1.NumOutStreams, c2.NumOutStreams);
    468  RINOZ_COMP(c1.MethodID, c2.MethodID);
    469  return CompareBuffers(c1.Props, c2.Props);
    470 }
    471 
    472 static int CompareBonds(const CBond &b1, const CBond &b2)
    473 {
    474  RINOZ_COMP(b1.InIndex, b2.InIndex);
    475  return MyCompare(b1.OutIndex, b2.OutIndex);
    476 }
    477 
    478 static int CompareFolders(const CFolder &f1, const CFolder &f2)
    479 {
    480  int s1 = f1.Coders.Size();
    481  int s2 = f2.Coders.Size();
    482  RINOZ_COMP(s1, s2);
    483  int i;
    484  for (i = 0; i < s1; i++)
    485    RINOZ(CompareCoders(f1.Coders[i], f2.Coders[i]));
    486  s1 = f1.Bonds.Size();
    487  s2 = f2.Bonds.Size();
    488  RINOZ_COMP(s1, s2);
    489  for (i = 0; i < s1; i++)
    490    RINOZ(CompareBonds(f1.Bonds[i], f2.Bonds[i]));
    491  return 0;
    492 }
    493 */
    494 
    495 /*
    496 static int CompareFiles(const CFileItem &f1, const CFileItem &f2)
    497 {
    498  return CompareFileNames(f1.Name, f2.Name);
    499 }
    500 */
    501 
    502 struct CFolderRepack
    503 {
    504  unsigned FolderIndex;
    505  CNum NumCopyFiles;
    506 };
    507 
    508 /*
    509 static int CompareFolderRepacks(const CFolderRepack *p1, const CFolderRepack *p2, void *)
    510 {
    511  int i1 = p1->FolderIndex;
    512  int i2 = p2->FolderIndex;
    513  // In that version we don't want to parse folders here, so we don't compare folders
    514  // probably it must be improved in future
    515  // const CDbEx &db = *(const CDbEx *)param;
    516  // RINOZ(CompareFolders(
    517  //     db.Folders[i1],
    518  //     db.Folders[i2]));
    519 
    520  return MyCompare(i1, i2);
    521  
    522  // RINOZ_COMP(
    523  //     db.NumUnpackStreamsVector[i1],
    524  //     db.NumUnpackStreamsVector[i2]);
    525  // if (db.NumUnpackStreamsVector[i1] == 0)
    526  //   return 0;
    527  // return CompareFiles(
    528  //     db.Files[db.FolderStartFileIndex[i1]],
    529  //     db.Files[db.FolderStartFileIndex[i2]]);
    530 }
    531 */
    532 
    533 /*
    534  we sort empty files and dirs in such order:
    535  - Dir.NonAnti   (name sorted)
    536  - File.NonAnti  (name sorted)
    537  - File.Anti     (name sorted)
    538  - Dir.Anti (reverse name sorted)
    539 */
    540 
    541 static int CompareEmptyItems(const unsigned *p1, const unsigned *p2, void *param)
    542 {
    543  const CObjectVector<CUpdateItem> &updateItems = *(const CObjectVector<CUpdateItem> *)param;
    544  const CUpdateItem &u1 = updateItems[*p1];
    545  const CUpdateItem &u2 = updateItems[*p2];
    546  // NonAnti < Anti
    547  if (u1.IsAnti != u2.IsAnti)
    548    return (u1.IsAnti ? 1 : -1);
    549  if (u1.IsDir != u2.IsDir)
    550  {
    551    // Dir.NonAnti < File < Dir.Anti
    552    if (u1.IsDir)
    553      return (u1.IsAnti ? 1 : -1);
    554    return (u2.IsAnti ? -1 : 1);
    555  }
    556  int n = CompareFileNames(u1.Name, u2.Name);
    557  return (u1.IsDir && u1.IsAnti) ? -n : n;
    558 }
    559 
    560 static const char *g_Exts =
    561  " 7z xz lzma ace arc arj bz tbz bz2 tbz2 cab deb gz tgz ha lha lzh lzo lzx pak rar rpm sit zoo"
    562  " zip jar ear war msi"
    563  " 3gp avi mov mpeg mpg mpe wmv"
    564  " aac ape fla flac la mp3 m4a mp4 ofr ogg pac ra rm rka shn swa tta wv wma wav"
    565  " swf"
    566  " chm hxi hxs"
    567  " gif jpeg jpg jp2 png tiff  bmp ico psd psp"
    568  " awg ps eps cgm dxf svg vrml wmf emf ai md"
    569  " cad dwg pps key sxi"
    570  " max 3ds"
    571  " iso bin nrg mdf img pdi tar cpio xpi"
    572  " vfd vhd vud vmc vsv"
    573  " vmdk dsk nvram vmem vmsd vmsn vmss vmtm"
    574  " inl inc idl acf asa"
    575  " h hpp hxx c cpp cxx m mm go swift"
    576  " rc java cs rs pas bas vb cls ctl frm dlg def"
    577  " f77 f f90 f95"
    578  " asm s"
    579  " sql manifest dep"
    580  " mak clw csproj vcproj sln dsp dsw"
    581  " class"
    582  " bat cmd bash sh"
    583  " xml xsd xsl xslt hxk hxc htm html xhtml xht mht mhtml htw asp aspx css cgi jsp shtml"
    584  " awk sed hta js json php php3 php4 php5 phptml pl pm py pyo rb tcl ts vbs"
    585  " text txt tex ans asc srt reg ini doc docx mcw dot rtf hlp xls xlr xlt xlw ppt pdf"
    586  " sxc sxd sxi sxg sxw stc sti stw stm odt ott odg otg odp otp ods ots odf"
    587  " abw afp cwk lwp wpd wps wpt wrf wri"
    588  " abf afm bdf fon mgf otf pcf pfa snf ttf"
    589  " dbf mdb nsf ntf wdb db fdb gdb"
    590  " exe dll ocx vbx sfx sys tlb awx com obj lib out o so"
    591  " pdb pch idb ncb opt";
    592 
    593 static unsigned GetExtIndex(const char *ext)
    594 {
    595  unsigned extIndex = 1;
    596  const char *p = g_Exts;
    597  for (;;)
    598  {
    599    char c = *p++;
    600    if (c == 0)
    601      return extIndex;
    602    if (c == ' ')
    603      continue;
    604    unsigned pos = 0;
    605    for (;;)
    606    {
    607      char c2 = ext[pos++];
    608      if (c2 == 0 && (c == 0 || c == ' '))
    609        return extIndex;
    610      if (c != c2)
    611        break;
    612      c = *p++;
    613    }
    614    extIndex++;
    615    for (;;)
    616    {
    617      if (c == 0)
    618        return extIndex;
    619      if (c == ' ')
    620        break;
    621      c = *p++;
    622    }
    623  }
    624 }
    625 
    626 struct CRefItem
    627 {
    628  const CUpdateItem *UpdateItem;
    629  UInt32 Index;
    630  unsigned ExtensionPos;
    631  unsigned NamePos;
    632  unsigned ExtensionIndex;
    633  
    634  CRefItem() {};
    635  CRefItem(UInt32 index, const CUpdateItem &ui, bool sortByType):
    636    UpdateItem(&ui),
    637    Index(index),
    638    ExtensionPos(0),
    639    NamePos(0),
    640    ExtensionIndex(0)
    641  {
    642    if (sortByType)
    643    {
    644      int slashPos = ui.Name.ReverseFind_PathSepar();
    645      NamePos = slashPos + 1;
    646      int dotPos = ui.Name.ReverseFind_Dot();
    647      if (dotPos <= slashPos)
    648        ExtensionPos = ui.Name.Len();
    649      else
    650      {
    651        ExtensionPos = dotPos + 1;
    652        if (ExtensionPos != ui.Name.Len())
    653        {
    654          AString s;
    655          for (unsigned pos = ExtensionPos;; pos++)
    656          {
    657            wchar_t c = ui.Name[pos];
    658            if (c >= 0x80)
    659              break;
    660            if (c == 0)
    661            {
    662              ExtensionIndex = GetExtIndex(s);
    663              break;
    664            }
    665            s += (char)MyCharLower_Ascii((char)c);
    666          }
    667        }
    668      }
    669    }
    670  }
    671 };
    672 
    673 struct CSortParam
    674 {
    675  // const CObjectVector<CTreeFolder> *TreeFolders;
    676  bool SortByType;
    677 };
    678 
    679 /*
    680  we sort files in such order:
    681  - Dir.NonAnti   (name sorted)
    682  - alt streams
    683  - Dirs
    684  - Dir.Anti (reverse name sorted)
    685 */
    686 
    687 
    688 static int CompareUpdateItems(const CRefItem *p1, const CRefItem *p2, void *param)
    689 {
    690  const CRefItem &a1 = *p1;
    691  const CRefItem &a2 = *p2;
    692  const CUpdateItem &u1 = *a1.UpdateItem;
    693  const CUpdateItem &u2 = *a2.UpdateItem;
    694 
    695  /*
    696  if (u1.IsAltStream != u2.IsAltStream)
    697    return u1.IsAltStream ? 1 : -1;
    698  */
    699  
    700  // Actually there are no dirs that time. They were stored in other steps
    701  // So that code is unused?
    702  if (u1.IsDir != u2.IsDir)
    703    return u1.IsDir ? 1 : -1;
    704  if (u1.IsDir)
    705  {
    706    if (u1.IsAnti != u2.IsAnti)
    707      return (u1.IsAnti ? 1 : -1);
    708    int n = CompareFileNames(u1.Name, u2.Name);
    709    return -n;
    710  }
    711  
    712  // bool sortByType = *(bool *)param;
    713  const CSortParam *sortParam = (const CSortParam *)param;
    714  bool sortByType = sortParam->SortByType;
    715  if (sortByType)
    716  {
    717    RINOZ_COMP(a1.ExtensionIndex, a2.ExtensionIndex);
    718    RINOZ(CompareFileNames(u1.Name.Ptr(a1.ExtensionPos), u2.Name.Ptr(a2.ExtensionPos)));
    719    RINOZ(CompareFileNames(u1.Name.Ptr(a1.NamePos), u2.Name.Ptr(a2.NamePos)));
    720    if (!u1.MTimeDefined && u2.MTimeDefined) return 1;
    721    if (u1.MTimeDefined && !u2.MTimeDefined) return -1;
    722    if (u1.MTimeDefined && u2.MTimeDefined) RINOZ_COMP(u1.MTime, u2.MTime);
    723    RINOZ_COMP(u1.Size, u2.Size);
    724  }
    725  /*
    726  int par1 = a1.UpdateItem->ParentFolderIndex;
    727  int par2 = a2.UpdateItem->ParentFolderIndex;
    728  const CTreeFolder &tf1 = (*sortParam->TreeFolders)[par1];
    729  const CTreeFolder &tf2 = (*sortParam->TreeFolders)[par2];
    730 
    731  int b1 = tf1.SortIndex, e1 = tf1.SortIndexEnd;
    732  int b2 = tf2.SortIndex, e2 = tf2.SortIndexEnd;
    733  if (b1 < b2)
    734  {
    735    if (e1 <= b2)
    736      return -1;
    737    // p2 in p1
    738    int par = par2;
    739    for (;;)
    740    {
    741      const CTreeFolder &tf = (*sortParam->TreeFolders)[par];
    742      par = tf.Parent;
    743      if (par == par1)
    744      {
    745        RINOZ(CompareFileNames(u1.Name, tf.Name));
    746        break;
    747      }
    748    }
    749  }
    750  else if (b2 < b1)
    751  {
    752    if (e2 <= b1)
    753      return 1;
    754    // p1 in p2
    755    int par = par1;
    756    for (;;)
    757    {
    758      const CTreeFolder &tf = (*sortParam->TreeFolders)[par];
    759      par = tf.Parent;
    760      if (par == par2)
    761      {
    762        RINOZ(CompareFileNames(tf.Name, u2.Name));
    763        break;
    764      }
    765    }
    766  }
    767  */
    768  // RINOZ_COMP(a1.UpdateItem->ParentSortIndex, a2.UpdateItem->ParentSortIndex);
    769  RINOK(CompareFileNames(u1.Name, u2.Name));
    770  RINOZ_COMP(a1.UpdateItem->IndexInClient, a2.UpdateItem->IndexInClient);
    771  RINOZ_COMP(a1.UpdateItem->IndexInArchive, a2.UpdateItem->IndexInArchive);
    772  return 0;
    773 }
    774 
    775 struct CSolidGroup
    776 {
    777  CRecordVector<UInt32> Indices;
    778 
    779  CRecordVector<CFolderRepack> folderRefs;
    780 };
    781 
    782 static const char * const g_ExeExts[] =
    783 {
    784    "dll"
    785  , "exe"
    786  , "ocx"
    787  , "sfx"
    788  , "sys"
    789 };
    790 
    791 static bool IsExeExt(const wchar_t *ext)
    792 {
    793  for (unsigned i = 0; i < ARRAY_SIZE(g_ExeExts); i++)
    794    if (StringsAreEqualNoCase_Ascii(ext, g_ExeExts[i]))
    795      return true;
    796  return false;
    797 }
    798 
    799 struct CAnalysis
    800 {
    801  CMyComPtr<IArchiveUpdateCallbackFile> Callback;
    802  CByteBuffer Buffer;
    803 
    804  bool ParseWav;
    805  bool ParseExe;
    806  bool ParseAll;
    807 
    808  CAnalysis():
    809      ParseWav(true),
    810      ParseExe(false),
    811      ParseAll(false)
    812  {}
    813 
    814  HRESULT GetFilterGroup(UInt32 index, const CUpdateItem &ui, CFilterMode &filterMode);
    815 };
    816 
    817 static const size_t kAnalysisBufSize = 1 << 14;
    818 
    819 HRESULT CAnalysis::GetFilterGroup(UInt32 index, const CUpdateItem &ui, CFilterMode &filterMode)
    820 {
    821  filterMode.Id = 0;
    822  filterMode.Delta = 0;
    823 
    824  CFilterMode filterModeTemp = filterMode;
    825 
    826  int slashPos = ui.Name.ReverseFind_PathSepar();
    827  int dotPos = ui.Name.ReverseFind_Dot();
    828 
    829  // if (dotPos > slashPos)
    830  {
    831    bool needReadFile = ParseAll;
    832 
    833    bool probablyIsSameIsa = false;
    834 
    835    if (!needReadFile || !Callback)
    836    {
    837      const wchar_t *ext;
    838      if (dotPos > slashPos)
    839        ext = ui.Name.Ptr(dotPos + 1);
    840      else
    841        ext = ui.Name.RightPtr(0);
    842      
    843      // p7zip uses the trick to store posix attributes in high 16 bits
    844      if (ui.Attrib & 0x8000)
    845      {
    846        unsigned st_mode = ui.Attrib >> 16;
    847        // st_mode = 00111;
    848        if ((st_mode & 00111) && (ui.Size >= 2048))
    849        {
    850          #ifndef _WIN32
    851          probablyIsSameIsa = true;
    852          #endif
    853          needReadFile = true;
    854        }
    855      }
    856 
    857      if (IsExeExt(ext))
    858      {
    859        needReadFile = true;
    860        #ifdef _WIN32
    861        probablyIsSameIsa = true;
    862        needReadFile = ParseExe;
    863        #endif
    864      }
    865      else if (StringsAreEqualNoCase_Ascii(ext, "wav"))
    866      {
    867        needReadFile = ParseWav;
    868      }
    869      /*
    870      else if (!needReadFile && ParseUnixExt)
    871      {
    872        if (StringsAreEqualNoCase_Ascii(ext, "so")
    873          || StringsAreEqualNoCase_Ascii(ext, ""))
    874          
    875          needReadFile = true;
    876      }
    877      */
    878    }
    879 
    880    if (needReadFile && Callback)
    881    {
    882      if (Buffer.Size() != kAnalysisBufSize)
    883      {
    884        Buffer.Alloc(kAnalysisBufSize);
    885      }
    886      {
    887        CMyComPtr<ISequentialInStream> stream;
    888        HRESULT result = Callback->GetStream2(index, &stream, NUpdateNotifyOp::kAnalyze);
    889        if (result == S_OK && stream)
    890        {
    891          size_t size = kAnalysisBufSize;
    892          result = ReadStream(stream, Buffer, &size);
    893          stream.Release();
    894          // RINOK(Callback->SetOperationResult2(index, NUpdate::NOperationResult::kOK));
    895          if (result == S_OK)
    896          {
    897            Bool parseRes = ParseFile(Buffer, size, &filterModeTemp);
    898            if (parseRes && filterModeTemp.Delta == 0)
    899            {
    900              filterModeTemp.SetDelta();
    901              if (filterModeTemp.Delta != 0 && filterModeTemp.Id != k_Delta)
    902              {
    903                if (ui.Size % filterModeTemp.Delta != 0)
    904                {
    905                  parseRes = false;
    906                }
    907              }
    908            }
    909            if (!parseRes)
    910            {
    911              filterModeTemp.Id = 0;
    912              filterModeTemp.Delta = 0;
    913            }
    914          }
    915        }
    916      }
    917    }
    918    else if ((needReadFile && !Callback) || probablyIsSameIsa)
    919    {
    920      #ifdef MY_CPU_X86_OR_AMD64
    921      if (probablyIsSameIsa)
    922        filterModeTemp.Id = k_X86;
    923      #endif
    924    }
    925  }
    926  
    927  filterMode = filterModeTemp;
    928  return S_OK;
    929 }
    930 
    931 static inline void GetMethodFull(UInt64 methodID, UInt32 numStreams, CMethodFull &m)
    932 {
    933  m.Id = methodID;
    934  m.NumStreams = numStreams;
    935 }
    936 
    937 static HRESULT AddBondForFilter(CCompressionMethodMode &mode)
    938 {
    939  for (unsigned c = 1; c < mode.Methods.Size(); c++)
    940  {
    941    if (!mode.IsThereBond_to_Coder(c))
    942    {
    943      CBond2 bond;
    944      bond.OutCoder = 0;
    945      bond.OutStream = 0;
    946      bond.InCoder = c;
    947      mode.Bonds.Add(bond);
    948      return S_OK;
    949    }
    950  }
    951  return E_INVALIDARG;
    952 }
    953 
    954 static HRESULT AddFilterBond(CCompressionMethodMode &mode)
    955 {
    956  if (!mode.Bonds.IsEmpty())
    957    return AddBondForFilter(mode);
    958  return S_OK;
    959 }
    960 
    961 static HRESULT AddBcj2Methods(CCompressionMethodMode &mode)
    962 {
    963  // mode.Methods[0] must be k_BCJ2 method !
    964 
    965  CMethodFull m;
    966  GetMethodFull(k_LZMA, 1, m);
    967  
    968  m.AddProp32(NCoderPropID::kDictionarySize, 1 << 20);
    969  m.AddProp32(NCoderPropID::kNumFastBytes, 128);
    970  m.AddProp32(NCoderPropID::kNumThreads, 1);
    971  m.AddProp32(NCoderPropID::kLitPosBits, 2);
    972  m.AddProp32(NCoderPropID::kLitContextBits, 0);
    973  // m.AddProp_Ascii(NCoderPropID::kMatchFinder, "BT2");
    974 
    975  unsigned methodIndex = mode.Methods.Size();
    976 
    977  if (mode.Bonds.IsEmpty())
    978  {
    979    for (unsigned i = 1; i + 1 < mode.Methods.Size(); i++)
    980    {
    981      CBond2 bond;
    982      bond.OutCoder = i;
    983      bond.OutStream = 0;
    984      bond.InCoder = i + 1;
    985      mode.Bonds.Add(bond);
    986    }
    987  }
    988 
    989  mode.Methods.Add(m);
    990  mode.Methods.Add(m);
    991  
    992  RINOK(AddBondForFilter(mode));
    993  CBond2 bond;
    994  bond.OutCoder = 0;
    995  bond.InCoder = methodIndex;      bond.OutStream = 1;  mode.Bonds.Add(bond);
    996  bond.InCoder = methodIndex + 1;  bond.OutStream = 2;  mode.Bonds.Add(bond);
    997  return S_OK;
    998 }
    999 
   1000 static HRESULT MakeExeMethod(CCompressionMethodMode &mode,
   1001    const CFilterMode &filterMode, /* bool addFilter, */ bool bcj2Filter)
   1002 {
   1003  if (mode.Filter_was_Inserted)
   1004  {
   1005    const CMethodFull &m = mode.Methods[0];
   1006    CMethodId id = m.Id;
   1007    if (id == k_BCJ2)
   1008      return AddBcj2Methods(mode);
   1009    if (!m.IsSimpleCoder())
   1010      return E_NOTIMPL;
   1011    // if (Bonds.IsEmpty()) we can create bonds later
   1012    return AddFilterBond(mode);
   1013  }
   1014 
   1015  if (filterMode.Id == 0)
   1016    return S_OK;
   1017 
   1018  CMethodFull &m = mode.Methods.InsertNew(0);
   1019 
   1020  {
   1021    FOR_VECTOR(k, mode.Bonds)
   1022    {
   1023      CBond2 &bond = mode.Bonds[k];
   1024      bond.InCoder++;
   1025      bond.OutCoder++;
   1026    }
   1027  }
   1028 
   1029  HRESULT res;
   1030  
   1031  if (bcj2Filter && Is86Filter(filterMode.Id))
   1032  {
   1033    GetMethodFull(k_BCJ2, 4, m);
   1034    res = AddBcj2Methods(mode);
   1035  }
   1036  else
   1037  {
   1038    GetMethodFull(filterMode.Id, 1, m);
   1039    if (filterMode.Id == k_Delta)
   1040      m.AddProp32(NCoderPropID::kDefaultProp, filterMode.Delta);
   1041    res = AddFilterBond(mode);
   1042 
   1043    int alignBits = -1;
   1044    if (filterMode.Id == k_Delta || filterMode.Delta != 0)
   1045    {
   1046           if (filterMode.Delta == 1) alignBits = 0;
   1047      else if (filterMode.Delta == 2) alignBits = 1;
   1048      else if (filterMode.Delta == 4) alignBits = 2;
   1049      else if (filterMode.Delta == 8) alignBits = 3;
   1050      else if (filterMode.Delta == 16) alignBits = 4;
   1051    }
   1052    else
   1053    {
   1054      // alignBits = GetAlignForFilterMethod(filterMode.Id);
   1055    }
   1056    
   1057    if (res == S_OK && alignBits >= 0)
   1058    {
   1059      unsigned nextCoder = 1;
   1060      if (!mode.Bonds.IsEmpty())
   1061      {
   1062        nextCoder = mode.Bonds.Back().InCoder;
   1063      }
   1064      if (nextCoder < mode.Methods.Size())
   1065      {
   1066        CMethodFull &nextMethod = mode.Methods[nextCoder];
   1067        if (nextMethod.Id == k_LZMA || nextMethod.Id == k_LZMA2)
   1068        {
   1069          if (!nextMethod.Are_Lzma_Model_Props_Defined())
   1070          {
   1071            if (alignBits != 0)
   1072            {
   1073              if (alignBits > 2 || filterMode.Id == k_Delta)
   1074                nextMethod.AddProp32(NCoderPropID::kPosStateBits, alignBits);
   1075              unsigned lc = 0;
   1076              if (alignBits < 3)
   1077                lc = 3 - alignBits;
   1078              nextMethod.AddProp32(NCoderPropID::kLitContextBits, lc);
   1079              nextMethod.AddProp32(NCoderPropID::kLitPosBits, alignBits);
   1080            }
   1081          }
   1082        }
   1083      }
   1084    }
   1085  }
   1086 
   1087  return res;
   1088 }
   1089 
   1090 
   1091 static void UpdateItem_To_FileItem2(const CUpdateItem &ui, CFileItem2 &file2)
   1092 {
   1093  file2.Attrib = ui.Attrib;  file2.AttribDefined = ui.AttribDefined;
   1094  file2.CTime = ui.CTime;  file2.CTimeDefined = ui.CTimeDefined;
   1095  file2.ATime = ui.ATime;  file2.ATimeDefined = ui.ATimeDefined;
   1096  file2.MTime = ui.MTime;  file2.MTimeDefined = ui.MTimeDefined;
   1097  file2.IsAnti = ui.IsAnti;
   1098  // file2.IsAux = false;
   1099  file2.StartPosDefined = false;
   1100  // file2.StartPos = 0;
   1101 }
   1102 
   1103 
   1104 static void UpdateItem_To_FileItem(const CUpdateItem &ui,
   1105    CFileItem &file, CFileItem2 &file2)
   1106 {
   1107  UpdateItem_To_FileItem2(ui, file2);
   1108 
   1109  file.Size = ui.Size;
   1110  file.IsDir = ui.IsDir;
   1111  file.HasStream = ui.HasStream();
   1112  // file.IsAltStream = ui.IsAltStream;
   1113 }
   1114 
   1115 
   1116 
   1117 class CRepackInStreamWithSizes:
   1118  public ISequentialInStream,
   1119  public ICompressGetSubStreamSize,
   1120  public CMyUnknownImp
   1121 {
   1122  CMyComPtr<ISequentialInStream> _stream;
   1123  // UInt64 _size;
   1124  const CBoolVector *_extractStatuses;
   1125  UInt32 _startIndex;
   1126 public:
   1127  const CDbEx *_db;
   1128 
   1129  void Init(ISequentialInStream *stream, UInt32 startIndex, const CBoolVector *extractStatuses)
   1130  {
   1131    _startIndex = startIndex;
   1132    _extractStatuses = extractStatuses;
   1133    // _size = 0;
   1134    _stream = stream;
   1135  }
   1136  // UInt64 GetSize() const { return _size; }
   1137 
   1138  MY_UNKNOWN_IMP2(ISequentialInStream, ICompressGetSubStreamSize)
   1139 
   1140  STDMETHOD(Read)(void *data, UInt32 size, UInt32 *processedSize);
   1141 
   1142  STDMETHOD(GetSubStreamSize)(UInt64 subStream, UInt64 *value);
   1143 };
   1144 
   1145 STDMETHODIMP CRepackInStreamWithSizes::Read(void *data, UInt32 size, UInt32 *processedSize)
   1146 {
   1147  return _stream->Read(data, size, processedSize);
   1148  /*
   1149  UInt32 realProcessedSize;
   1150  HRESULT result = _stream->Read(data, size, &realProcessedSize);
   1151  _size += realProcessedSize;
   1152  if (processedSize)
   1153    *processedSize = realProcessedSize;
   1154  return result;
   1155  */
   1156 }
   1157 
   1158 STDMETHODIMP CRepackInStreamWithSizes::GetSubStreamSize(UInt64 subStream, UInt64 *value)
   1159 {
   1160  *value = 0;
   1161  if (subStream >= _extractStatuses->Size())
   1162    return S_FALSE; // E_FAIL;
   1163  unsigned index = (unsigned)subStream;
   1164  if ((*_extractStatuses)[index])
   1165  {
   1166    const CFileItem &fi = _db->Files[_startIndex + index];
   1167    if (fi.HasStream)
   1168      *value = fi.Size;
   1169  }
   1170  return S_OK;
   1171 }
   1172 
   1173 
   1174 class CRepackStreamBase
   1175 {
   1176 protected:
   1177  bool _needWrite;
   1178  bool _fileIsOpen;
   1179  bool _calcCrc;
   1180  UInt32 _crc;
   1181  UInt64 _rem;
   1182 
   1183  const CBoolVector *_extractStatuses;
   1184  UInt32 _startIndex;
   1185  unsigned _currentIndex;
   1186 
   1187  HRESULT OpenFile();
   1188  HRESULT CloseFile();
   1189  HRESULT ProcessEmptyFiles();
   1190 
   1191 public:
   1192  const CDbEx *_db;
   1193  CMyComPtr<IArchiveUpdateCallbackFile> _opCallback;
   1194  CMyComPtr<IArchiveExtractCallbackMessage> _extractCallback;
   1195 
   1196  HRESULT Init(UInt32 startIndex, const CBoolVector *extractStatuses);
   1197  HRESULT CheckFinishedState() const { return (_currentIndex == _extractStatuses->Size()) ? S_OK: E_FAIL; }
   1198 };
   1199 
   1200 HRESULT CRepackStreamBase::Init(UInt32 startIndex, const CBoolVector *extractStatuses)
   1201 {
   1202  _startIndex = startIndex;
   1203  _extractStatuses = extractStatuses;
   1204 
   1205  _currentIndex = 0;
   1206  _fileIsOpen = false;
   1207  
   1208  return ProcessEmptyFiles();
   1209 }
   1210 
   1211 HRESULT CRepackStreamBase::OpenFile()
   1212 {
   1213  UInt32 arcIndex = _startIndex + _currentIndex;
   1214  const CFileItem &fi = _db->Files[arcIndex];
   1215  
   1216  _needWrite = (*_extractStatuses)[_currentIndex];
   1217  if (_opCallback)
   1218  {
   1219    RINOK(_opCallback->ReportOperation(
   1220        NEventIndexType::kInArcIndex, arcIndex,
   1221        _needWrite ?
   1222            NUpdateNotifyOp::kRepack :
   1223            NUpdateNotifyOp::kSkip));
   1224  }
   1225 
   1226  _crc = CRC_INIT_VAL;
   1227  _calcCrc = (fi.CrcDefined && !fi.IsDir);
   1228 
   1229  _fileIsOpen = true;
   1230  _rem = fi.Size;
   1231  return S_OK;
   1232 }
   1233 
   1234 const HRESULT k_My_HRESULT_CRC_ERROR = 0x20000002;
   1235 
   1236 HRESULT CRepackStreamBase::CloseFile()
   1237 {
   1238  UInt32 arcIndex = _startIndex + _currentIndex;
   1239  const CFileItem &fi = _db->Files[arcIndex];
   1240  _fileIsOpen = false;
   1241  _currentIndex++;
   1242  if (!_calcCrc || fi.Crc == CRC_GET_DIGEST(_crc))
   1243    return S_OK;
   1244 
   1245  if (_extractCallback)
   1246  {
   1247    RINOK(_extractCallback->ReportExtractResult(
   1248        NEventIndexType::kInArcIndex, arcIndex,
   1249        NExtract::NOperationResult::kCRCError));
   1250  }
   1251  // return S_FALSE;
   1252  return k_My_HRESULT_CRC_ERROR;
   1253 }
   1254 
   1255 HRESULT CRepackStreamBase::ProcessEmptyFiles()
   1256 {
   1257  while (_currentIndex < _extractStatuses->Size() && _db->Files[_startIndex + _currentIndex].Size == 0)
   1258  {
   1259    RINOK(OpenFile());
   1260    RINOK(CloseFile());
   1261  }
   1262  return S_OK;
   1263 }
   1264  
   1265 
   1266 
   1267 #ifndef _7ZIP_ST
   1268 
   1269 class CFolderOutStream2:
   1270  public CRepackStreamBase,
   1271  public ISequentialOutStream,
   1272  public CMyUnknownImp
   1273 {
   1274 public:
   1275  CMyComPtr<ISequentialOutStream> _stream;
   1276 
   1277  MY_UNKNOWN_IMP
   1278 
   1279  STDMETHOD(Write)(const void *data, UInt32 size, UInt32 *processedSize);
   1280 };
   1281 
   1282 STDMETHODIMP CFolderOutStream2::Write(const void *data, UInt32 size, UInt32 *processedSize)
   1283 {
   1284  if (processedSize)
   1285    *processedSize = 0;
   1286  
   1287  while (size != 0)
   1288  {
   1289    if (_fileIsOpen)
   1290    {
   1291      UInt32 cur = (size < _rem ? size : (UInt32)_rem);
   1292      HRESULT result = S_OK;
   1293      if (_needWrite)
   1294        result = _stream->Write(data, cur, &cur);
   1295      if (_calcCrc)
   1296        _crc = CrcUpdate(_crc, data, cur);
   1297      if (processedSize)
   1298        *processedSize += cur;
   1299      data = (const Byte *)data + cur;
   1300      size -= cur;
   1301      _rem -= cur;
   1302      if (_rem == 0)
   1303      {
   1304        RINOK(CloseFile());
   1305        RINOK(ProcessEmptyFiles());
   1306      }
   1307      RINOK(result);
   1308      if (cur == 0)
   1309        break;
   1310      continue;
   1311    }
   1312 
   1313    RINOK(ProcessEmptyFiles());
   1314    if (_currentIndex == _extractStatuses->Size())
   1315    {
   1316      // we don't support write cut here
   1317      return E_FAIL;
   1318    }
   1319    RINOK(OpenFile());
   1320  }
   1321 
   1322  return S_OK;
   1323 }
   1324 
   1325 #endif
   1326 
   1327 
   1328 
   1329 static const UInt32 kTempBufSize = 1 << 16;
   1330 
   1331 class CFolderInStream2:
   1332  public CRepackStreamBase,
   1333  public ISequentialInStream,
   1334  public CMyUnknownImp
   1335 {
   1336  Byte *_buf;
   1337 public:
   1338  CMyComPtr<ISequentialInStream> _inStream;
   1339  HRESULT Result;
   1340 
   1341  MY_UNKNOWN_IMP
   1342 
   1343  CFolderInStream2():
   1344      Result(S_OK)
   1345  {
   1346    _buf = new Byte[kTempBufSize];
   1347  }
   1348 
   1349  ~CFolderInStream2()
   1350  {
   1351    delete []_buf;
   1352  }
   1353 
   1354  void Init() { Result = S_OK; }
   1355  STDMETHOD(Read)(void *data, UInt32 size, UInt32 *processedSize);
   1356 };
   1357 
   1358 STDMETHODIMP CFolderInStream2::Read(void *data, UInt32 size, UInt32 *processedSize)
   1359 {
   1360  if (processedSize)
   1361    *processedSize = 0;
   1362  
   1363  while (size != 0)
   1364  {
   1365    if (_fileIsOpen)
   1366    {
   1367      UInt32 cur = (size < _rem ? size : (UInt32)_rem);
   1368      
   1369      void *buf;
   1370      if (_needWrite)
   1371        buf = data;
   1372      else
   1373      {
   1374        buf = _buf;
   1375        if (cur > kTempBufSize)
   1376          cur = kTempBufSize;
   1377      }
   1378 
   1379      HRESULT result = _inStream->Read(buf, cur, &cur);
   1380      _crc = CrcUpdate(_crc, buf, cur);
   1381      _rem -= cur;
   1382 
   1383      if (_needWrite)
   1384      {
   1385        data = (Byte *)data + cur;
   1386        size -= cur;
   1387        if (processedSize)
   1388          *processedSize += cur;
   1389      }
   1390 
   1391      if (result != S_OK)
   1392        Result = result;
   1393 
   1394      if (_rem == 0)
   1395      {
   1396        RINOK(CloseFile());
   1397        RINOK(ProcessEmptyFiles());
   1398      }
   1399 
   1400      RINOK(result);
   1401      
   1402      if (cur == 0)
   1403        return E_FAIL;
   1404 
   1405      continue;
   1406    }
   1407 
   1408    RINOK(ProcessEmptyFiles());
   1409    if (_currentIndex == _extractStatuses->Size())
   1410    {
   1411      return S_OK;
   1412    }
   1413    RINOK(OpenFile());
   1414  }
   1415  
   1416  return S_OK;
   1417 }
   1418 
   1419 
   1420 class CThreadDecoder
   1421  #ifndef _7ZIP_ST
   1422    : public CVirtThread
   1423  #endif
   1424 {
   1425 public:
   1426  CDecoder Decoder;
   1427 
   1428  CThreadDecoder(bool multiThreadMixer):
   1429      Decoder(multiThreadMixer)
   1430  {
   1431    #ifndef _7ZIP_ST
   1432    if (multiThreadMixer)
   1433    {
   1434      MtMode = false;
   1435      NumThreads = 1;
   1436      FosSpec = new CFolderOutStream2;
   1437      Fos = FosSpec;
   1438      Result = E_FAIL;
   1439    }
   1440    #endif
   1441    // UnpackSize = 0;
   1442    // send_UnpackSize = false;
   1443  }
   1444 
   1445  #ifndef _7ZIP_ST
   1446  
   1447  bool dataAfterEnd_Error;
   1448  HRESULT Result;
   1449  CMyComPtr<IInStream> InStream;
   1450 
   1451  CFolderOutStream2 *FosSpec;
   1452  CMyComPtr<ISequentialOutStream> Fos;
   1453 
   1454  UInt64 StartPos;
   1455  const CFolders *Folders;
   1456  int FolderIndex;
   1457 
   1458  // bool send_UnpackSize;
   1459  // UInt64 UnpackSize;
   1460  
   1461  #ifndef _NO_CRYPTO
   1462  CMyComPtr<ICryptoGetTextPassword> getTextPassword;
   1463  #endif
   1464 
   1465  DECL_EXTERNAL_CODECS_LOC_VARS2;
   1466 
   1467  #ifndef _7ZIP_ST
   1468  bool MtMode;
   1469  UInt32 NumThreads;
   1470  #endif
   1471 
   1472  
   1473  ~CThreadDecoder() { CVirtThread::WaitThreadFinish(); }
   1474  virtual void Execute();
   1475 
   1476  #endif
   1477 };
   1478 
   1479 #ifndef _7ZIP_ST
   1480 
   1481 void CThreadDecoder::Execute()
   1482 {
   1483  try
   1484  {
   1485    #ifndef _NO_CRYPTO
   1486      bool isEncrypted = false;
   1487      bool passwordIsDefined = false;
   1488      UString password;
   1489    #endif
   1490 
   1491    dataAfterEnd_Error = false;
   1492      
   1493    Result = Decoder.Decode(
   1494      EXTERNAL_CODECS_LOC_VARS
   1495      InStream,
   1496      StartPos,
   1497      *Folders, FolderIndex,
   1498      
   1499      // send_UnpackSize ? &UnpackSize : NULL,
   1500      NULL, // unpackSize : FULL unpack
   1501      
   1502      Fos,
   1503      NULL, // compressProgress
   1504 
   1505      NULL  // *inStreamMainRes
   1506      , dataAfterEnd_Error
   1507 
   1508      _7Z_DECODER_CRYPRO_VARS
   1509      #ifndef _7ZIP_ST
   1510        , MtMode, NumThreads,
   1511        0 // MemUsage
   1512      #endif
   1513 
   1514      );
   1515  }
   1516  catch(...)
   1517  {
   1518    Result = E_FAIL;
   1519  }
   1520  
   1521  /*
   1522  if (Result == S_OK)
   1523    Result = FosSpec->CheckFinishedState();
   1524  */
   1525  FosSpec->_stream.Release();
   1526 }
   1527 
   1528 #endif
   1529 
   1530 #ifndef _NO_CRYPTO
   1531 
   1532 class CCryptoGetTextPassword:
   1533  public ICryptoGetTextPassword,
   1534  public CMyUnknownImp
   1535 {
   1536 public:
   1537  UString Password;
   1538 
   1539  MY_UNKNOWN_IMP
   1540  STDMETHOD(CryptoGetTextPassword)(BSTR *password);
   1541 };
   1542 
   1543 STDMETHODIMP CCryptoGetTextPassword::CryptoGetTextPassword(BSTR *password)
   1544 {
   1545  return StringToBstr(Password, password);
   1546 }
   1547 
   1548 #endif
   1549 
   1550 
   1551 static void GetFile(const CDatabase &inDb, unsigned index, CFileItem &file, CFileItem2 &file2)
   1552 {
   1553  file = inDb.Files[index];
   1554  file2.CTimeDefined = inDb.CTime.GetItem(index, file2.CTime);
   1555  file2.ATimeDefined = inDb.ATime.GetItem(index, file2.ATime);
   1556  file2.MTimeDefined = inDb.MTime.GetItem(index, file2.MTime);
   1557  file2.StartPosDefined = inDb.StartPos.GetItem(index, file2.StartPos);
   1558  file2.AttribDefined = inDb.Attrib.GetItem(index, file2.Attrib);
   1559  file2.IsAnti = inDb.IsItemAnti(index);
   1560  // file2.IsAux = inDb.IsItemAux(index);
   1561 }
   1562 
   1563 HRESULT Update(
   1564    DECL_EXTERNAL_CODECS_LOC_VARS
   1565    IInStream *inStream,
   1566    const CDbEx *db,
   1567    const CObjectVector<CUpdateItem> &updateItems,
   1568    // const CObjectVector<CTreeFolder> &treeFolders,
   1569    // const CUniqBlocks &secureBlocks,
   1570    COutArchive &archive,
   1571    CArchiveDatabaseOut &newDatabase,
   1572    ISequentialOutStream *seqOutStream,
   1573    IArchiveUpdateCallback *updateCallback,
   1574    const CUpdateOptions &options
   1575    #ifndef _NO_CRYPTO
   1576    , ICryptoGetTextPassword *getDecoderPassword
   1577    #endif
   1578    )
   1579 {
   1580  UInt64 numSolidFiles = options.NumSolidFiles;
   1581  if (numSolidFiles == 0)
   1582    numSolidFiles = 1;
   1583 
   1584  CMyComPtr<IArchiveUpdateCallbackFile> opCallback;
   1585  updateCallback->QueryInterface(IID_IArchiveUpdateCallbackFile, (void **)&opCallback);
   1586 
   1587  CMyComPtr<IArchiveExtractCallbackMessage> extractCallback;
   1588  updateCallback->QueryInterface(IID_IArchiveExtractCallbackMessage, (void **)&extractCallback);
   1589 
   1590  // size_t totalSecureDataSize = (size_t)secureBlocks.GetTotalSizeInBytes();
   1591 
   1592  /*
   1593  CMyComPtr<IOutStream> outStream;
   1594  RINOK(seqOutStream->QueryInterface(IID_IOutStream, (void **)&outStream));
   1595  if (!outStream)
   1596    return E_NOTIMPL;
   1597  */
   1598 
   1599  UInt64 startBlockSize = db ? db->ArcInfo.StartPosition: 0;
   1600  if (startBlockSize > 0 && !options.RemoveSfxBlock)
   1601  {
   1602    RINOK(WriteRange(inStream, seqOutStream, 0, startBlockSize, NULL));
   1603  }
   1604 
   1605  CIntArr fileIndexToUpdateIndexMap;
   1606  UInt64 complexity = 0;
   1607  UInt64 inSizeForReduce2 = 0;
   1608  bool needEncryptedRepack = false;
   1609 
   1610  CRecordVector<CFilterMode2> filters;
   1611  CObjectVector<CSolidGroup> groups;
   1612  bool thereAreRepacks = false;
   1613 
   1614  bool useFilters = options.UseFilters;
   1615  if (useFilters)
   1616  {
   1617    const CCompressionMethodMode &method = *options.Method;
   1618 
   1619    FOR_VECTOR (i, method.Methods)
   1620      if (IsFilterMethod(method.Methods[i].Id))
   1621      {
   1622        useFilters = false;
   1623        break;
   1624      }
   1625  }
   1626  
   1627  if (db)
   1628  {
   1629    fileIndexToUpdateIndexMap.Alloc(db->Files.Size());
   1630    unsigned i;
   1631    
   1632    for (i = 0; i < db->Files.Size(); i++)
   1633      fileIndexToUpdateIndexMap[i] = -1;
   1634 
   1635    for (i = 0; i < updateItems.Size(); i++)
   1636    {
   1637      int index = updateItems[i].IndexInArchive;
   1638      if (index != -1)
   1639        fileIndexToUpdateIndexMap[(unsigned)index] = i;
   1640    }
   1641 
   1642    for (i = 0; i < db->NumFolders; i++)
   1643    {
   1644      CNum indexInFolder = 0;
   1645      CNum numCopyItems = 0;
   1646      CNum numUnpackStreams = db->NumUnpackStreamsVector[i];
   1647      UInt64 repackSize = 0;
   1648      
   1649      for (CNum fi = db->FolderStartFileIndex[i]; indexInFolder < numUnpackStreams; fi++)
   1650      {
   1651        const CFileItem &file = db->Files[fi];
   1652        if (file.HasStream)
   1653        {
   1654          indexInFolder++;
   1655          int updateIndex = fileIndexToUpdateIndexMap[fi];
   1656          if (updateIndex >= 0 && !updateItems[updateIndex].NewData)
   1657          {
   1658            numCopyItems++;
   1659            repackSize += file.Size;
   1660          }
   1661        }
   1662      }
   1663 
   1664      if (numCopyItems == 0)
   1665        continue;
   1666 
   1667      CFolderRepack rep;
   1668      rep.FolderIndex = i;
   1669      rep.NumCopyFiles = numCopyItems;
   1670      CFolderEx f;
   1671      db->ParseFolderEx(i, f);
   1672 
   1673      const bool isEncrypted = f.IsEncrypted();
   1674      const bool needCopy = (numCopyItems == numUnpackStreams);
   1675      const bool extractFilter = (useFilters || needCopy);
   1676 
   1677      unsigned groupIndex = Get_FilterGroup_for_Folder(filters, f, extractFilter);
   1678      
   1679      while (groupIndex >= groups.Size())
   1680        groups.AddNew();
   1681 
   1682      groups[groupIndex].folderRefs.Add(rep);
   1683      
   1684      if (needCopy)
   1685        complexity += db->GetFolderFullPackSize(i);
   1686      else
   1687      {
   1688        thereAreRepacks = true;
   1689        complexity += repackSize;
   1690        if (inSizeForReduce2 < repackSize)
   1691          inSizeForReduce2 = repackSize;
   1692        if (isEncrypted)
   1693          needEncryptedRepack = true;
   1694      }
   1695    }
   1696  }
   1697 
   1698  UInt64 inSizeForReduce = 0;
   1699  {
   1700    bool isSolid = (numSolidFiles > 1 && options.NumSolidBytes != 0);
   1701    FOR_VECTOR (i, updateItems)
   1702    {
   1703      const CUpdateItem &ui = updateItems[i];
   1704      if (ui.NewData)
   1705      {
   1706        complexity += ui.Size;
   1707        if (isSolid)
   1708          inSizeForReduce += ui.Size;
   1709        else if (inSizeForReduce < ui.Size)
   1710          inSizeForReduce = ui.Size;
   1711      }
   1712    }
   1713  }
   1714 
   1715  if (inSizeForReduce < inSizeForReduce2)
   1716    inSizeForReduce = inSizeForReduce2;
   1717 
   1718  RINOK(updateCallback->SetTotal(complexity));
   1719 
   1720  CLocalProgress *lps = new CLocalProgress;
   1721  CMyComPtr<ICompressProgressInfo> progress = lps;
   1722  lps->Init(updateCallback, true);
   1723 
   1724  #ifndef _7ZIP_ST
   1725  
   1726  CStreamBinder sb;
   1727  if (options.MultiThreadMixer)
   1728  {
   1729    RINOK(sb.CreateEvents());
   1730  }
   1731  
   1732  #endif
   1733 
   1734  CThreadDecoder threadDecoder(options.MultiThreadMixer);
   1735  
   1736  #ifndef _7ZIP_ST
   1737  if (options.MultiThreadMixer && thereAreRepacks)
   1738  {
   1739    #ifdef EXTERNAL_CODECS
   1740    threadDecoder.__externalCodecs = __externalCodecs;
   1741    #endif
   1742    RINOK(threadDecoder.Create());
   1743  }
   1744  #endif
   1745 
   1746  {
   1747    CAnalysis analysis;
   1748    if (options.AnalysisLevel == 0)
   1749    {
   1750      analysis.ParseWav = false;
   1751      analysis.ParseExe = false;
   1752      analysis.ParseAll = false;
   1753    }
   1754    else
   1755    {
   1756      analysis.Callback = opCallback;
   1757      if (options.AnalysisLevel > 0)
   1758      {
   1759        analysis.ParseWav = true;
   1760        if (options.AnalysisLevel >= 7)
   1761        {
   1762          analysis.ParseExe = true;
   1763          if (options.AnalysisLevel >= 9)
   1764            analysis.ParseAll = true;
   1765        }
   1766      }
   1767    }
   1768 
   1769    // ---------- Split files to groups ----------
   1770 
   1771    const CCompressionMethodMode &method = *options.Method;
   1772    
   1773    FOR_VECTOR (i, updateItems)
   1774    {
   1775      const CUpdateItem &ui = updateItems[i];
   1776      if (!ui.NewData || !ui.HasStream())
   1777        continue;
   1778 
   1779      CFilterMode2 fm;
   1780      if (useFilters)
   1781      {
   1782        RINOK(analysis.GetFilterGroup(i, ui, fm));
   1783      }
   1784      fm.Encrypted = method.PasswordIsDefined;
   1785 
   1786      unsigned groupIndex = GetGroup(filters, fm);
   1787      while (groupIndex >= groups.Size())
   1788        groups.AddNew();
   1789      groups[groupIndex].Indices.Add(i);
   1790    }
   1791  }
   1792 
   1793 
   1794  #ifndef _NO_CRYPTO
   1795 
   1796  CCryptoGetTextPassword *getPasswordSpec = NULL;
   1797  CMyComPtr<ICryptoGetTextPassword> getTextPassword;
   1798  if (needEncryptedRepack)
   1799  {
   1800    getPasswordSpec = new CCryptoGetTextPassword;
   1801    getTextPassword = getPasswordSpec;
   1802    
   1803    #ifndef _7ZIP_ST
   1804    threadDecoder.getTextPassword = getPasswordSpec;
   1805    #endif
   1806 
   1807    if (options.Method->PasswordIsDefined)
   1808      getPasswordSpec->Password = options.Method->Password;
   1809    else
   1810    {
   1811      if (!getDecoderPassword)
   1812        return E_NOTIMPL;
   1813      CMyComBSTR password;
   1814      RINOK(getDecoderPassword->CryptoGetTextPassword(&password));
   1815      if (password)
   1816        getPasswordSpec->Password = password;
   1817    }
   1818  }
   1819 
   1820  #endif
   1821 
   1822  
   1823  // ---------- Compress ----------
   1824 
   1825  RINOK(archive.Create(seqOutStream, false));
   1826  RINOK(archive.SkipPrefixArchiveHeader());
   1827 
   1828  /*
   1829  CIntVector treeFolderToArcIndex;
   1830  treeFolderToArcIndex.Reserve(treeFolders.Size());
   1831  for (i = 0; i < treeFolders.Size(); i++)
   1832    treeFolderToArcIndex.Add(-1);
   1833  // ---------- Write Tree (only AUX dirs) ----------
   1834  for (i = 1; i < treeFolders.Size(); i++)
   1835  {
   1836    const CTreeFolder &treeFolder = treeFolders[i];
   1837    CFileItem file;
   1838    CFileItem2 file2;
   1839    file2.Init();
   1840    int secureID = 0;
   1841    if (treeFolder.UpdateItemIndex < 0)
   1842    {
   1843      // we can store virtual dir item wuthout attrib, but we want all items have attrib.
   1844      file.SetAttrib(FILE_ATTRIBUTE_DIRECTORY);
   1845      file2.IsAux = true;
   1846    }
   1847    else
   1848    {
   1849      const CUpdateItem &ui = updateItems[treeFolder.UpdateItemIndex];
   1850      // if item is not dir, then it's parent for alt streams.
   1851      // we will write such items later
   1852      if (!ui.IsDir)
   1853        continue;
   1854      secureID = ui.SecureIndex;
   1855      if (ui.NewProps)
   1856        UpdateItem_To_FileItem(ui, file, file2);
   1857      else
   1858        GetFile(*db, ui.IndexInArchive, file, file2);
   1859    }
   1860    file.Size = 0;
   1861    file.HasStream = false;
   1862    file.IsDir = true;
   1863    file.Parent = treeFolder.Parent;
   1864    
   1865    treeFolderToArcIndex[i] = newDatabase.Files.Size();
   1866    newDatabase.AddFile(file, file2, treeFolder.Name);
   1867    
   1868    if (totalSecureDataSize != 0)
   1869      newDatabase.SecureIDs.Add(secureID);
   1870  }
   1871  */
   1872 
   1873  {
   1874    /* ---------- Write non-AUX dirs and Empty files ---------- */
   1875    CUIntVector emptyRefs;
   1876    
   1877    unsigned i;
   1878 
   1879    for (i = 0; i < updateItems.Size(); i++)
   1880    {
   1881      const CUpdateItem &ui = updateItems[i];
   1882      if (ui.NewData)
   1883      {
   1884        if (ui.HasStream())
   1885          continue;
   1886      }
   1887      else if (ui.IndexInArchive != -1 && db->Files[ui.IndexInArchive].HasStream)
   1888        continue;
   1889      /*
   1890      if (ui.TreeFolderIndex >= 0)
   1891        continue;
   1892      */
   1893      emptyRefs.Add(i);
   1894    }
   1895    
   1896    emptyRefs.Sort(CompareEmptyItems, (void *)&updateItems);
   1897    
   1898    for (i = 0; i < emptyRefs.Size(); i++)
   1899    {
   1900      const CUpdateItem &ui = updateItems[emptyRefs[i]];
   1901      CFileItem file;
   1902      CFileItem2 file2;
   1903      UString name;
   1904      if (ui.NewProps)
   1905      {
   1906        UpdateItem_To_FileItem(ui, file, file2);
   1907        file.CrcDefined = false;
   1908        name = ui.Name;
   1909      }
   1910      else
   1911      {
   1912        GetFile(*db, ui.IndexInArchive, file, file2);
   1913        db->GetPath(ui.IndexInArchive, name);
   1914      }
   1915      
   1916      /*
   1917      if (totalSecureDataSize != 0)
   1918        newDatabase.SecureIDs.Add(ui.SecureIndex);
   1919      file.Parent = ui.ParentFolderIndex;
   1920      */
   1921      newDatabase.AddFile(file, file2, name);
   1922    }
   1923  }
   1924 
   1925  lps->ProgressOffset = 0;
   1926 
   1927  {
   1928    // ---------- Sort Filters ----------
   1929    
   1930    FOR_VECTOR (i, filters)
   1931    {
   1932      filters[i].GroupIndex = i;
   1933    }
   1934    filters.Sort2();
   1935  }
   1936 
   1937  for (unsigned groupIndex = 0; groupIndex < filters.Size(); groupIndex++)
   1938  {
   1939    const CFilterMode2 &filterMode = filters[groupIndex];
   1940 
   1941    CCompressionMethodMode method = *options.Method;
   1942    {
   1943      HRESULT res = MakeExeMethod(method, filterMode,
   1944        #ifdef _7ZIP_ST
   1945          false
   1946        #else
   1947          options.MaxFilter && options.MultiThreadMixer
   1948        #endif
   1949        );
   1950 
   1951      RINOK(res);
   1952    }
   1953 
   1954    if (filterMode.Encrypted)
   1955    {
   1956      if (!method.PasswordIsDefined)
   1957      {
   1958        #ifndef _NO_CRYPTO
   1959        if (getPasswordSpec)
   1960          method.Password = getPasswordSpec->Password;
   1961        #endif
   1962        method.PasswordIsDefined = true;
   1963      }
   1964    }
   1965    else
   1966    {
   1967      method.PasswordIsDefined = false;
   1968      method.Password.Empty();
   1969    }
   1970 
   1971    CEncoder encoder(method);
   1972 
   1973    // ---------- Repack and copy old solid blocks ----------
   1974 
   1975    const CSolidGroup &group = groups[filterMode.GroupIndex];
   1976    
   1977    FOR_VECTOR(folderRefIndex, group.folderRefs)
   1978    {
   1979      const CFolderRepack &rep = group.folderRefs[folderRefIndex];
   1980 
   1981      unsigned folderIndex = rep.FolderIndex;
   1982      
   1983      CNum numUnpackStreams = db->NumUnpackStreamsVector[folderIndex];
   1984 
   1985      if (rep.NumCopyFiles == numUnpackStreams)
   1986      {
   1987        if (opCallback)
   1988        {
   1989          RINOK(opCallback->ReportOperation(
   1990              NEventIndexType::kBlockIndex, (UInt32)folderIndex,
   1991              NUpdateNotifyOp::kReplicate));
   1992 
   1993          // ---------- Copy old solid block ----------
   1994          {
   1995            CNum indexInFolder = 0;
   1996            for (CNum fi = db->FolderStartFileIndex[folderIndex]; indexInFolder < numUnpackStreams; fi++)
   1997            {
   1998              if (db->Files[fi].HasStream)
   1999              {
   2000                indexInFolder++;
   2001                RINOK(opCallback->ReportOperation(
   2002                    NEventIndexType::kInArcIndex, (UInt32)fi,
   2003                    NUpdateNotifyOp::kReplicate));
   2004              }
   2005            }
   2006          }
   2007        }
   2008 
   2009        UInt64 packSize = db->GetFolderFullPackSize(folderIndex);
   2010        RINOK(WriteRange(inStream, archive.SeqStream,
   2011            db->GetFolderStreamPos(folderIndex, 0), packSize, progress));
   2012        lps->ProgressOffset += packSize;
   2013        
   2014        CFolder &folder = newDatabase.Folders.AddNew();
   2015        db->ParseFolderInfo(folderIndex, folder);
   2016        CNum startIndex = db->FoStartPackStreamIndex[folderIndex];
   2017        FOR_VECTOR(j, folder.PackStreams)
   2018        {
   2019          newDatabase.PackSizes.Add(db->GetStreamPackSize(startIndex + j));
   2020          // newDatabase.PackCRCsDefined.Add(db.PackCRCsDefined[startIndex + j]);
   2021          // newDatabase.PackCRCs.Add(db.PackCRCs[startIndex + j]);
   2022        }
   2023 
   2024        size_t indexStart = db->FoToCoderUnpackSizes[folderIndex];
   2025        size_t indexEnd = db->FoToCoderUnpackSizes[folderIndex + 1];
   2026        for (; indexStart < indexEnd; indexStart++)
   2027          newDatabase.CoderUnpackSizes.Add(db->CoderUnpackSizes[indexStart]);
   2028      }
   2029      else
   2030      {
   2031        // ---------- Repack old solid block ----------
   2032 
   2033        CBoolVector extractStatuses;
   2034        
   2035        CNum indexInFolder = 0;
   2036 
   2037        if (opCallback)
   2038        {
   2039          RINOK(opCallback->ReportOperation(
   2040              NEventIndexType::kBlockIndex, (UInt32)folderIndex,
   2041              NUpdateNotifyOp::kRepack))
   2042        }
   2043 
   2044        /* We could reduce data size of decoded folder, if we don't need to repack
   2045           last files in folder. But the gain in speed is small in most cases.
   2046           So we unpack full folder. */
   2047           
   2048        UInt64 sizeToEncode = 0;
   2049  
   2050        /*
   2051        UInt64 importantUnpackSize = 0;
   2052        unsigned numImportantFiles = 0;
   2053        UInt64 decodeSize = 0;
   2054        */
   2055 
   2056        for (CNum fi = db->FolderStartFileIndex[folderIndex]; indexInFolder < numUnpackStreams; fi++)
   2057        {
   2058          bool needExtract = false;
   2059          const CFileItem &file = db->Files[fi];
   2060  
   2061          if (file.HasStream)
   2062          {
   2063            indexInFolder++;
   2064            int updateIndex = fileIndexToUpdateIndexMap[fi];
   2065            if (updateIndex >= 0 && !updateItems[updateIndex].NewData)
   2066              needExtract = true;
   2067            // decodeSize += file.Size;
   2068          }
   2069          
   2070          extractStatuses.Add(needExtract);
   2071          if (needExtract)
   2072          {
   2073            sizeToEncode += file.Size;
   2074            /*
   2075            numImportantFiles = extractStatuses.Size();
   2076            importantUnpackSize = decodeSize;
   2077            */
   2078          }
   2079        }
   2080 
   2081        // extractStatuses.DeleteFrom(numImportantFiles);
   2082 
   2083        unsigned startPackIndex = newDatabase.PackSizes.Size();
   2084        UInt64 curUnpackSize;
   2085        {
   2086 
   2087          CMyComPtr<ISequentialInStream> sbInStream;
   2088          CRepackStreamBase *repackBase;
   2089          CFolderInStream2 *FosSpec2 = NULL;
   2090 
   2091          CRepackInStreamWithSizes *inStreamSizeCountSpec = new CRepackInStreamWithSizes;
   2092          CMyComPtr<ISequentialInStream> inStreamSizeCount = inStreamSizeCountSpec;
   2093          {
   2094            #ifndef _7ZIP_ST
   2095            if (options.MultiThreadMixer)
   2096            {
   2097              repackBase = threadDecoder.FosSpec;
   2098              CMyComPtr<ISequentialOutStream> sbOutStream;
   2099              sb.CreateStreams(&sbInStream, &sbOutStream);
   2100              sb.ReInit();
   2101              
   2102              threadDecoder.FosSpec->_stream = sbOutStream;
   2103              
   2104              threadDecoder.InStream = inStream;
   2105              threadDecoder.StartPos = db->ArcInfo.DataStartPosition; // db->GetFolderStreamPos(folderIndex, 0);
   2106              threadDecoder.Folders = (const CFolders *)db;
   2107              threadDecoder.FolderIndex = folderIndex;
   2108             
   2109              // threadDecoder.UnpackSize = importantUnpackSize;
   2110              // threadDecoder.send_UnpackSize = true;
   2111            }
   2112            else
   2113            #endif
   2114            {
   2115              FosSpec2 = new CFolderInStream2;
   2116              FosSpec2->Init();
   2117              sbInStream = FosSpec2;
   2118              repackBase = FosSpec2;
   2119 
   2120              #ifndef _NO_CRYPTO
   2121              bool isEncrypted = false;
   2122              bool passwordIsDefined = false;
   2123              UString password;
   2124              #endif
   2125              
   2126              CMyComPtr<ISequentialInStream> decodedStream;
   2127              bool dataAfterEnd_Error = false;
   2128 
   2129              HRESULT res = threadDecoder.Decoder.Decode(
   2130                  EXTERNAL_CODECS_LOC_VARS
   2131                  inStream,
   2132                  db->ArcInfo.DataStartPosition, // db->GetFolderStreamPos(folderIndex, 0);,
   2133                  *db, folderIndex,
   2134                  // &importantUnpackSize, // *unpackSize
   2135                  NULL, // *unpackSize : FULL unpack
   2136                
   2137                  NULL, // *outStream
   2138                  NULL, // *compressProgress
   2139 
   2140                  &decodedStream
   2141                  , dataAfterEnd_Error
   2142                
   2143                  _7Z_DECODER_CRYPRO_VARS
   2144                  #ifndef _7ZIP_ST
   2145                    , false // mtMode
   2146                    , 1 // numThreads
   2147                    , 0 // memUsage
   2148                  #endif
   2149                );
   2150          
   2151              RINOK(res);
   2152              if (!decodedStream)
   2153                return E_FAIL;
   2154 
   2155              FosSpec2->_inStream = decodedStream;
   2156            }
   2157 
   2158            repackBase->_db = db;
   2159            repackBase->_opCallback = opCallback;
   2160            repackBase->_extractCallback = extractCallback;
   2161 
   2162            UInt32 startIndex = db->FolderStartFileIndex[folderIndex];
   2163            RINOK(repackBase->Init(startIndex, &extractStatuses));
   2164 
   2165            inStreamSizeCountSpec->_db = db;
   2166            inStreamSizeCountSpec->Init(sbInStream, startIndex, &extractStatuses);
   2167 
   2168            #ifndef _7ZIP_ST
   2169            if (options.MultiThreadMixer)
   2170            {
   2171              threadDecoder.Start();
   2172            }
   2173            #endif
   2174          }
   2175 
   2176          curUnpackSize = sizeToEncode;
   2177 
   2178          HRESULT encodeRes = encoder.Encode(
   2179              EXTERNAL_CODECS_LOC_VARS
   2180              inStreamSizeCount,
   2181              // NULL,
   2182              &inSizeForReduce,
   2183              newDatabase.Folders.AddNew(), newDatabase.CoderUnpackSizes, curUnpackSize,
   2184              archive.SeqStream, newDatabase.PackSizes, progress);
   2185 
   2186          if (encodeRes == k_My_HRESULT_CRC_ERROR)
   2187            return E_FAIL;
   2188 
   2189          #ifndef _7ZIP_ST
   2190          if (options.MultiThreadMixer)
   2191          {
   2192            // 16.00: hang was fixed : for case if decoding was not finished.
   2193            // We close CBinderInStream and it calls CStreamBinder::CloseRead()
   2194            inStreamSizeCount.Release();
   2195            sbInStream.Release();
   2196            
   2197            threadDecoder.WaitExecuteFinish();
   2198            
   2199            HRESULT decodeRes = threadDecoder.Result;
   2200            // if (res == k_My_HRESULT_CRC_ERROR)
   2201            if (decodeRes == S_FALSE || threadDecoder.dataAfterEnd_Error)
   2202            {
   2203              if (extractCallback)
   2204              {
   2205                RINOK(extractCallback->ReportExtractResult(
   2206                    NEventIndexType::kInArcIndex, db->FolderStartFileIndex[folderIndex],
   2207                    // NEventIndexType::kBlockIndex, (UInt32)folderIndex,
   2208                    (decodeRes != S_OK ?
   2209                      NExtract::NOperationResult::kDataError :
   2210                      NExtract::NOperationResult::kDataAfterEnd)));
   2211              }
   2212              if (decodeRes != S_OK)
   2213                return E_FAIL;
   2214            }
   2215            RINOK(decodeRes);
   2216            if (encodeRes == S_OK)
   2217              if (sb.ProcessedSize != sizeToEncode)
   2218                encodeRes = E_FAIL;
   2219          }
   2220          else
   2221          #endif
   2222          {
   2223            if (FosSpec2->Result == S_FALSE)
   2224            {
   2225              if (extractCallback)
   2226              {
   2227                RINOK(extractCallback->ReportExtractResult(
   2228                    NEventIndexType::kBlockIndex, (UInt32)folderIndex,
   2229                    NExtract::NOperationResult::kDataError));
   2230              }
   2231              return E_FAIL;
   2232            }
   2233            RINOK(FosSpec2->Result);
   2234          }
   2235 
   2236          RINOK(encodeRes);
   2237          RINOK(repackBase->CheckFinishedState());
   2238 
   2239          if (curUnpackSize != sizeToEncode)
   2240            return E_FAIL;
   2241        }
   2242 
   2243        for (; startPackIndex < newDatabase.PackSizes.Size(); startPackIndex++)
   2244          lps->OutSize += newDatabase.PackSizes[startPackIndex];
   2245        lps->InSize += curUnpackSize;
   2246      }
   2247      
   2248      newDatabase.NumUnpackStreamsVector.Add(rep.NumCopyFiles);
   2249      
   2250      CNum indexInFolder = 0;
   2251      for (CNum fi = db->FolderStartFileIndex[folderIndex]; indexInFolder < numUnpackStreams; fi++)
   2252      {
   2253        if (db->Files[fi].HasStream)
   2254        {
   2255          indexInFolder++;
   2256          int updateIndex = fileIndexToUpdateIndexMap[fi];
   2257          if (updateIndex >= 0)
   2258          {
   2259            const CUpdateItem &ui = updateItems[updateIndex];
   2260            if (ui.NewData)
   2261              continue;
   2262 
   2263            UString name;
   2264            CFileItem file;
   2265            CFileItem2 file2;
   2266            GetFile(*db, fi, file, file2);
   2267 
   2268            if (ui.NewProps)
   2269            {
   2270              UpdateItem_To_FileItem2(ui, file2);
   2271              file.IsDir = ui.IsDir;
   2272              name = ui.Name;
   2273            }
   2274            else
   2275              db->GetPath(fi, name);
   2276 
   2277            /*
   2278            file.Parent = ui.ParentFolderIndex;
   2279            if (ui.TreeFolderIndex >= 0)
   2280              treeFolderToArcIndex[ui.TreeFolderIndex] = newDatabase.Files.Size();
   2281            if (totalSecureDataSize != 0)
   2282              newDatabase.SecureIDs.Add(ui.SecureIndex);
   2283            */
   2284            newDatabase.AddFile(file, file2, name);
   2285          }
   2286        }
   2287      }
   2288    }
   2289 
   2290 
   2291    // ---------- Compress files to new solid blocks ----------
   2292 
   2293    unsigned numFiles = group.Indices.Size();
   2294    if (numFiles == 0)
   2295      continue;
   2296    CRecordVector<CRefItem> refItems;
   2297    refItems.ClearAndSetSize(numFiles);
   2298    // bool sortByType = (options.UseTypeSorting && isSoid); // numSolidFiles > 1
   2299    bool sortByType = options.UseTypeSorting;
   2300    
   2301    unsigned i;
   2302 
   2303    for (i = 0; i < numFiles; i++)
   2304      refItems[i] = CRefItem(group.Indices[i], updateItems[group.Indices[i]], sortByType);
   2305 
   2306    CSortParam sortParam;
   2307    // sortParam.TreeFolders = &treeFolders;
   2308    sortParam.SortByType = sortByType;
   2309    refItems.Sort(CompareUpdateItems, (void *)&sortParam);
   2310    
   2311    CObjArray<UInt32> indices(numFiles);
   2312 
   2313    for (i = 0; i < numFiles; i++)
   2314    {
   2315      UInt32 index = refItems[i].Index;
   2316      indices[i] = index;
   2317      /*
   2318      const CUpdateItem &ui = updateItems[index];
   2319      CFileItem file;
   2320      if (ui.NewProps)
   2321        UpdateItem_To_FileItem(ui, file);
   2322      else
   2323        file = db.Files[ui.IndexInArchive];
   2324      if (file.IsAnti || file.IsDir)
   2325        return E_FAIL;
   2326      newDatabase.Files.Add(file);
   2327      */
   2328    }
   2329    
   2330    for (i = 0; i < numFiles;)
   2331    {
   2332      UInt64 totalSize = 0;
   2333      unsigned numSubFiles;
   2334      
   2335      const wchar_t *prevExtension = NULL;
   2336      
   2337      for (numSubFiles = 0; i + numSubFiles < numFiles && numSubFiles < numSolidFiles; numSubFiles++)
   2338      {
   2339        const CUpdateItem &ui = updateItems[indices[i + numSubFiles]];
   2340        totalSize += ui.Size;
   2341        if (totalSize > options.NumSolidBytes)
   2342          break;
   2343        if (options.SolidExtension)
   2344        {
   2345          int slashPos = ui.Name.ReverseFind_PathSepar();
   2346          int dotPos = ui.Name.ReverseFind_Dot();
   2347          const wchar_t *ext = ui.Name.Ptr(dotPos <= slashPos ? ui.Name.Len() : dotPos + 1);
   2348          if (numSubFiles == 0)
   2349            prevExtension = ext;
   2350          else if (!StringsAreEqualNoCase(ext, prevExtension))
   2351            break;
   2352        }
   2353      }
   2354 
   2355      if (numSubFiles < 1)
   2356        numSubFiles = 1;
   2357 
   2358      RINOK(lps->SetCur());
   2359 
   2360      CFolderInStream *inStreamSpec = new CFolderInStream;
   2361      CMyComPtr<ISequentialInStream> solidInStream(inStreamSpec);
   2362      inStreamSpec->Init(updateCallback, &indices[i], numSubFiles);
   2363      
   2364      unsigned startPackIndex = newDatabase.PackSizes.Size();
   2365      UInt64 curFolderUnpackSize = totalSize;
   2366      // curFolderUnpackSize = (UInt64)(Int64)-1;
   2367      
   2368      RINOK(encoder.Encode(
   2369          EXTERNAL_CODECS_LOC_VARS
   2370          solidInStream,
   2371          // NULL,
   2372          &inSizeForReduce,
   2373          newDatabase.Folders.AddNew(), newDatabase.CoderUnpackSizes, curFolderUnpackSize,
   2374          archive.SeqStream, newDatabase.PackSizes, progress));
   2375 
   2376      if (!inStreamSpec->WasFinished())
   2377        return E_FAIL;
   2378 
   2379      for (; startPackIndex < newDatabase.PackSizes.Size(); startPackIndex++)
   2380        lps->OutSize += newDatabase.PackSizes[startPackIndex];
   2381 
   2382      lps->InSize += curFolderUnpackSize;
   2383      // for ()
   2384      // newDatabase.PackCRCsDefined.Add(false);
   2385      // newDatabase.PackCRCs.Add(0);
   2386 
   2387      CNum numUnpackStreams = 0;
   2388      UInt64 skippedSize = 0;
   2389      
   2390      for (unsigned subIndex = 0; subIndex < numSubFiles; subIndex++)
   2391      {
   2392        const CUpdateItem &ui = updateItems[indices[i + subIndex]];
   2393        CFileItem file;
   2394        CFileItem2 file2;
   2395        UString name;
   2396        if (ui.NewProps)
   2397        {
   2398          UpdateItem_To_FileItem(ui, file, file2);
   2399          name = ui.Name;
   2400        }
   2401        else
   2402        {
   2403          GetFile(*db, ui.IndexInArchive, file, file2);
   2404          db->GetPath(ui.IndexInArchive, name);
   2405        }
   2406        if (file2.IsAnti || file.IsDir)
   2407          return E_FAIL;
   2408        
   2409        /*
   2410        CFileItem &file = newDatabase.Files[
   2411              startFileIndexInDatabase + i + subIndex];
   2412        */
   2413        if (!inStreamSpec->Processed[subIndex])
   2414        {
   2415          skippedSize += ui.Size;
   2416          continue;
   2417          // file.Name += ".locked";
   2418        }
   2419 
   2420        file.Crc = inStreamSpec->CRCs[subIndex];
   2421        file.Size = inStreamSpec->Sizes[subIndex];
   2422        
   2423        // if (file.Size >= 0) // test purposes
   2424        if (file.Size != 0)
   2425        {
   2426          file.CrcDefined = true;
   2427          file.HasStream = true;
   2428          numUnpackStreams++;
   2429        }
   2430        else
   2431        {
   2432          file.CrcDefined = false;
   2433          file.HasStream = false;
   2434        }
   2435 
   2436        /*
   2437        file.Parent = ui.ParentFolderIndex;
   2438        if (ui.TreeFolderIndex >= 0)
   2439          treeFolderToArcIndex[ui.TreeFolderIndex] = newDatabase.Files.Size();
   2440        if (totalSecureDataSize != 0)
   2441          newDatabase.SecureIDs.Add(ui.SecureIndex);
   2442        */
   2443        newDatabase.AddFile(file, file2, name);
   2444      }
   2445 
   2446      // numUnpackStreams = 0 is very bad case for locked files
   2447      // v3.13 doesn't understand it.
   2448      newDatabase.NumUnpackStreamsVector.Add(numUnpackStreams);
   2449      i += numSubFiles;
   2450 
   2451      if (skippedSize != 0 && complexity >= skippedSize)
   2452      {
   2453        complexity -= skippedSize;
   2454        RINOK(updateCallback->SetTotal(complexity));
   2455      }
   2456    }
   2457  }
   2458 
   2459  RINOK(lps->SetCur());
   2460 
   2461  /*
   2462  fileIndexToUpdateIndexMap.ClearAndFree();
   2463  groups.ClearAndFree();
   2464  */
   2465 
   2466  /*
   2467  for (i = 0; i < newDatabase.Files.Size(); i++)
   2468  {
   2469    CFileItem &file = newDatabase.Files[i];
   2470    file.Parent = treeFolderToArcIndex[file.Parent];
   2471  }
   2472 
   2473  if (totalSecureDataSize != 0)
   2474  {
   2475    newDatabase.SecureBuf.SetCapacity(totalSecureDataSize);
   2476    size_t pos = 0;
   2477    newDatabase.SecureSizes.Reserve(secureBlocks.Sorted.Size());
   2478    for (i = 0; i < secureBlocks.Sorted.Size(); i++)
   2479    {
   2480      const CByteBuffer &buf = secureBlocks.Bufs[secureBlocks.Sorted[i]];
   2481      size_t size = buf.GetCapacity();
   2482      if (size != 0)
   2483        memcpy(newDatabase.SecureBuf + pos, buf, size);
   2484      newDatabase.SecureSizes.Add((UInt32)size);
   2485      pos += size;
   2486    }
   2487  }
   2488  */
   2489  newDatabase.ReserveDown();
   2490 
   2491  if (opCallback)
   2492    RINOK(opCallback->ReportOperation(NEventIndexType::kNoIndex, (UInt32)(Int32)-1, NUpdateNotifyOp::kHeader));
   2493 
   2494  return S_OK;
   2495 }
   2496 
   2497 }}
	tor-browser The Tor Browser
	git clone https://git.dasho.dev/tor-browser.git
	Log \| Files \| Refs \| README \| LICENSE