Add support for reading Unicode Filename packet

[animetosho]

par2cmdline doesn't support reading the optional Unicode Filename packet.
This packet can be useful as it has a defined character set (unlike the "ASCII" File Description packet).

I'm creating this issue because a patch was submitted to add support, which I thought should be noted down (instead of being lost). I've attached a copy below.
I haven't reviewed it, but it might be useful as a starting point if anyone wants to implement and merge this functionality into par2cmdline.

par2turbo.unifilen.gitdiff.patch

Implementers may need to be wary of the Windows unicode implementation, as well as issues like #231 .
For non-Windows, assuming a UTF-8 locale seems reasonable, to avoid the need to handle many different encodings.

[thezoggy]

sharing the patch of pastebin.com/raw/gEgXBrAD so it doesnt get lost

diff --git a/src/par2repairer.cpp b/src/par2repairer.cpp
index 798c8a6..efcd1e9 100644
--- a/src/par2repairer.cpp
+++ b/src/par2repairer.cpp
@@ -563,7 +563,81 @@ bool Par2Repairer::LoadPacketsFromFile(string filename)
             packets++;
           }
         }
-      }
+        // Ad-hoc implementation for UniFileN packets. header.hash and header.setid are not going to be verified.
+        else if (header.type == (PACKETTYPE){{'P', 'A', 'R', ' ', '2', '.', '0', '\0', 'U', 'n', 'i', 'F', 'i', 'l', 'e', 'N'}})
+        {
+        // Sanitization
+          if (
+            header.length <= sizeof(PACKET_HEADER) + sizeof(MD5Hash) + 0x02 ||     // Is the packet big enough (header + fileid + one letter in utf16)
+            header.length - sizeof(PACKET_HEADER) - sizeof(MD5Hash) > 100000       // Is the packet too large (what is the longest permissible filename)
+          )
+          {
+            //goto unifilen_end;
+            throw "broken file";
+          }
+
+          typedef struct _tag_UNIFILEN_PACKET {
+            PACKET_HEADER          header;
+            MD5Hash                file_id;
+            uint8_t                filename_utf16[];
+          } UNIFILEN_PACKET;
+
+          // Allocate the packet (with a little extra so we will have NULLs after the filename)
+          // Allocate and clear the requested packet length plus the extra.
+          //u8[] packetdata = new u8[(header.length+4)];
+          uint8_t* packetdata = new uint8_t[(header.length+4)];
+          memset(packetdata, 0, (header.length+4));
+          UNIFILEN_PACKET *packet = (UNIFILEN_PACKET *)packetdata;
+
+          packet->header = header;
+
+          // Read the rest of the packet from disk
+          if ( !diskfile->Read( offset + sizeof(PACKET_HEADER), &packet->file_id, (size_t)packet->header.length - sizeof(PACKET_HEADER) ) )
+          {
+            delete packet;
+            //goto unifilen_end;
+            throw "broken file";
+          }
+
+          assert(packet != 0);
+          const MD5Hash &file_id = ((const UNIFILEN_PACKET*)packet)->file_id;
+
+          uint8_t* ptr_filename_utf16 = packet->filename_utf16;
+          size_t filename_utf16_size = (size_t)packet->header.length - ( sizeof(PACKET_HEADER) + sizeof(MD5Hash) );
+          vector<uint8_t> filename_utf16(ptr_filename_utf16, ptr_filename_utf16 + filename_utf16_size);
+
+          delete packet;
+
+          // Look up the fileid in the source file map for an existing source file entry
+          map<MD5Hash, Par2RepairerSourceFile*>::iterator sfmi = sourcefilemap.find(file_id);
+          Par2RepairerSourceFile *sourcefile = (sfmi == sourcefilemap.end()) ? 0 :sfmi->second;
+
+          // Was there an existing source file
+          if (sourcefile)
+          {
+            // Does the source file already have a description packet
+            if (sourcefile->GetUniFileN().size() != 0)
+            {
+              // does nothing as a UniFileN byte array is already set.
+            }
+            else
+            {
+              // sets a UniFileN array.
+              sourcefile->SetUniFileN(filename_utf16);
+              //sout << "[DEBUG] UniFileN | size: " << filename_utf16.size() << endl;
+            }
+          }
+          else
+          {
+            // creates a new instance of Par2RepairerSourceFile() without a description packet and a verification packet, sets the UniFileN byte array, then inserts the instance to the map.
+            sourcefile = new Par2RepairerSourceFile(NULL, NULL);
+            sourcefile->SetUniFileN(filename_utf16);
+            sourcefilemap.insert(pair<MD5Hash, Par2RepairerSourceFile*>(file_id, sourcefile));
+          }
+// End of the implementation for UniFileN packets
+//unifilen_end:
+        }
+      }
 
       // Advance to the next packet
       offset += header.length;
diff --git a/src/par2repairersourcefile.cpp b/src/par2repairersourcefile.cpp
index b853535..689ce46 100644
--- a/src/par2repairersourcefile.cpp
+++ b/src/par2repairersourcefile.cpp
@@ -68,11 +68,170 @@ void Par2RepairerSourceFile::SetVerificationPacket(VerificationPacket *_verifica
   verificationpacket = _verificationpacket;
 }
 
+// ad-hoc implementation for UniFileN, const needed?
+vector<uint8_t> Par2RepairerSourceFile::GetUniFileN()
+{
+  return unifilen;
+}
+
+bool Par2RepairerSourceFile::SetUniFileN(vector<uint8_t> arg1)
+{
+  unifilen = arg1;
+  return true;
+}
+
+vector<uint32_t> utf16le_to_unicode_codepoints(vector<uint8_t> utf16_byte_array)
+{
+  vector<uint32_t> codepoints;
+
+  for (auto iter = utf16_byte_array.begin(); iter < utf16_byte_array.end();)
+  {
+    uint32_t c1 = *iter;
+    iter++;
+    uint32_t c2 = *iter;
+    iter++;
+    uint32_t w1 = (c2 << 8) + c1; // little-endian
+    uint32_t codepoint = 0;
+
+    if ( w1 == 0x0000 )
+    {
+      // Sanitization for null-terminated and null-padded strings
+      while ( iter < utf16_byte_array.end() )
+      {
+        uint32_t c1 = *iter;
+        iter++;
+        uint32_t c2 = *iter;
+        iter++;
+        uint32_t w1 = (c2 << 8) + c1; // little-endian
+        if ( w1 != 0x0000 )
+        {
+          throw "This UTF-16 array has a non-null character trailing after the null termination, which is considered malformed.";
+        }
+      }
+      // The array is padded only with 0x0000, which is OK.
+      break;
+    }
+
+    if ( w1 < 0xD800 || 0xDFFF < w1 ) // U+0000 to U+D7FF and U+E000 to U+FFFF
+    {
+      codepoint = w1;
+    }
+    else if ( (w1 >> 10) == 0b110110 ) // Code points from U+010000 to U+10FFFF
+    {
+      uint32_t c3 = *iter;
+      iter++;
+      uint32_t c4 = *iter;
+      iter++;
+      uint32_t w2 = (c4 << 8) + c3; // little-endian
+
+      if ((w2 >> 10) != 0b110111)
+      {
+        throw "One of the surrogate pairs in the UTF-16 byte array doesn't have the latter byte with the prefix of 0b110111";
+      }
+      codepoint = ((w1 & 0x3ff) << 10) + (w2 & 0x3ff) + 0x10000;
+    }
+    else // (w1 >> 10) == 0b110111, which should never be happened.
+    {
+      throw "The prefix of 0b110111 is not allowed except for latter bytes of surrogate pairs.";
+    }
+
+    codepoints.push_back(codepoint);
+  }
+
+  return codepoints;
+}
+
+string unicode_codepoints_to_utf8_string(vector<uint32_t> codepoints)
+{
+  std::stringstream ss;
+
+  for (auto iter = codepoints.begin(); iter < codepoints.end(); /* iter++ */ )
+  {
+    uint32_t codepoint = *iter;
+    iter++;
+
+    if (0x0000 <= codepoint && codepoint <= 0x007F)
+    {
+      ss << (char)codepoint;
+    }
+    else if (0x0080 <= codepoint && codepoint <= 0x07FF)
+    {
+      char c1 = 0b1100'0000 + ( ( codepoint & 0b0111'1100'0000 ) >> 6 );
+      char c2 = 0b1000'0000 + ( ( codepoint & 0b0000'0011'1111 ) >> 0 );
+      ss << c1 << c2;
+    }
+    else if (0x0800 <= codepoint && codepoint <= 0xFFFF)
+    {
+      char c1 = 0b1110'0000 + ( ( codepoint & 0b1111'0000'0000'0000 ) >> 12 );
+      char c2 = 0b1000'0000 + ( ( codepoint & 0b0000'1111'1100'0000 ) >>  6 );
+      char c3 = 0b1000'0000 + ( ( codepoint & 0b0000'0000'0011'1111 ) >>  0 );
+      ss << c1 << c2 << c3;
+    }
+    else if (0x10000 <= codepoint && codepoint <= 0x10FFFF)
+    {
+      char c1 = 0b1111'0000 + ( ( codepoint & 0b0001'1100'0000'0000'0000'0000 ) >> 18 );
+      char c2 = 0b1000'0000 + ( ( codepoint & 0b0000'0011'1111'0000'0000'0000 ) >> 12 );
+      char c3 = 0b1000'0000 + ( ( codepoint & 0b0000'0000'0000'1111'1100'0000 ) >>  6 );
+      char c4 = 0b1000'0000 + ( ( codepoint & 0b0000'0000'0000'0000'0011'1111 ) >>  0 );
+      ss << c1 << c2 << c3 << c4;
+    }
+    else
+    {
+      throw "One of the codepoints exceeds 0x10FFFF, which is the maximum of the current Unicode specification.";
+    }
+  }
+
+  return ss.str();
+}
+
 void Par2RepairerSourceFile::ComputeTargetFileName(std::ostream &sout, std::ostream &serr, const NoiseLevel noiselevel, const string &path)
 {
   // Get a version of the filename compatible with the OS
   string filename = DescriptionPacket::TranslateFilenameFromPar2ToLocal(sout, serr, noiselevel, descriptionpacket->FileName());
 
+  // https://en.wikipedia.org/wiki/UTF-16#Examples
+  // UTF-16LE to UTF-8
+  if ( unifilen.size() != 0 )
+  {
+    try
+    {
+      auto codepoints = utf16le_to_unicode_codepoints(unifilen);
+
+      // Sanitization
+      for(auto iter = codepoints.begin(); iter < codepoints.end(); )
+      {
+        uint32_t codepoint = *iter;
+        iter++;
+        if (codepoint < 32 || codepoint == '"' || codepoint == '*' || codepoint == ':' || codepoint == '<' || codepoint == '>' || codepoint == '?' || codepoint == '|' || codepoint == '/' || codepoint == '\\')
+        {
+          // Ditches the entire array and not bothers to do sanitization in a graceful way.
+          throw "An illegai-ish character (based on DescriptionPacket::TranslateFilenameFromPar2ToLocal()) is found in the UniFileN. This UniFileN array will be ignored.";
+        }
+      }
+
+      filename = unicode_codepoints_to_utf8_string(codepoints);
+
+      if (noiselevel >= nlDebug || true) // outputs the logs anyway
+      {
+        std::stringstream ss;
+        ss << std::hex;
+        for (auto iter = codepoints.begin(); iter < codepoints.end(); )
+        {
+          uint32_t codepoint = *iter;
+          iter++;
+          ss << std::setw(4) << std::setfill('0') << codepoint << " ";
+        }
+        sout << "[DEBUG] codepoints in ComputeTargetFileName() : " << ss.str() << endl;
+        sout << "[DEBUG]   filename in ComputeTargetFileName() : " << filename << endl;
+      }
+    }
+    catch (const char* e)
+    {
+      sout << "[WARNING] " << e << endl;
+      sout << "[WARNING] The filename from FileDesc will be used as a fallback." << endl;
+    }
+  }
+
   targetfilename = path + filename;
 }
 
diff --git a/src/par2repairersourcefile.h b/src/par2repairersourcefile.h
index bfcff38..437b61e 100644
--- a/src/par2repairersourcefile.h
+++ b/src/par2repairersourcefile.h
@@ -90,6 +90,10 @@ public:
   // Set/Get "filesize on disk" needed for mt progress line
   void SetDiskFileSize();
   u64 DiskFileSize(void) const {return diskfilesize;}
+
+  vector<uint8_t> GetUniFileN();
+  bool SetUniFileN(vector<uint8_t> arg1);
+  vector<uint8_t> unifilen;
 
 protected:
   DescriptionPacket           *descriptionpacket;   // The file description packet